def nevada_timing(dated_input_path, bin_dict, output_file_path, logg):
if not os.path.exists(dated_input_path):
logg.write_error("The dated input file %s does not exist." %
(dated_input_path))
else:
reader = csv.reader(open(dated_input_path, "rb"))
for index, row in enumerate(reader):
# Skip the header line
if (index != 0):
# Get the obs time and convert to unix time
UTC_obs_date = row[1]
UTC_obs_time = row[2]
date = "%s%s%s" % (UTC_obs_date[0:4], UTC_obs_date[5:7],
UTC_obs_date[8:10])
date_time = "%s%s%s%s" % (date, UTC_obs_time[0:2],
UTC_obs_time[3:5], UTC_obs_time[6:8])
obs_time = tim.datetogmt(date_time)
# Get the noticed time and convert to unix time
date_time = "%s%s%s%s" % (date, row[-1][0:2], row[-1][3:5],
row[-1][6:8])
noticed_time = tim.datetogmt(date_time)
if noticed_time < obs_time:
logg.write_error(
"Error with date or time provided in file %s because the obs_time is greater than the noticed time."
% (dated_input_path))
continue
time_diff_minutes = (noticed_time - obs_time) / NUM_MIN_IN_SEC
time_bin = math.ceil(time_diff_minutes / BIN_BY_NUM)
if time_bin in bin_dict:
bin_dict[time_bin] += 1
else:
bin_dict[time_bin] = 1
# Get the total number of messages
total_num_msgs = 0
ordered_bin_dict = ordered_dict.OrderedDict(sorted(bin_dict.items()))
for key, value in ordered_bin_dict.iteritems():
total_num_msgs = value + total_num_msgs
# Open output file and print information
f = open(output_file_path, 'a')
for key, value in ordered_bin_dict.iteritems():
upper_min_bin = key * 5
num_hours_late = (
upper_min_bin -
(upper_min_bin % NUM_MIN_IN_HOUR)) / NUM_MIN_IN_HOUR
num_min_late = upper_min_bin % NUM_MIN_IN_HOUR
percent_total_msgs = (float(value) / total_num_msgs) * 100
msg = "%s msgs (%.2f%%): %s hr and %s min late\n" % (
value, percent_total_msgs, int(num_hours_late),
int(num_min_late))
f.write(msg)
f.close()
def sample1():
ns = Name_schema("%D_i%I_f%F_%B.%S",
["YYYYMMDD", "HH", "HHH", "RUC252b", "grb"])
print ns.get_base()
print ns.get_suffix()
print ns.get_date("20070213_i05_f000_RUC252b.grb")
print ns.get_ft("20070213_i05_f000_RUC252b.grb")
print ns.get_it("20070213_i05_f000_RUC252b.grb")
print ns.make_name("20080213", "29", "115")
print ns.make_name_utime(tim.datetogmt("2008021310"),
tim.datetogmt("2008021316"))
print ns.get_name_len()
def get_recent_files(delta_seconds, file_list):
"""
Parameters
----------
delta_seconds : only use files within delta_seconds of current time
file_list : list of files of interest
Returns
-------
list of latest files
"""
reduced_list = []
current_time = time.time()
for fname in file_list:
date = get_date(fname)
try:
value = int(date)
except ValueError:
continue
except TypeError:
continue
# Vid-000351067-00-06-2014-10-29-13-20.jpg
file_time = tim.datetogmt(date)
delta_time = abs(file_time - current_time)
print "delta_time: ", delta_time, delta_seconds
if delta_time < delta_seconds:
reduced_list.append(fname)
print "appending: ", fname
return reduced_list
def process(base_dir, out_dir):
for date in sorted(os.listdir(base_dir)):
subdir = os.path.join(base_dir, date)
for fname in sorted(os.listdir(subdir)):
file_path = os.path.join(base_dir, subdir, fname)
(base_name, yyyymmdd, hh) = wxt_file_parse(fname)
base_time = tim.datetogmt(yyyymmdd + hh)
begin_time = base_time - 1800
end_time = base_time + 1800
begin_time_string = time.strftime("%Y%m%d%H%M",
time.gmtime(begin_time))
end_time_string = time.strftime("%Y%m%d%H%M",
time.gmtime(end_time))
input_probe_message_file = file_path
probe_message_qc_file_name = "%s.%s.%s.qc.nc" % (base_name,
yyyymmdd, hh)
grid_file_name = "%s.%s.%s.grid.nc" % (base_name, yyyymmdd, hh)
stat_file_name = "%s.%s.%s.stat.nc" % (base_name, yyyymmdd, hh)
assess_file_name = "%s.%s.%s.assess.nc" % (base_name, yyyymmdd, hh)
output_probe_message_qc_file = os.path.join(
out_dir, probe_message_qc_file_name)
output_grid_file = os.path.join(out_dir, grid_file_name)
output_stat_file = os.path.join(out_dir, stat_file_name)
output_assess_file = os.path.join(out_dir, assess_file_name)
command_template = "./vdt4 -g %s -n vdt_config.cfg %s %s %s %s %s %s %s" % (
output_grid_file, begin_time_string, end_time_string,
Road_segment_file, input_probe_message_file,
output_probe_message_qc_file, output_stat_file,
output_assess_file)
os.system(command_template)
def process(base_dir, out_dir):
time_increment = 300
for date in sorted(os.listdir(base_dir)):
subdir = os.path.join(base_dir, date)
for fname in sorted(os.listdir(subdir)):
file_path = os.path.join(base_dir, subdir, fname)
(base_name, yyyymmdd, hh) = wxt_file_parse(fname)
if hh < "12":
continue
base_time = tim.datetogmt(yyyymmdd+hh)
for begin_time in range(base_time, base_time+3600, time_increment):
end_time = begin_time + time_increment
begin_time_string = time.strftime("%Y%m%d%H%M", time.gmtime(begin_time))
end_time_string = time.strftime("%Y%m%d%H%M", time.gmtime(end_time))
input_probe_message_file = file_path
yyyymmdd = begin_time_string[0:8]
hhmm = begin_time_string[8:12]
probe_message_qc_file_name = "%s.%s.%s.qc.nc" % (base_name, yyyymmdd, hhmm)
grid_file_name = "%s.%s.%s.grid.nc" % (base_name, yyyymmdd, hhmm)
stat_file_name = "%s.%s.%s.stat.nc" % (base_name, yyyymmdd, hhmm)
assess_file_name = "%s.%s.%s.assess.nc" % (base_name, yyyymmdd, hhmm)
output_probe_message_qc_file = os.path.join(out_dir, probe_message_qc_file_name)
output_grid_file = os.path.join(out_dir, grid_file_name)
output_stat_file = os.path.join(out_dir, stat_file_name)
output_assess_file = os.path.join(out_dir, assess_file_name)
command_template = "%s -g %s -n %s %s %s %s %s %s %s %s" % (Exec, output_grid_file, Config, begin_time_string, end_time_string, Road_segment_file, input_probe_message_file, output_probe_message_qc_file, output_stat_file, output_assess_file)
print command_template
os.system(command_template)
def __init__(self, base_dir):
self.base_dir = base_dir
self.file_list = sorted(os.listdir(base_dir))
self.file_times = []
self.file_time_strings = []
for fname in self.file_list:
yyyymmdd = fname[-16:-8]
hhmm = fname[-7:-3]
date_time = yyyymmdd + hhmm
file_time = tim.datetogmt(date_time)
self.file_times.append(file_time)
self.file_time_strings.append(date_time)
def get_time(self):
"""Convert date and time to Unix seconds"""
if not hasattr(self, "unix_time"):
self.unix_time = None
date = ""
self.unix_time = []
# date is in format yyyy-mm-dd
# time is in format hh:mm:ss
for index, value in enumerate(self.data_dict["Date(UTC)"]):
date = "%s%s%s" % (value[0:4], value[5:7], value[8:10])
time_string = self.data_dict["Time(UTC)"][index]
date_time = "%s%s%s%s" % (date, time_string[0:2],
time_string[3:5], time_string[6:8])
self.unix_time.append(tim.datetogmt(date_time))
return self.unix_time
def update_state_group(params):
logg = params["logg"]
dt = datetime.utcnow() - timedelta(minutes=filter_gte_min)
filter_gte = int(tim.datetogmt(dt.strftime('%Y%m%d%H%M00')))
#params["filter_gte"] = filter_gte
#Configuration required before calling group_msg.group_probe_message_files
cf = __import__(config_file, globals(), locals(), [])
name_format = "%s_format" % name_schema_base
name_patterns = "%s_patterns" % name_schema_base
out_name_schema = name_schema.Name_schema(getattr(cf, name_format),
getattr(cf, name_patterns))
for state in state_bounds:
if state_probe_key(state) in params:
group_msg.group_probe_message_files(params[state_probe_key(state)], time_interval, cdl_file, work_dir, \
group_dest_map[state], out_name_schema, logg, filter_gte)
params["update_state_group"] = True
return params
def main():
usage_str = "%prog config begin_time end_time cdl_file work_dir in_dir(s) out_dir"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 4:
parser.print_help()
sys.exit(2)
config_file = args[0]
begin_time = args[1]
end_time = args[2]
cdl_file = args[3]
work_dir = args[4]
in_dir_list = args[5:-1]
out_dir = args[-1]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
if config_file[-3:] == ".py":
config_file = config_file[:-3]
cf = __import__(config_file, globals(), locals(), [])
logg.set_suffix("%s" % cf.LOG_PY_SUFFIX)
logg.write_starting()
out_name_schema = name_schema.Name_schema(cf.umtri_format,
cf.umtri_patterns)
file_list = []
lookback_file_list = []
print "begin_time, end_time: ", begin_time, end_time
begin_unix_time = tim.datetogmt(begin_time)
end_unix_time = tim.datetogmt(end_time)
dated_out_dir = os.path.join(out_dir, begin_time[0:8])
out_file_name = out_name_schema.make_name_utime(begin_unix_time, None)
out_file_path = os.path.join(dated_out_dir, out_file_name)
begin_unix_time -= 300 # need to be more lenient
if not os.path.exists(dated_out_dir):
cmd = "mkdir %s" % (dated_out_dir)
ret = os.system(cmd)
if ret != 0:
logg.write_error("mkdir error")
logg.write_ending(1, "group_umtri_probe_msg_files.py")
for index, in_dir in enumerate(in_dir_list):
# Collect files to check
for update_time in xrange(begin_unix_time, end_unix_time, 86400):
date = time.strftime("%Y%m%d", time.gmtime(update_time))
#print update_time, date, os.path.join(in_dir, "%s/*nc" % (date))
glob_path = os.path.join(in_dir, "%s/*nc" % (date))
file_list.extend(glob.glob(glob_path))
file_list = sorted(file_list)
# Check each file to determine if it was modified within LOOKBACK_FILE_RECEIVED_INTERVAL_SEC
for fpath in file_list:
#print fpath
fname = fpath.split("/")
fname = fname[-1]
fname_split = fname.split("_")
print "fname_split: ", fname_split
mmddyyyy = fname_split[1]
#hhmmss = fname_split[2]
if "." in fname_split[2]:
split_nc = fname_split[2].split(".")
hhmmss = split_nc[0]
else:
hhmmss = fname_split[2]
date = mmddyyyy[4:8] + mmddyyyy[0:2] + mmddyyyy[2:4]
date_time = date + hhmmss
print date_time
#print date_time
#unix_date_time = tim.datetogmt(date_time)
# Need to convert MI local time to UTC unix time
if (fname_split[0] != "inrix"):
obs_time = datetime.datetime.strptime(date_time,
"%Y%m%d%H%M%S")
eastern = timezone("US/Eastern")
utc = timezone("UTC")
local_date_time = eastern.localize(obs_time)
utc_date_time = local_date_time.astimezone(utc)
unix_date_time = time.mktime(utc_date_time.timetuple())
else:
obs_time = datetime.datetime.strptime(date_time,
"%Y%m%d%H%M%S")
unix_date_time = time.mktime(obs_time.timetuple())
print "fpath time: ", unix_date_time
print end_unix_time, unix_date_time, (end_unix_time -
unix_date_time)
if ((end_unix_time - unix_date_time) <=
LOOKBACK_FILE_RECEIVED_INTERVAL_SEC) and (
(end_unix_time - unix_date_time) >= 0):
print "appending: ", fpath
lookback_file_list.append(fpath)
else:
print "NOT app: ", fpath
file_list = lookback_file_list
command_string = "ncrcat "
if file_list != []:
for fname in file_list:
if not os.path.exists(fname):
logg.write_error(
"group_umtri_probe_msg_files: input file %s does not exist"
% fname)
logg.write_ending(1, "group_umtri_probe_msg_files.py")
exit_value = 1
command_string = command_string + fname + " "
command_string = command_string + out_file_path
else:
logg.write_info("no input files were found")
logg.write_ending(0, "group_umtri_probe_msg_files.py")
exit(0)
split_str = command_string.split()
if len(split_str) > 2:
# If the output file exists, remove it and create a new one
if os.path.exists(out_file_path):
rm_cmd = "rm %s" % out_file_path
ret = log_msg.run_command(rm_cmd, logg)
if ret != 0:
logg.write_error(
"Error: The existing output file, %s, could not be removed"
% out_file_path)
logg.write_ending(1, "group_umtri_probe_msg_files.py")
return 1
ret = log_msg.run_command(command_string, logg)
if ret != 0:
logg.write_error("ncrcat error")
logg.write_ending(1, "group_umtri_probe_msg_files.py")
return 1
else:
logg.write_info("There are no files to concatenate at this time.")
logg.write_ending(0, "group_umtri_probe_msg_files.py")
def get_wxt_data(number, offset, begin_yyyymmddhhmmss, end_yyyymmddhhmmss,
out_file_base, logg):
"""
Get Weather Telematics data starting from begin_yyyymmddhhmmss and ending at end_yyyymmddhhmmss
Write data to daily files using out_file_base.yyyymmdd.xml template for file names.
Loop:
. Use urllib2 to fetch data
. Append data to appropriate daily file
. Find record number in fetched data
. Use record number to fetch next data set
. Stop when reach final data period
Wx telematics token: See WXToken above
"""
time_delta = 1
unix_end_time = calendar.timegm(
time.strptime(end_yyyymmddhhmmss, "%Y%m%d%H%M%S"))
begin_time = tim.datetogmt(begin_yyyymmddhhmmss)
for ind in range(number):
content = get_single_update(begin_yyyymmddhhmmss, end_yyyymmddhhmmss,
offset, logg)
try:
diction = json.loads(content)
#pprint_json(diction)
except ValueError:
logg.write_error("ValueError thrown parsing json")
begin_time += 300
offset = ""
time.sleep(1)
continue
except TypeError as te:
logg.write_error("%s. Returning from get_wxt_data" % te)
return
# store observations in hourly buckets based on observation time and not insertion time
observation_sets = diction["observation_sets"]
if len(observation_sets) > 1:
insertion_time = observation_sets[-1]["insertion_time"]
unix_time = calendar.timegm(
time.strptime(insertion_time, "%Y-%m-%dT%H:%M:%SZ"))
begin_time = unix_time + time_delta
else:
begin_time += 300
if diction["errors"] != None:
logg.write_error("errors in diction: %s" % diction["errors"])
continue
try:
out_file = "%s.%s.%s.json" % (out_file_base,
begin_yyyymmddhhmmss[0:8],
begin_yyyymmddhhmmss[8:])
logg.write_debug("out_file: %s" % out_file)
fp_out = file(out_file, "w")
fp_out.write(content)
fp_out.close()
if ind % MAX_QUERY_PER_SECOND == 0:
time.sleep(.2)
except ValueError:
logg.write_error("bad offset value %s" % offset)
return
if begin_time >= unix_end_time:
logg.write_debug(
"begin_time is less than unix end_time so returning")
return
begin_yyyymmddhhmmss = time.strftime("%Y%m%d%H%M%S",
time.gmtime(begin_time))
def main():
usage_str = "%prog config cdl_file work_dir in_dir(s) out_dir"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
parser.add_option("-d",
"--days_lookback",
dest="days_lookback",
help="lookback specified number of days")
parser.add_option("-b",
"--begin_date",
dest="begin_date",
help="start at particular date yyyymmddhh")
parser.add_option("-e",
"--end_date",
dest="end_date",
help="end at particular date yyyymmddhh")
parser.add_option(
"-D",
"--date",
dest="user_date",
help=
"replace current time with a particular date in the form yyyymmddhh")
(options, args) = parser.parse_args()
if len(args) < 4:
parser.print_help()
sys.exit(2)
config_file = args[0]
cdl_file = args[1]
work_dir = args[2]
in_dir_list = args[3:-1]
out_dir = args[-1]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
if config_file[-3:] == ".py":
config_file = config_file[:-3]
cf = __import__(config_file, globals(), locals(), [])
days_lookback = 0
if options.days_lookback:
days_lookback = int(options.days_lookback)
user_date = -1
if options.user_date:
user_date = tim.datetogmt(options.user_date)
begin_date = None
end_date = None
if options.begin_date:
begin_date = tim.datetogmt(options.begin_date)
if options.end_date:
end_date = tim.datetogmt(options.end_date)
if (begin_date != None and end_date == None) or (begin_date == None
and end_date != None):
print "Error: both begin_date and end_date must be set or neither should be set"
sys.exit(2)
logg.set_suffix("%s" % cf.LOG_PY_SUFFIX)
logg.write_starting()
out_name_schema = name_schema.Name_schema(cf.nevada_format,
cf.nevada_patterns)
file_list = []
for index, in_dir in enumerate(in_dir_list):
if begin_date != None:
for update_time in xrange(begin_date, end_date, 86400):
date = time.strftime("%Y%m%d", time.gmtime(update_time))
glob_path = os.path.join(in_dir, "%s/*nc" % (date))
file_list.extend(glob.glob(glob_path))
elif days_lookback == 0:
curr_time = time.time()
if user_date != 1:
curr_time = user_date
date = time.strftime("%Y%m%d", time.gmtime(curr_time))
glob_path = os.path.join(in_dir, "%s/*.nc" % date)
file_list = glob.glob(glob_path)
else:
curr_time = time.time()
if user_date != -1:
curr_time = user_date
for day in range(days_lookback):
date = time.strftime("%Y%m%d",
time.gmtime(curr_time - day * 86400))
glob_path = os.path.join(in_dir, "%s/*nc" % (date))
file_list.extend(glob.glob(glob_path))
final_ret = 0
for fname in file_list:
ret_value = group_nevada_probe_message_files(fname, TIME_INTERVAL,
cdl_file, work_dir,
out_dir, out_name_schema,
logg)
if ret_value != 0:
logg.write_warning("problem occurred while processing %s" % fname)
final_ret = ret_value
else:
logg.write_info("successfully finished processing file %s" % fname)
logg.write_ending(0, "group_nevada_probe_message_files.py")
sys.exit(final_ret)
def create_little_r(omit_wind_speed_dir, road_data_in_file, little_r_out_file):
"""
Read in road_data_in_file and output little_r_out_file
"""
print "before road_data: reading %s" % road_data_in_file
road_data = RoadData(road_data_in_file)
# Sample road data file name road_data.20140331.0135.csv
yyyymmddhhmm = road_data_in_file[-17:-9] + road_data_in_file[-8:-4]
identifier = "road segment"
name = "simulation"
platform = 'vehicle'
source = 'data simulator'
num_valid_field = 5 # wait on this for Jared
num_error = 0
num_warning = 0
num_duplicates = 0
is_sounding = "F"
bogus = "F"
discard = "F"
utc_seconds = tim.datetogmt(yyyymmddhhmm)
tms = time.gmtime(utc_seconds)
julian = tms.tm_yday
date = time.strftime("%Y%m%d%H%M%S", tms)
sea_level_pressure = LITTLE_R_MISSING
sea_level_pressure_qc = LITTLE_R_MISSING
ref_pressure = LITTLE_R_MISSING
ref_pressure_qc = LITTLE_R_MISSING
ground_temp = LITTLE_R_MISSING
ground_temp_qc = LITTLE_R_MISSING
sea_surface_temp = LITTLE_R_MISSING
sea_surface_temp_qc = LITTLE_R_MISSING
surface_pressure = LITTLE_R_MISSING
surface_pressure_qc = LITTLE_R_MISSING
precip = LITTLE_R_MISSING
precip_qc = LITTLE_R_MISSING
max_temp = LITTLE_R_MISSING
max_temp_qc = LITTLE_R_MISSING
min_temp = LITTLE_R_MISSING
min_temp_qc = LITTLE_R_MISSING
night_min_temp = LITTLE_R_MISSING
night_min_temp_qc = LITTLE_R_MISSING
pressure_change_3h = LITTLE_R_MISSING
pressure_change_3h_qc = LITTLE_R_MISSING
pressure_change_24h = LITTLE_R_MISSING
pressure_change_24h_qc = LITTLE_R_MISSING
cloud_cover = LITTLE_R_MISSING
cloud_cover_qc = LITTLE_R_MISSING
ceiling = LITTLE_R_MISSING
ceiling_qc = LITTLE_R_MISSING
sequence_num = 1
fp_out = open(little_r_out_file, "w")
"""
for x in [latitude, longitude, identifier, name, platform, source, elevation, num_valid_field, num_error, num_warning, sequence_num, num_duplicates, bogus, discard, utc_seconds, julian, date, sea_level_pressure, sea_level_pressure_qc, ref_pressure, ref_pressure_qc, ground_temp, ground_temp_qc, sea_surface_temp, sea_surface_temp_qc, surface_pressure, surface_pressure_qc, precip, precip_qc, max_temp, max_temp_qc, min_temp, min_temp_qc, night_min_temp, night_min_temp_qc, pressure_change_3h, pressure_change_3h_qc, pressure_change_24h, pressure_change_24h_qc, cloud_cover, cloud_cover_qc, ceiling, ceiling_qc]:
print(type(x))
"""
for index in range(len(road_data.table["case"])):
latitude = float(road_data.table['lat'][index])
longitude = float(road_data.table['lon'][index])
elevation = float(road_data.table['model_geopotential_height'][index]) # getting height from RTMA surface geopotential height field
little_r.write_little_r_header(latitude, longitude, identifier, name, platform, source, elevation, num_valid_field, num_error, num_warning, sequence_num, num_duplicates, is_sounding, bogus, discard, utc_seconds, julian, date, sea_level_pressure, sea_level_pressure_qc, ref_pressure, ref_pressure_qc, ground_temp, ground_temp_qc, sea_surface_temp, sea_surface_temp_qc, surface_pressure, surface_pressure_qc, precip, precip_qc, max_temp, max_temp_qc, min_temp, min_temp_qc, night_min_temp, night_min_temp_qc, pressure_change_3h, pressure_change_3h_qc, pressure_change_24h, pressure_change_24h_qc, cloud_cover, cloud_cover_qc, ceiling, ceiling_qc, fp_out)
pressure = road_data.table["nss_bar_press_mean"][index]
if pressure == MISSING:
pressure = road_data.table["model_bar_press"][index]
pressure = hPa_to_Pa(MISSING, pressure)
pressure_qc = 0
height = road_data.table["model_geopotential_height"][index]
height_qc = 0
if height == MISSING:
height = LITTLE_R_MISSING
height_qc = LITTLE_R_MISSING
air_temperature = road_data.table["nss_air_temp_mean"][index]
if air_temperature == MISSING:
air_temperature = road_data.table["model_air_temp"][index]
air_temperature = celsius_to_kelvin(MISSING, air_temperature)
air_temperature_qc = 0
dew_point = road_data.table["nss_dew_temp_mean"][index]
if dew_point == MISSING:
dew_point = road_data.table["model_dewpoint"][index]
dew_point = celsius_to_kelvin(MISSING, dew_point)
dew_point_qc = 0
if omit_wind_speed_dir:
wind_speed = MISSING
wind_dir = MISSING
wind_speed_qc = 0
wind_dir_qc = 0
u = MISSING
v = MISSING
u_qc = 0
v_qc = 0
else:
wind_speed_qc = 0
wind_speed = road_data.table["nss_wind_speed_mean"][index]
if wind_speed == MISSING:
wind_speed = road_data.table["model_wind_speed"][index]
if wind_speed == MISSING:
wind_speed = LITTLE_R_MISSING
wind_speed_qc = LITTLE_R_MISSING
wind_dir = road_data.table["nss_wind_dir_mean"][index]
wind_dir_qc = 0
if wind_dir == MISSING:
wind_dir = road_data.table["model_wind_dir"][index]
if wind_dir == MISSING:
wind_dir = LITTLE_R_MISSING
wind_dir_qc = LITTLE_R_MISSING
if wind_dir != LITTLE_R_MISSING and wind_speed != LITTLE_R_MISSING:
(u, v) = speed_dir_to_uv(wind_speed, wind_dir)
u_qc = 0
v_qc = 0
else:
u = LITTLE_R_MISSING
v = LITTLE_R_MISSING
u_qc = LITTLE_R_MISSING
v_qc = LITTLE_R_MISSING
rh = LITTLE_R_MISSING
rh_qc = LITTLE_R_MISSING
thickness = LITTLE_R_MISSING
thickness_qc = LITTLE_R_MISSING
little_r.write_little_r_obs(pressure, pressure_qc, height, height_qc, air_temperature, air_temperature_qc, dew_point, dew_point_qc, wind_speed, wind_speed_qc, wind_dir, wind_dir_qc, u, u_qc, v, v_qc, rh, rh_qc, thickness, thickness_qc, fp_out)
# write end data record
pressure = -777777
height = -777777
# write end report record
little_r.write_little_r_obs(pressure, pressure_qc, height, height_qc, air_temperature, air_temperature_qc, dew_point, dew_point_qc, wind_speed, wind_speed_qc, wind_dir, wind_dir_qc, u, u_qc, v, v_qc, rh, rh_qc, thickness, thickness_qc, fp_out)
fp_out.write("-777777-777777-777777\n")
sequence_num += 1
fp_out.close()
def get_latest_file_using_time(time_delta, date_string, directory):
"""Gets the latest file prior to and within time_delta seconds of the specified time in date_string of form yyyymmddhhmm. Note that data_time is in unix seconds"""
time_value = tim.datetogmt(date_string)
yyyymmdd = date_string[0:8]
dirlist = os.listdir(directory)
index_dict = {}
for name in dirlist:
if name[0:5] == 'index' or name[0:9] == 'Processed' and name[-1] != '~':
spl = name.split('.')
date = spl[1]
index_dict[date] = name
sorted_dates = sorted(index_dict.keys())
ind = bisect.bisect_left(sorted_dates, yyyymmdd)
index_try_list = []
if ind < len(sorted_dates):
if yyyymmdd == sorted_dates[ind]:
if ind > 0:
index_try_list = [ind, ind-1]
else:
index_try_list = [ind]
else:
previous_date = time.strftime("%Y%m%d", time.gmtime(time_value - 86400))
if previous_date == sorted_dates[ind]:
index_try_list = [ind]
elif ind == len(sorted_dates):
previous_date = time.strftime("%Y%m%d", time.gmtime(time_value - 86400))
if previous_date == sorted_dates[ind-1]:
index_try_list = [ind-1]
# look in potential index files for dates near date_string
return_path = ""
return_date = ""
for ind in index_try_list:
index_file = index_dict[sorted_dates[ind]]
path = os.path.join(directory, index_file)
date_list = []
file_list = []
# get date and file list in index file
with open(path) as in_fp:
lines = in_fp.readlines()
for line in lines:
spl = line.split()
if len(spl) >= 2:
file_components = spl[0].split('.')
if len(file_components) >= 3:
date_list.append(file_components[1] + file_components[2])
file_list.append(spl[0])
if len(date_list) == 0:
continue
# look for date_string in date_list or nearest previous date
ind = bisect.bisect_left(date_list, date_string)
if ind < len(date_list):
if date_string == date_list[ind]:
return_path = os.path.join(directory, date_string[0:8], file_list[ind])
return_date = date_list[ind]
break
else:
if ind > 0:
# check time difference
file_time_value = tim.datetogmt(date_list[ind-1])
if time_value - file_time_value < time_delta:
return_path = os.path.join(directory, date_list[ind-1][0:8], file_list[ind-1])
return_date = date_list[ind-1]
break
else:
# ind == len(date_list)
file_time_value = tim.datetogmt(date_list[ind-1])
if time_value - file_time_value < time_delta:
return_path = os.path.join(directory, date_list[ind-1][0:8], file_list[ind-1])
return_date = date_list[ind-1]
break
return (return_path, return_date)
def main():
usage_str = "%prog config_file in_dir out_dir"
parser = OptionParser(usage=usage_str)
parser.add_option("-n",
"--nothing",
dest="nothing",
action="store_true",
help="nothing at all")
parser.add_option("-c",
"--config_file",
dest="config",
help="read specified configuration file")
parser.add_option("-b",
"--begin_time",
dest="begin_time",
help="start at particular time yyyymmddhhmm")
parser.add_option("-e",
"--end_time",
dest="end_time",
help="end at particular time yyyymmddhhmm (exclusive)")
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 2:
parser.print_help()
sys.exit(2)
config_file = args[0]
in_dir = args[1]
out_dir = args[2]
cf = __import__(config_file, globals(), locals(), [])
pf = processed_file.ProcessedFile(cf.paths.PROCESSED_DIR,
cf.paths.HISTORY_PROCESSED_BASE)
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
begin_time = None
end_time = None
if options.begin_time:
begin_time = tim.datetogmt(options.begin_time)
if options.end_time:
end_time = tim.datetogmt(options.end_time)
if (begin_time != None and end_time == None) or (begin_time == None
and end_time != None):
print "Error: both begin_time and end_time must be set or neither should be set"
sys.exit(2)
if (begin_time > end_time):
print "Error: end_time must be greater than begin_time"
sys.exit(2)
logg.set_suffix("%s" % cf.paths.LOG_PY_SUFFIX)
logg.write_starting("create_probe_msg_history.py")
create_probe_msg_history(cf, in_dir, out_dir, begin_time, end_time, logg)
logg.write_ending(0, "create_probe_msg_history.py")