def get_utime(self, fname): """ get unix time using date, hhmm information """ date = self.get_date(fname) it = self.get_it(fname) it_tuple = tim.datetotuple("%s%s" % (date, it)) it_utime = tim.mkgmtime(it_tuple) return it_utime
def get_utime(self, fname): # get utime of date date = self.get_date(fname) it = self.get_it(fname) ft = self.get_ft(fname) it_tuple = tim.datetotuple("%s%s" % (date, it)) it_utime = tim.mkgmtime(it_tuple) fh = int(ft) ft_utime = it_utime + 3600 * fh return (it_utime, ft_utime)
def grid_convert(pres_time, infile_path, var_names_file, cdl_file, output_dir, out_index_file, logfile, input_base, output_base, testing): success = 0 logfile_path = logfile.get_log_path() # Construct output path names. ifo = name_schema.Fcst_fname(input_base, "nc") infile = os.path.basename(infile_path) file_date_str = ifo.get_date(infile) file_date_tup = tim.datetotuple(file_date_str) file_date = tim.mkgmtime(file_date_tup) output_path = os.path.join(output_dir, file_date_str) if (not os.path.exists(output_path)): logfile.write_time("Info: Executing mkdir -p %s\n" % output_path) if not testing: ret = os.system("mkdir -p %s 2> /dev/null" % output_path) if (ret != 0): logfile.write_time("Error: Unable to make directory.\n") return (0) ofo = name_schema.Fcst_fname(output_base, "nc") outfile = ofo.make_name(file_date_str, ifo.get_it(infile), ifo.get_ft(infile)) outfile_path = os.path.join(output_path, outfile) if (out_index_file.file_processed(outfile, file_date)): logfile.write_time("Info: File %s already exists.\n" % outfile) return (1) logfile_arg = "" if (logfile_path != ""): logfile_arg = "-l %s" % logfile_path command = "%s %s %s %s %s %s" % (convert_command, infile_path, var_names_file, cdl_file, outfile_path, logfile_arg) logfile.write_time("Info: Executing %s\n" % command) if not testing: ret = os.system(command) if (ret == 0): write_str = "%s %d" % (outfile, int(pres_time)) out_index_file.write(write_str, file_date) success = 1 else: logfile.write_time("Error: Unable to convert to file %s. \n" % outfile) return (success)
def get_concat_meso_file(concat_meso_dir, date_string): ttup = time.strptime(date_string, "%Y%m%d.%H%M%S") time_val = tim.mkgmtime(ttup) index = index_file.Index_file(concat_meso_dir, sys_path.Index_base, sys_path.Index_period, "r", sys_path.Index_wait) f, d = get_file.get_file(index, time_val - (12 * 3600), time_val, 1, 1) if (len(f) == 0): obs_file = "none" else: obs_file = f[0] return obs_file
def get_utime(self, fname): """get issue and forecast utc times""" date = self.get_date(fname) it = self.get_it(fname) if it != "": date_tuple = tim.datetotuple("%s" % date) date_utime = tim.mkgmtime( date_tuple) # The unix time of the date in the file's name it_utime = date_utime + tim.tp2sec(self.get_it_pattern(), it) else: it_utime = None ft = self.get_ft(fname) if it != "" and ft != "": ft_utime = it_utime + tim.tp2sec(self.get_ft_pattern(), ft) else: ft_utime = None return (it_utime, ft_utime)
def get_obs_time(obs_time_str): year = obs_time_str[0:4] month = obs_time_str[5:7] mday = obs_time_str[8:10] hour = obs_time_str[11:13] minute = obs_time_str[14:16] second = obs_time_str[17:19] # Set up strings for user-supplied date and real-time obs_date = "%s%s%s" % (year, month, mday) obs_time = "%s%s%s" % (hour, minute, second) # Create a Unix time from date/time info dt = "%s.%s" % (obs_date, obs_time) ttup = time.strptime(dt, "%Y%m%d.%H%M%S") time_val = tim.mkgmtime(ttup) return (time_val)
def main(): parser = OptionParser( usage= "%prog [options] rwh_config_file fcst_road_seg_file vdt_road_seg_file state forecast_hours_out" ) parser.add_option( "-d", "--date", dest="date", help="run rwh application for specified date (YYYYMMDD.HHMM)") parser.add_option("-l", "--log", dest="log", help="base name of log file") (options, args) = parser.parse_args() if len(args) < 5: parser.print_help() sys.exit(2) config_file = args[0] fcst_road_seg_file = args[1] vdt_road_seg_file = args[2] state = args[3] fcst_hours_out = int(args[4]) ret = 0 if options.log: logg = log_msg.LogMessage(options.log, "pyl") else: logg = log_msg.LogMessage("") logg.write_starting("run_rwh.py") # Resolve date and time # Get optional date and time if we have it if options.date: date_time = options.date curr_date = date_time[:8] curr_hhmmss = date_time[9:] + "00" curr_hhmm = date_time[9:] else: curr_date = time.strftime("%Y%m%d", time.gmtime(time.time())) curr_hhmmss = time.strftime("%H%M%S", time.gmtime(time.time())) curr_hhmm = time.strftime("%H%M", time.gmtime(time.time())) dt = "%s.%s" % (curr_date, curr_hhmmss) ttup = time.strptime(dt, "%Y%m%d.%H%M%S") time_val = tim.mkgmtime(ttup) # Get the latest road-weather forecast file rdwx_dir = "%s/%s" % (vii_paths.FORECAST_DIR, "rdwx_fcst") index = index_file.Index_file(rdwx_dir, "index", 24, "r", 0) f, d = get_file.get_file(index, time_val - 7200, time_val, 1, 1) road_wx_opt_str = "" if (len(f) == 0): logg.write_warning( "No recent road-weather forecast file found in %s." % rdwx_dir) else: rdwx_file = f[0] road_wx_opt_str = "-w %s" % rdwx_file # Get the latest road-conditions forecast file road_cond_dir = "%s/%s" % (vii_paths.FORECAST_DIR, "merge_rec_tmt") index = index_file.Index_file(road_cond_dir, "index", 24, "r", 0) f, d = get_file.get_file(index, time_val - 7200, time_val, 1, 1) road_cond_opt_str = "" if (len(f) == 0): logg.write_warning("No recent road-cond forecast file found in %s." % road_cond_dir) else: road_cond_file = f[0] road_cond_opt_str = "-r %s" % road_cond_file # Get the latest VDT segment-statistics file # Look back for the latest 5 minute file # Only look back 16 mintues to get the latest run # otherwise we do not run rwh # nearest_1min_time = (int(time_val) / 60) * 60 max_lookback_time = nearest_1min_time - 1860 loop_time = nearest_1min_time found = 0 seg_stats_state_dir = "%s/%s_vdt_output" % (vii_paths.PROCESSED_DIR, state) output_date = "" output_hhmm = "" while (loop_time >= max_lookback_time): file_date = time.strftime("%Y%m%d", time.gmtime(loop_time)) file_hhmm = time.strftime("%H%M", time.gmtime(loop_time)) seg_stats_file = "%s/%s/segment_statistics.%s.%s.nc" % ( seg_stats_state_dir, file_date, file_date, file_hhmm) if os.path.exists(seg_stats_file): found = 1 output_date = file_date output_hhmm = file_hhmm break loop_time = loop_time - 60 seg_stats_opt_str = "" if (not found): logg.write_warning( "No recent segment-statistics file found in %s, running with just forecast files." % seg_stats_state_dir) else: seg_stats_opt_str = "-s %s" % seg_stats_file # If we don't have any of the input files, exit. if (road_wx_opt_str == "" and road_cond_opt_str == "" and seg_stats_opt_str == ""): logg.write_error( "No recent road-wx, road-cond or seg-stats file found, can't run rwh." ) logg.write_ending(exit_status=1, msg="run_rwh.py") sys.exit(1) # Setup command to run rwh # begin_time_val = time_val end_time_val_init = time_val + (fcst_hours_out * 3600) end_time_val = (int(end_time_val_init) / 3600) * 3600 rwh_begin_time_string = time.strftime("%Y%m%d%H%M", time.gmtime(begin_time_val)) rwh_end_time_string = time.strftime("%Y%m%d%H%M", time.gmtime(end_time_val)) output_state_dir = "%s/%s_rwh_output" % (vii_paths.PROCESSED_DIR, state) output_dir = "%s/%s" % (output_state_dir, output_date) output_file = "%s_rwh.%s.%s.nc" % (state, output_date, output_hhmm) output_path = "%s/%s" % (output_dir, output_file) # Make dated output dir if necessary mkdir_path(output_dir) cmd_str = "rwh45 %s %s %s %s %s %s -l %s %s %s %s" % ( config_file, rwh_begin_time_string, rwh_end_time_string, fcst_road_seg_file, vdt_road_seg_file, output_path, options.log, road_wx_opt_str, road_cond_opt_str, seg_stats_opt_str) # Setup Processed file object proc_file = processed_file.ProcessedFile(output_state_dir, "Processed") # Run the command line ret = run_cmd(cmd_str, logg) if ret == 0: proc_file.write_processed(curr_date, output_file, processed_file.PROC_SUCCESS) else: logg.write_error("run_cmd() failed for rwh command_str.") logg.write_ending(exit_status=ret, msg="run_rwh.py") return ret
age = "86400 86400" obs_dir = "None" concat_meso_dir = "%s/%s" % (rctm_path.Rctm_root_dir, "concat_meso") output_dir = "%s/%s" % (rctm_path.Rctm_root_dir, "fec_rc_cur_tmt") log_base = "%s/%s" % (rctm_path.Log_dir, "fec_rc_cur_tmt") params_file = "%s/%s" % (Params_dir, "fec.rc.params") try: date_time = os.environ['TEST_MODE_TIME'] date_tup = time.strptime("%s" % (date_time), "%Y%m%d.%H%M%S") fcst_time = tim.mkgmtime(date_tup) date_str = "-d %s" % (date_time) except: fcst_time = time.time() date_tup = time.gmtime(fcst_time) date_time = time.strftime("%Y%m%d.%H%M%S", date_tup) date_str = '' fcst_time = (int(fcst_time) / 3600) * 3600 static_files = "%s %s %s" % (site_list, nbr_file, cdl_file) concat_meso_file = obs.get_concat_meso_file(concat_meso_dir, date_time) if (os.path.exists(concat_meso_file)): concat_meso_str = "-f -c %s" % concat_meso_file else:
log_str = "" # Set up strings for user-supplied date and real-time if utime: curr_date = utime[:8] curr_time = utime[9:] + "00" else: curr_date = time.strftime("%Y%m%d", time.gmtime(time.time())) curr_time = time.strftime("%H%M%S", time.gmtime(time.time())) # Create a Unix time from date/time info dt = "%s.%s" % (curr_date, curr_time) ttup = time.strptime(dt,"%Y%m%d.%H%M%S") time_val = tim.mkgmtime(ttup) fcst_hr = curr_time[:2] cmd = "dump_fcst_data" # ********************** Dump Mesh Forecast ******************************** # Get latest mesh input files index = index_file.Index_file(wx_input_dir, sys_path.Index_base, sys_path.Index_period, "r", sys_path.Index_wait) f, d = get_file.get_file(index, time_val - 86400, time_val, 1, 1) # If no input file, exit if (len(f) == 0): logf.write_time("Warning: No recent weather forecast file found.\n") wx_fcst_file = "" else:
def get_utime(self, fname): """ get unix time of fname """ date = self.get_date(fname) date_tuple = tim.datetotuple(date) date_utime = tim.mkgmtime(date_tuple) return date_utime
def main(): parser = OptionParser( usage="%prog [options] cdf_to_csv_config_file csv_output_dir") parser.add_option( "-d", "--date", dest="date", help= "run cdf_to_csv_dicast application for specified date (YYYYMMDD.HHMM)") parser.add_option("-l", "--log", dest="log", help="base name of log file") (options, args) = parser.parse_args() if len(args) < 2: parser.print_help() sys.exit(2) config_file = args[0] output_dir = args[1] ret = 0 if options.log: logg = log_msg.LogMessage(options.log, "pyl") else: logg = log_msg.LogMessage("") logg.write_starting("run_cdf_to_csv_dicast.py") # Resolve date and time # Get optional date and time if we have it if options.date: date_time = options.date curr_date = date_time[:8] curr_hhmmss = date_time[9:] + "00" curr_hhmm = date_time[9:] else: curr_date = time.strftime("%Y%m%d", time.gmtime(time.time())) curr_hhmmss = time.strftime("%H%M%S", time.gmtime(time.time())) curr_hhmm = time.strftime("%H%M", time.gmtime(time.time())) dt = "%s.%s" % (curr_date, curr_hhmmss) ttup = time.strptime(dt, "%Y%m%d.%H%M%S") time_val = tim.mkgmtime(ttup) # Make output dirs if necessary tmp_output_dir = "%s/tmp" % (output_dir) mkdir_path(output_dir) mkdir_path(tmp_output_dir) # Run cdf_to_cvs_dicast for all three states # then concatenate states = ["mi", "mn", "nv"] cat_str = "" for state in states: cap_state = "" if state == "mi": cap_state = "MI" elif state == "mn": cap_state = "MN" elif state == "nv": cap_state = "NV" fcst_site_list = "%s/%s_logicast_segment_sites_only.asc" % ( vii_paths.SITE_LIST_DIR, cap_state) vdt_seg_file = "%s/%s_roads.20131111.nc" % (vii_paths.CDL_DIR, state) seg_stats_dir = "%s/%s_vdt_output" % (vii_paths.PROCESSED_DIR, state) #print "fcst_site_list: %s" % fcst_site_list #print "vdt_seg_file: %s" % vdt_seg_file #print "seg_stats_dir: %s" % seg_stats_dir # Get the latest VDT segment-statistics file # Look back for the latest 5 minute file # Only look back 30 mintues to get the latest run # nearest_5min_time = (int(time_val) / 300) * 300 max_lookback_time = nearest_5min_time - (30 * 60) loop_time = nearest_5min_time found = 0 while (loop_time >= max_lookback_time): file_date = time.strftime("%Y%m%d", time.gmtime(loop_time)) file_hhmm = time.strftime("%H%M", time.gmtime(loop_time)) file_base = "segment_statistics.%s.%s" % (file_date, file_hhmm) seg_stats_file = "%s/%s/%s.nc" % (seg_stats_dir, file_date, file_base) tmp_output_file = "%s/%s_%s.csv" % (tmp_output_dir, state, file_base) if os.path.exists(seg_stats_file): found = 1 break loop_time = loop_time - 60 if (found): cmd_str = "cdf_to_csv_dicast %s %s %s %s %s" % ( config_file, fcst_site_list, vdt_seg_file, seg_stats_file, tmp_output_file) # Run the command line ret = run_cmd(cmd_str, logg) if ret == 0: cat_str = cat_str + tmp_output_file + " " if ret != 0: logg.write_error( "run_cmd() failed for cdf_to_csv_dicast cmd_str.") else: logg.write_warning( "No recent segment-statistics file found in %s, not running cdf_to_csv_dicast for %s" % (seg_stats_dir, state)) #print "cat_str = ", cat_str # Set up paths and command for concatenating the state csv files output_file = "all_segment_statistics.%s.%s.csv" % (curr_date, curr_hhmm) output_path = "%s/%s" % (output_dir, output_file) cmd_str = "cat %s > %s" % (cat_str, output_path) # Run the command line ret = run_cmd(cmd_str, logg) if ret != 0: logg.write_error("run_cmd() failed for cat cmd_str.") logg.write_ending(exit_status=ret, msg="run_cdf_to_csv_dicast.py") return ret
Cdl_dir = "%s/%s" % (rctm_path.Rctm_root_dir, "static_data/cdl") Params_dir = "%s/%s" % (rctm_path.Rctm_root_dir, "static_data/params") log_base = "%s/%s_rc" % (rctm_path.Log_dir, rctm_path.Dump_meso_base) meso_input_dir = "%s/%s" % (rctm_path.Rctm_root_dir, "dec_data/obs/mesonet") output_dir = "%s/%s" % (rctm_path.Rctm_root_dir, "dump_meso") site_var_file = "%s/%s" % (Site_list_dir, "dump_meso_site_var_file.asc") site_list = "%s/%s" % (Site_list_dir, "road_cond_sites.asc") try: date_time = os.environ['TEST_MODE_TIME'] date_tup = time.strptime("%s" % (date_time),"%Y%m%d.%H%M%S") cur_time = tim.mkgmtime(date_tup) except: cur_time = time.time() cur_date = time.strftime("%Y%m%d.%H%M", time.gmtime(cur_time)) one_hour_time = cur_time - 3600 one_hour_date = time.strftime("%Y%m%d.%H%M", time.gmtime(one_hour_time)) # Run for two hours back #command = "%s -d %s -l %s %s %s %s %s" % (proc_script, two_hour_date, log_base, meso_input_dir, output_dir, site_var_file, site_list) #ret = os.system(command) # Run for one hour back command = "%s -d %s -l %s %s %s %s %s" % (proc_script, one_hour_date, log_base, meso_input_dir, output_dir, site_var_file, site_list) ret = os.system(command)
def main(): parser = OptionParser(usage="%prog [options] ftp_input_dir output_dir") parser.add_option("-l", "--log", dest="log", help="base name of log file") (options, args) = parser.parse_args() if len(args) < 2: parser.print_help() sys.exit(2) input_dir = args[0] output_dir = args[1] input_file_name = "AkDOT_Road_Temps.csv" input_path = "%s/%s" % (input_dir, input_file_name) ret = 0 if options.log: logg = log_msg.LogMessage(options.log, "pyl") else: logg = log_msg.LogMessage("") logg.write_starting("get_AK_mobile_data.py") # Resolve date and time curr_date = time.strftime("%Y%m%d", time.gmtime(time.time())) curr_hhmmss = time.strftime("%H%M%S", time.gmtime(time.time())) curr_hhmm = time.strftime("%H%M", time.gmtime(time.time())) dt = "%s.%s" % (curr_date, curr_hhmmss) ttup = time.strptime(dt, "%Y%m%d.%H%M%S") time_val = tim.mkgmtime(ttup) # Make output dir if necessary dest_dir = "%s/%s" % (output_dir, curr_date) mkdir_path(dest_dir) output_file_name = "alaska.%s.%s.csv" % (curr_date, curr_hhmm) output_path = "%s/%s" % (dest_dir, output_file_name) # Open input file and read in lines # Create a dictionary that contains the data for each site site_data = {} num_columns = 5 num_valid_lines = 0 if os.path.exists(input_path): in_fh = open(input_path, "r") for line in in_fh: line = line.rstrip('\n' + '\r') fields = line.split(',') if (len(fields) != 6): logg.write_error("Found %d fields in line %s, expecting 6" % (len(fields), line)) continue # Skip the header if (re.match("label", fields[0])): continue # Valid entries start with the site-id which is a decimal (integer) if (re.match('\d', line)): #print "line: %s" % line site = fields[0] if site not in site_data.keys(): site_data[site] = [] for x in xrange(0, num_columns): site_data[site].append([]) site_data[site][0].append(fields[1]) # Date-Time (0) site_data[site][1].append(fields[2]) # Variable-name (1) site_data[site][2].append(fields[3]) # Value (2) site_data[site][3].append(fields[4]) # Latitude (3) site_data[site][4].append(fields[5]) # Longitude (4) num_valid_lines = num_valid_lines + 1 # Check if we have any valid data, if not exit #print "num_valid_lines: %d" % num_valid_lines if (num_valid_lines == 0): logg.write_warning( "No data found in most recent input file, not creating output file" ) logg.write_ending(exit_status=0, msg="get_AK_mobile_data.py") sys.exit(0) # Loop over the data dictionary and output the data # The key is the site-id out_fh = open(output_path, "w") # Write header header_line = "VehicleID,Time(UTC),Latitude,Longitude,AirTemp,RelativeHumidity,RoadTemp,DewpointTemp" out_fh.write("%s\n" % header_line) for key in site_data.keys(): #print "key : %s" % key # Get the date-time for each site # The date-time is the first column in the dict (x = 0) # Use the first date-time row in the dict for output (y = 0) x = 0 y = 0 date_time_fields = site_data[key][x][y].split() date_fields = date_time_fields[0].split('/') month = date_fields[0] day = date_fields[1] year = date_fields[2] time_fields = date_time_fields[1].split(':') hour = time_fields[0] minute = time_fields[1] second = time_fields[2] site_dt = "%s%s%s.%s%s%s" % (year, month, day, hour, minute, second) site_ttup = time.strptime(site_dt, "%Y%m%d.%H%M%S") site_time_val = tim.mkgmtime(site_ttup) # Convert to UTC = AKST + 9hours site_time_val = site_time_val + (9 * 3600) # Get the latitude for each site # The latitude is the fourth column in the dict (x = 3) # Use the first latitude row in the dict (y = 0) x = 3 y = 0 lat = "%.4f" % float(site_data[key][x][y]) # Get the longitude for each site # The longitude is the fith column in the dict (x = 4) # Use the first longitude row in the dict (y = 0) x = 4 y = 0 lon = "%.4f" % float(site_data[key][x][y]) # Get the data value (T, rh, road-T, dewpt) # The data values is the third column in the dict (x = 2) # The first row (y = 0) is T, second row (y = 1) is rh, third row (y = 2) is road-T, fourth row (y = 3) is dewpt x = 2 T = float(site_data[key][x][0]) rh = float(site_data[key][x][1]) road_T = float(site_data[key][x][2]) dewpt = float(site_data[key][x][3]) # QC the value if (T < -70.0 or T > 70.0): T_str = "" else: T_str = "%.2f" % T if (rh < 0.0 or rh > 100.0): rh_str = "" else: rh_str = "%.0f" % rh if (road_T < -70.0 or road_T > 70.0): road_T_str = "" else: road_T_str = "%.2f" % road_T if (dewpt < -70.0 or dewpt > 70.0): dewpt_str = "" else: dewpt_str = "%.2f" % dewpt output_line = "%s,%s,%s,%s,%s,%s,%s,%s" % ( key, site_time_val, lat, lon, T_str, rh_str, road_T_str, dewpt_str) #print "%s" % output_line out_fh.write("%s\n" % output_line) out_fh.close() logg.write_ending(exit_status=ret, msg="get_AK_mobile_data.py") return ret
def main(): parser = OptionParser( usage= "%prog [options] ftp_input_dir raw_output_dir cdl_file processed_output_dir" ) parser.add_option("-l", "--log", dest="log", help="base name of log file") (options, args) = parser.parse_args() if len(args) < 4: parser.print_help() sys.exit(2) input_dir = args[0] raw_output_dir = args[1] cdl_file = args[2] proc_output_dir = args[3] ret = 0 if options.log: logg = log_msg.LogMessage(options.log, "pyl") else: logg = log_msg.LogMessage("") logg.write_starting("frmt_and_run_alaska2probe_message.py") # Resolve date and time curr_date = time.strftime("%Y%m%d", time.gmtime(time.time())) curr_hhmmss = time.strftime("%H%M%S", time.gmtime(time.time())) curr_hhmm = time.strftime("%H%M", time.gmtime(time.time())) dt = "%s.%s" % (curr_date, curr_hhmmss) ttup = time.strptime(dt, "%Y%m%d.%H%M%S") time_val = tim.mkgmtime(ttup) # Find the latest csv file in the ftp directory # Loop back 24 hours and look for a file that has a specific month/day/hour format # input_file_prefix = "AKD" found = 0 loop_time = time_val end_time = time_val - (24 * 3600) while (loop_time >= end_time): loop_year = time.strftime("%Y", time.gmtime(loop_time)) loop_month = time.strftime("%m", time.gmtime(loop_time)) loop_day = time.strftime("%d", time.gmtime(loop_time)) tmp_loop_hh = curr_hhmm = time.strftime("%H", time.gmtime(loop_time)) dec_loop_hh = int(tmp_loop_hh) loop_mm = curr_hhmm = time.strftime("%M", time.gmtime(loop_time)) #print "loop_year: %s, loop_month: %s, loop_day: %s, dec_loop_hh: %d, loop_mm: %s" % (loop_year, loop_month, loop_day, dec_loop_hh, loop_mm) input_file_name = "AKD%s%s%d%s.csv" % (loop_month, loop_day, dec_loop_hh, loop_mm) input_path = "%s/%s" % (input_dir, input_file_name) if (os.path.exists(input_path)): #print "FOUND: %s" % input_path found = 1 break loop_time = loop_time - 60 if (not found): logg.write_warning( "No recent input file found, not creating output file") logg.write_ending(exit_status=0, msg="frmt_and_run_alaska2probe_message.py") sys.exit(0) #print "input_path: %s" % input_path # Open input file and read in lines # Create a dictionary that contains the data for each site site_data = {} num_columns = 7 num_valid_lines = 0 if os.path.exists(input_path): in_fh = open(input_path, "r") for line in in_fh: line = line.rstrip('\n' + '\r') fields = line.split(',') if (len(fields) != 8): logg.write_error("Found %d fields in line %s, expecting 8" % (len(fields), line)) continue # Skip the header if (re.match("label", fields[0])): continue # Valid entries start with the site-id which is a decimal (integer) if (re.match('\d', line)): #print "line: %s" % line site = fields[0] if site not in site_data.keys(): site_data[site] = [] for x in xrange(0, num_columns): site_data[site].append([]) site_data[site][0].append(fields[1]) # Date-Time (0) site_data[site][1].append(fields[2]) # Variable-name (1) site_data[site][2].append(fields[3]) # Value (2) site_data[site][3].append(fields[4]) # Latitude (3) site_data[site][4].append(fields[5]) # Longitude (4) site_data[site][5].append(fields[6]) # Speed (5) site_data[site][6].append(fields[7]) # Heading (6) num_valid_lines = num_valid_lines + 1 # Check if we have any valid data, if not exit if (num_valid_lines == 0): logg.write_warning( "No data found in most recent input file, not creating output file" ) logg.write_ending(exit_status=0, msg="frmt_and_run_alaska2probe_message.py") sys.exit(0) # Make raw output dir if necessary raw_dest_dir = "%s/%s" % (raw_output_dir, curr_date) mkdir_path(raw_dest_dir) raw_output_file_name = "alaska.%s.csv" % curr_date raw_output_path = "%s/%s" % (raw_dest_dir, raw_output_file_name) # Check if output file exists, if it does don't write header header_flag = 1 if (os.path.exists(raw_output_path)): header_flag = 0 # Open the output file and append data to it. out_fh = open(raw_output_path, "a") # Write header header_line = "VehicleID,Time(UTC),Latitude,Longitude,Speed,Heading,AirTemp,RelativeHumidity,RoadTemp,DewpointTemp" if (header_flag): out_fh.write("%s\n" % header_line) # Loop over the data dictionary and output the data # The key is the site-id for key in site_data.keys(): #print "key : %s" % key # Get the date-time for each site # The date-time is the first column in the dict (x = 0) # Use the first date-time row in the dict for output (y = 0) x = 0 y = 0 date_time_fields = site_data[key][x][y].split() date_fields = date_time_fields[0].split('/') month = date_fields[0] day = date_fields[1] year = date_fields[2] time_fields = date_time_fields[1].split(':') hour = time_fields[0] minute = time_fields[1] second = time_fields[2] site_dt = "%s%s%s.%s%s%s" % (year, month, day, hour, minute, second) site_ttup = time.strptime(site_dt, "%Y%m%d.%H%M%S") site_time_val = tim.mkgmtime(site_ttup) # Convert to UTC = AKST + 9hours for AST # Convert to UTC = AKST + 8hours for ADT site_time_val = site_time_val + (8 * 3600) # Get the latitude for each site # The latitude is the fourth column in the dict (x = 3) # Use the first latitude row in the dict (y = 0) x = 3 y = 0 lat = "%.4f" % float(site_data[key][x][y]) # Get the longitude for each site # The longitude is the fith column in the dict (x = 4) # Use the first longitude row in the dict (y = 0) x = 4 y = 0 lon = "%.4f" % float(site_data[key][x][y]) # Get the speed for each site # The speed is the sixth column in the dict (x = 5) # Use the first speed row in the dict (y = 0) x = 5 y = 0 speed = float(site_data[key][x][y]) # Get the heading for each site # The heading is the seventh column in the dict (x = 6) # Use the first speed row in the dict (y = 0) x = 6 y = 0 heading = float(site_data[key][x][y]) # Get the data value (T, rh, road-T, dewpt) # The data values is the third column in the dict (x = 2) # The first row (y = 0) is T, second row (y = 1) is rh, third row (y = 2) is road-T, fourth row (y = 3) is dewpt x = 2 T = float(site_data[key][x][0]) rh = float(site_data[key][x][1]) road_T = float(site_data[key][x][2]) dewpt = float(site_data[key][x][3]) # QC the values and convert units (degF to degC) # # T is in degF # rh is in % # road_T is in degF # dewpt is in degF # if (T < -70.0 or T > 90.0): T_str = "" else: T_str = "%.2f" % ((T - 32.0) / 1.8) if (rh < 0.0 or rh > 100.0): rh_str = "" else: rh_str = "%.0f" % rh if (road_T < -70.0 or road_T > 90.0): road_T_str = "" else: road_T_str = "%.2f" % ((road_T - 32.0) / 1.8) if (dewpt < -70.0 or dewpt > 90.0): dewpt_str = "" else: dewpt_str = "%.2f" % ((dewpt - 32.0) / 1.8) if (speed < 0 or speed > 200): speed_str = "" else: speed_str = "%.2f" % (speed * 0.44704) if (heading < 0 or heading > 360): heading_str = "" else: heading_str = "%.0f" % heading output_line = "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" % ( key, site_time_val, lat, lon, speed_str, heading_str, T_str, rh_str, road_T_str, dewpt_str) #print "%s" % output_line out_fh.write("%s\n" % output_line) out_fh.close() # # Run alaska2probe_message.py (this uses the alaska_reader.py class) # # Make processed output dir if necessary proc_dest_dir = "%s/%s" % (proc_output_dir, curr_date) mkdir_path(proc_dest_dir) proc_output_file_name = "alaska.%s.nc" % curr_date proc_output_path = "%s/%s" % (proc_dest_dir, proc_output_file_name) cmd_str = "alaska2probe_message.py %s %s %s 0" % ( cdl_file, raw_output_path, proc_output_path) # Run the command line ret = run_cmd(cmd_str, logg) if ret != 0: logg.write_error( "run_cmd() failed for alaska2probe_message.py command_str.") logg.write_ending(exit_status=ret, msg="frmt_and_run_alaska2probe_message.py") return ret