def main():
usage_str = "%prog out_file"
parser = OptionParser(usage=usage_str)
parser.add_option("-n",
"--nothing",
dest="nothing",
action="store_true",
help="nothing at all")
parser.add_option("-c",
"--config_file",
dest="config",
help="read specified configuration file")
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_help()
sys.exit(2)
out_file = args[0]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting()
create_generic_belle_isle(out_file)
logg.write_ending()
def main():
usage_str = "%prog config_dir date input_dir output_dir state_abbrev \n\tNOTE: date format should be YYYYMMDD"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 4:
parser.print_help()
sys.exit(2)
config_dir = args[0]
date = args[1]
input_dir = args[2]
output_dir = args[3]
state_abbrev = args[4]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.set_suffix("pyl")
logg.write_starting("run_cdf_to_csv.py")
run_cdf_to_csv(config_dir, date, input_dir, output_dir, state_abbrev, logg)
logg.write_ending(0, "run_cdf_to_csv.py")
def main():
usage_str = "%prog cdl in_file out_file nevada_ret"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 2:
parser.print_help()
sys.exit(2)
cdl = args[0]
in_file = args[1]
out_file = args[2]
nevada_ret = args[3]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting()
ret = nevada2probe_message(cdl, in_file, out_file, nevada_ret, logg)
if ret != 0:
logg.write_error("error calling nevada2probe_message")
logg.write_ending(1, "nevada2probe_message.py")
return -1
logg.write_ending()
def main():
usage_str = "%prog config"
parser = OptionParser(usage = usage_str)
parser.add_option("-n", "--nothing", dest="nothing", action="store_true", help="nothing at all")
parser.add_option("-c", "--config_file", dest="config", help="read specified configuration file")
parser.add_option("-l", "--log", dest="log", help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_help()
sys.exit(2)
if options.log:
logg = log_msg.LogMessage(options.log,"pyl")
else:
logg = log_msg.LogMessage("")
cfg = __import__(args[0], globals(), locals(), [])
acf = __import__("ameritrak_config", globals(), locals(), [])
log_dir = vii_paths.LOG_DIR
logg.write_starting("")
cmd = "ameritrak_ingest.py -l %s/ameritrak_ingest ameritrak_config stop" % log_dir
logg.write_info(cmd)
os.system(cmd)
logg.write_ending(0)
def main():
usage_str = "%prog year month state input_dir output_dir \n\tNote: year should be in the format YYYY; month should be in the format MM; state should be in the same format as the basename in the daily files (e.g. Minnesota)"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 2:
parser.print_help()
sys.exit(2)
year = args[0]
month = args[1]
state = args[2]
input_dir = args[3]
output_dir = args[4]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.set_suffix(".pyl")
logg.write_starting("create_monthly_stats.py")
create_monthly_stats(year, month, state, input_dir, output_dir, logg)
logg.write_ending(0, "create_montly_stats.py")
def main():
usage_str = "%prog location where location is belle_isle, belle_isle_low, belle_isle_high, mdot"
parser = OptionParser(usage=usage_str)
parser.add_option("-n",
"--nothing",
dest="nothing",
action="store_true",
help="nothing at all")
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_help()
sys.exit(2)
location = args[0]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
cdl_file = vii_paths.PROBE_MESSAGE_CDL
logg.write_starting()
simulate_probe(location, cdl_file, vii_paths.MI_SIMULATED_DIR,
vii_paths.MI_PROBE_MESSAGE_DIR, logg)
logg.write_ending()
def main():
usage_str = "%prog YYYYMMDDHHMM output_directory time_interval_secs"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 3:
parser.print_help()
sys.exit(2)
yyyymmddhhmm = args[0]
output_directory = args[1]
time_interval = args[2]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting()
run_get_ameritrak_images(yyyymmddhhmm, output_directory, time_interval,
logg)
logg.write_ending()
def main():
usage_str = " %prog begin_date end_date\n\tNote: begin_date and end_date should be in the format YYYYMMDD"
parser = OptionParser(usage=usage_str)
parser.add_option("-l", "--log", dest="log", help="log")
(options, args) = parser.parse_args()
if len(sys.argv) < 3:
parser.print_help()
sys.exit(2)
begin_date = args[0]
end_date = args[1]
if options.log:
logg = log_msg.LogMessage(options.log, "pyl")
else:
logg = log_msg.LogMessage("")
logg.write_starting("run_ameritrak_data_status.py")
mydatelist = gen_date_list(begin_date, end_date)
for index, mydate in enumerate(mydatelist):
cmd = "ameritrak_data_status.py -l %s %s %s %s" % (log_prefix, mydate,
in_dir, out_dir)
#cmd = "ameritrak_data_status.py %s %s %s" % (mydate, in_dir, out_dir)
logg.write_info(cmd)
os.system(cmd)
logg.write_ending(0)
def main():
usage_str = "%prog begin_date end_date in_dir out_dir"
parser = OptionParser(usage = usage_str)
parser.add_option("-l", "--log", dest="log", help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 4:
parser.print_help()
sys.exit(2)
begin_date = args[0]
end_date = args[1]
in_dir = args[2]
out_dir = args[3]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting("run_nevada_timing.py")
mydatelist = gen_date_list(begin_date, end_date)
for date in mydatelist:
dated_in_dir = os.path.join(in_dir, date)
if os.path.exists(dated_in_dir):
cmd = "nevada_timing.py %s %s %s" % (date, in_dir, out_dir)
os.system(cmd)
else:
logg.write_info("%s does not exist, skipping" % dated_in_dir)
logg.write_ending(0, "run_nevada_timing.py")
def main():
usage_str = "%prog time_delta in_dir"
parser = OptionParser(usage=usage_str)
parser.add_option("-n",
"--nothing",
dest="nothing",
action="store_true",
help="nothing at all")
parser.add_option("-c",
"--config_file",
dest="config",
help="read specified configuration file")
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 2:
parser.print_help()
sys.exit(2)
time_delta = int(args[0])
in_dir = args[1]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting()
get_latest_files(time_delta, in_dir)
logg.write_ending()
def main():
usage_str = "%prog time_delta site base_dir\nFor example:\n\t%prog 1800 72644078 /d2/vii/data/raw/minnesota_images\n\t%prog 1800 74341091 /d2/vii/data/raw/minnesota_images\n\t%prog 1800 74341090 /d2/vii/data/raw/minnesota_images"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 2:
parser.print_help()
sys.exit(2)
time_delta = int(args[0])
site = args[1]
base_dir = args[2]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting()
image_file = get_latest_minnesota_image_file(time_delta, int(site),
base_dir)
print "image_file: ", image_file
logg.write_ending()
def main():
usage_str = "%prog"
parser = OptionParser(usage = usage_str)
#parser.add_option("-c", "--config_file", dest="config", help="read specified configuration file")
parser.add_option("-l", "--log", dest="log", help="write log messages to specified file")
(options, args) = parser.parse_args()
if options.log:
logg = log_msg.LogMessage(options.log,"pyl")
else:
logg = log_msg.LogMessage(vii_paths.Run_ameritrak_ingest_log_file)
logg.write_starting()
#cfg = __import__(args[0], globals(), locals(), [])
acf = __import__("ameritrak_config", globals(), locals(), [])
log_dir = vii_paths.LOG_DIR
try:
pidf = open(acf.Ameritrak_ingest_pid_file,"r")
pid_line = pidf.readline().strip()
pid = int(pid_line)
os.kill(pid,0)
except:
cmd = "ameritrak_ingest.py -l %s/ameritrak_ingest ameritrak_config start" % log_dir
logg.write_info("process not running, restarting...")
logg.write_info(cmd)
os.system(cmd)
logg.write_ending(0)
def main():
usage_str = "%prog delta_seconds in_dir out_dir\nFor example:\n\tcopy_recent_files.py 600 /ftp/restricted/alaska/root/upload_images /d2/vii/data/raw/alaska_images/latest"
parser = OptionParser(usage = usage_str)
parser.add_option("-n", "--nothing", dest="nothing", action="store_true", help="nothing at all")
parser.add_option("-c", "--config_file", dest="config", help="read specified configuration file")
parser.add_option("-l", "--log", dest="log", help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 3:
parser.print_help()
sys.exit(2)
delta_seconds = int(args[0])
in_dir = args[1]
out_dir = args[2]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting()
file_count = copy_recent_files(delta_seconds, in_dir, out_dir)
logg.write_info("copying %d files from %s to %s\n" % (file_count, in_dir, out_dir))
logg.write_ending()
def main():
usage_str = "%prog out_dir"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_help()
sys.exit(2)
out_dir = args[0]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting("get_minnesota_images.py")
get_minnesota_images(out_dir, logg)
logg.write_ending(error_ret_flag, "get_minnesota_images.py")
def main():
usage_str = "%prog config [start|stop|restart]"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 2:
parser.print_help()
sys.exit(2)
cf = cf = __import__(args[0], globals(), locals(), [])
act = args[1]
pid_file = cf.Ameritrak_ingest_pid_file
if options.log:
logg = log_msg.LogMessage(options.log, "pyl")
else:
logg = log_msg.LogMessage("")
d = AmeritrakIngest(pid_file, cf.Ameritrak_ip, cf.Ameritrak_port,
cf.Ameritrak_user, cf.Ameritrak_pwd,
cf.Ameritrak_out_dir, logg)
if "start" == act:
d.start()
elif "restart" == act:
d.restart()
elif "stop" == act:
d.stop()
def main():
usage_str = "%prog cdl_file in_file out_file"
parser = OptionParser(usage=usage_str)
parser.add_option("-D",
"--directory",
dest="directory",
action="store_true",
help="process all files in directory")
parser.add_option("-d", "--date", dest="date", help="date")
parser.add_option("-l", "--log", dest="log", help="log")
parser.add_option("-n",
"--netcdf",
dest="netcdf",
action="store_true",
help="netcdf input")
(options, args) = parser.parse_args()
if len(args) < 3:
parser.print_help()
sys.exit(2)
cdl = args[0]
in_file = args[1]
out_file = args[2]
# ED need to change 300 to a parameter fed in on the command line
ptime = time.mktime(time.gmtime()) - 300
if options.date:
ptime = time.mktime(time.strptime(options.date, "%Y%m%d.%H%M"))
ptime = ptime - (ptime % 300)
if options.log:
logg = log_msg.LogMessage(options.log, "pyl")
else:
logg = log_msg.LogMessage("")
logg.write_starting()
logg.write_info("Executing: %s" % " ".join(sys.argv))
logg.write_info("reading obs list from %s" % in_file)
if not os.path.exists(in_file):
logg.write_info("%s not found" % in_file)
logg.write_ending(0)
return
final_ret = 0
if options.directory:
listdir = os.listdir(in_file)
for fname in sorted(listdir):
in_path = os.path.join(in_file, fname)
out_path = os.path.join(out_file, fname)
ret = process(options.netcdf, cdl, logg, in_path, out_path)
if ret != 0:
final_ret = ret
else:
final_ret = process(options.netcdf, cdl, logg, in_file, out_file)
logg.write_ending(final_ret)
return final_ret
def main():
usage_str = "%prog date raw_input_dir timing_input_dir\n\tNote: date must be in the format YYYYMMDD"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 3:
parser.print_help()
sys.exit(2)
date = args[0]
raw_input_dir = args[1]
timing_input_dir = args[2]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting("create_nevada_noticed_time.py")
file_name = "nevada_%s.csv" % (date)
tmp_file_name = "nevada_%s.csv.tmp" % (date)
full_raw_input_path = os.path.join(raw_input_dir, date, file_name)
full_timing_input_path = os.path.join(timing_input_dir, date, file_name)
tmp_full_timing_input_path = os.path.join(timing_input_dir, date,
tmp_file_name)
ret = get_noticed_time_data(date, full_raw_input_path,
full_timing_input_path, logg)
if (ret == -1):
return
get_raw_data(date, full_raw_input_path, logg)
# Write timing data
writer = csv.writer(open(tmp_full_timing_input_path, "wb"))
for key in header_dict.iterkeys():
#print header_dict[key]
writer.writerow(header_dict[key])
for key in current_timing_dict.iterkeys():
writer.writerow(current_timing_dict[key])
cmd = "mv %s %s" % (tmp_full_timing_input_path, full_timing_input_path)
ret = os.system(cmd)
if (ret != 0):
logg.write_error("Unable to execute: %s" % cmd)
logg.write_ending(0, "create_nevada_noticed_time.py")
def main():
usage_str = "%s yyyymmddhhmmss yyyymmddhhmmss outbase" % sys.argv[0]
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
parser.add_option("-k",
"--keep",
action="store_true",
dest="keep",
help="keep all intermediate file (defaults to False)",
default=False)
(options, args) = parser.parse_args()
if len(args) < 3:
print parser.print_help()
exit(2)
#Dictionary of parameters that is passed through the chain of functions
params = {}
params["begin"] = args[0]
params["end"] = args[1]
params["out_base"] = args[2]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
params["logg"] = logg
logg.write_starting()
funcs = funchain(logg)
funcs.append(get_json_data)
funcs.append(partition_json_state)
funcs.append(json_2_cdf)
funcs.append(cdf_2_probe)
funcs.append(update_state_group)
#Clean Up
if not options.keep:
funcs.append(rm_json_in)
funcs.append(rm_json_state)
funcs.append(rm_nc_in)
ret = funcs.exec_chain(params)
logg.write_debug(ret)
exit_code = int(not ret)
logg.write_ending(exit_code)
exit(exit_code)
def main():
parser = OptionParser(usage="%prog [options] out_dir")
parser.add_option("-d", "--date", dest="date", help="date")
parser.add_option("-l", "--log", dest="log", help="base name of log file")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_help()
sys.exit(2)
out_dir = args[0]
if options.log:
logg = log_msg.LogMessage(options.log, "pyl")
else:
logg = log_msg.LogMessage("")
date_str = time.strftime("%Y%m%d.%H%M")
if options.date != None:
date_str = options.date
time_tup = time.strptime(date_str, "%Y%m%d.%H%M")
ptime = time.mktime(time_tup)
ptime = ptime - (ptime % 300) - 300
time_tup = time.gmtime(ptime)
day_str = time.strftime("%Y%m%d", time_tup)
hh = time.strftime("%H", time_tup)
logg.write_starting("")
day_ftp_dir = time.strftime(ftp_dir, time_tup)
fnames = [
"rtma2p5.t%sz.2dvarges_ndfd.grb2" % hh,
"rtma2p5.t%sz.2dvaranl_ndfd.grb2" % hh
]
out_day_dir = os.path.join(out_dir, day_str)
cmd = "mkdir -p %s" % out_day_dir
logg.write_info(cmd)
os.system(cmd)
for fname in fnames:
fpath = "%s/%s/%s" % (ftp_site, day_ftp_dir, fname)
ofpath = os.path.join(out_day_dir, fname)
cmd = "curl %s > %s" % (fpath, ofpath)
logg.write_info(cmd)
ret = os.system(cmd)
if ret != 0:
logg.write_error("couldn't grab %s, cleaning up..." % fpath)
cmd = "rm -f %s" % ofpath
logg.write_info(cmd)
os.system(cmd)
logg.write_ending(ret)
return ret
logg.write_ending(0)
return 0
def main():
usage_str = "%prog begin_yyyymmddhhmmss end_yyyymmddhhmmss out_file_base\nPurpose: Get weather telematics for specified time interval"
parser = OptionParser(usage=usage_str)
parser.add_option("-e",
"--eval",
dest="evaluate",
action="store_true",
help="use eval (useful when gunzipping is required)")
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
parser.add_option("-o",
"--offset",
dest="offset",
help="specify starting offset")
parser.add_option("-n",
"--number",
dest="number",
help="number of queries")
(options, args) = parser.parse_args()
if len(args) < 3:
parser.print_help()
sys.exit(2)
begin_yyyymmddhhmmss = args[0]
end_yyyymmddhhmmss = args[1]
out_file = args[2]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
offset = ""
number = 1
if options.offset:
offset = options.offset
if options.number:
number = int(options.number)
if number > QUERY_LIMIT:
print "Error: exceeded query limit of %d\n" % QUERY_LIMIT
logg.write_starting()
get_wxt_data(options.evaluate, number, offset, begin_yyyymmddhhmmss,
end_yyyymmddhhmmss, out_file)
logg.write_ending()
def main():
usage_str = "%prog begin_date end_date in_file[s] out_file_base_name\nBegin and end dates should be provided in the form yyyymmdd.\nEnd date is one day beyond the days of interest and its hour should be set to the 0th hour.\nInput files are converted from json format into hourly netcdf files"
parser = OptionParser(usage=usage_str)
parser.add_option("-d",
"--directory",
dest="directory",
help="use all files in specified directory as input")
parser.add_option("-n",
"--netcdf_file",
dest="cdf",
action="store_true",
help="write output netcdf file")
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if (not options.directory and len(args) < 4) or (options.directory
and len(args) < 3):
parser.print_help()
sys.exit(2)
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting()
begin_date = args[0]
end_date = args[1]
in_files = []
if options.directory:
# create a list of file paths for all files in directory
dirlist = os.listdir(options.directory)
for fname in dirlist:
path = os.path.join(options.directory, fname)
in_files.append(path)
else:
in_files = args[2:-1]
out_file = args[-1]
parse_hourly(begin_date, end_date, in_files, out_file, options.cdf, logg)
logg.write_ending()
def main():
usage_str = "%prog"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 0:
parser.print_help()
sys.exit(2)
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.set_suffix(".pyl")
logg.write_starting("run_create_probe_msg_history.py")
curr_time = time.time()
end_time = time.mktime(time.gmtime(curr_time))
begin_time = end_time - LOOKBACK_INTERVAL_SEC
end_time_tup = time.gmtime(end_time)
day = time.strftime("%Y%m%d", end_time_tup)
hhmm = time.strftime("%H%M", end_time_tup)
begin_time_tup = time.gmtime(begin_time)
begin_day = time.strftime("%Y%m%d", begin_time_tup)
begin_hhmm = time.strftime("%H%M", begin_time_tup)
end_time = day + hhmm
begin_time = begin_day + begin_hhmm
cmd = "group_umtri_probe_msg_files.py -l /d2/vii/data/log/group_umtri_probe_msg_files vii_paths.py %s %s /d2/vii/data/static/cdl/probe_message.cdl $VII_DATA/tmp $VII_DATA/processed/mi_csv2nc $VII_DATA/processed/mi_probe_message" % (
begin_time, end_time)
#cmd = "group_umtri_probe_msg_files.py vii_paths.py %s %s /d2/vii/data/static/cdl/probe_message.cdl $VII_DATA/tmp $VII_DATA/processed/mi_csv2nc /d2/test/lookback_test" % (begin_time, end_time)
#cmd = "group_umtri_probe_msg_files.py vii_paths.py 201405081052 201405081122 /d2/vii/data/static/cdl/probe_message.cdl $VII_DATA/tmp $VII_DATA/processed/mi_csv2nc /d2/test/lookback_test"
logg.write_info("executing: %s" % cmd)
ret = os.system(cmd)
if ret != 0:
logg.write_error("group_umtri_probe_msg_files.py returned %d" % ret)
logg.write_ending(1, "run_group_umtri_probe_msg_files.py")
else:
logg.write_ending(0, "run_group_umtri_probe_msg_files.py")
def main():
parser = OptionParser(
usage=
"%prog [options] config state_abbreviation interval_time_sec\n\tNote: state_abbreviation should be in lower case e.g. mn, mi, nv"
)
parser.add_option("-l", "--log", dest="log", help="base name of log file")
parser.add_option("-b",
"--begin_date",
dest="begin_date",
help="start at particular date yyyymmdd")
parser.add_option("-e",
"--end_date",
dest="end_date",
help="end at particular date yyyymmdd")
(options, args) = parser.parse_args()
if options.log:
logg = log_msg.LogMessage(options.log, "pyl")
else:
logg = log_msg.LogMessage("")
if len(args) < 2:
parser.print_help()
sys.exit(2)
config = args[0]
state = args[1]
interval_time_sec = args[2]
interval_time_sec = int(interval_time_sec)
if (options.begin_date != None
and options.end_date == None) or (options.begin_date == None
and options.end_date != None):
print "Error: both begin_date and end_date must be set or neither should be set"
sys.exit(2)
cf = __import__(config, globals(), locals(), [])
# Get state object
full_state = cf.state_abbrev_to_full[state]
state_object = cf.State_dictionary[full_state + "_vdt"]
logg.write_starting("rwa_to_json.py")
rwa_to_json(state_object, cf, interval_time_sec, logg, options)
logg.write_ending(error_ret_flag, "rwa_to_json.py")
def main():
parser = OptionParser(usage="%prog")
parser.add_option("-l", "--log", dest="log", help="base name of log file")
parser.add_option("-b",
"--begin_date",
dest="begin_date",
help="start at particular date yyyymmdd")
parser.add_option("-e",
"--end_date",
dest="end_date",
help="end at particular date yyyymmdd")
parser.add_option(
"-i",
"--in_dir",
dest="in_dir",
help=
"provide input directory other than the one specified in vii_paths.py")
parser.add_option(
"-o",
"--out_dir",
dest="out_dir",
help=
"provide output directory other than the one specified in vii_paths.py"
)
(options, args) = parser.parse_args()
if options.log:
logg = log_msg.LogMessage(options.log, "pyl")
else:
logg = log_msg.LogMessage("")
if (options.begin_date != None
and options.end_date == None) or (options.begin_date == None
and options.end_date != None):
print "Error: both begin_date and end_date must be set or neither should be set"
sys.exit(2)
in_base_dir = vii_paths.RTMA_DIR
out_base_dir = vii_paths.RTMA_NC_DIR
if options.in_dir:
in_base_dir = options.in_dir
if options.out_dir:
out_base_dir = options.out_dir
logg.write_starting("rtma_grib2_to_netcdf.py")
ret = run_rtma_grib2_to_netcdf(options, in_base_dir, out_base_dir, logg)
logg.write_ending(msg="rtma_grib2_to_netcdf.py", exit_status=ret)
def main():
usage_str = "%prog in_file output_file"
parser = OptionParser(usage = usage_str)
parser.add_option("-l", "--log", dest="log", help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 2:
parser.print_help()
sys.exit(2)
if options.log:
logg = log_msg.LogMessage(options.log)
logg.set_suffix(".asc")
else:
logg = log_msg.LogMessage("")
def main():
usage1 = "Standard simulation examples:\n\tsimulator.py -d 1 /d4/gerry/simulation_dir/simulation_config/mn_simulation.cfg 300 201405300500 201406010500 /d4/gerry/simulation_dir/simulation_may_mn_more_dense_no_winds\n\tsimulator.py /d4/gerry/simulation_dir/simulation_config/mn_simulation.cfg 300 201405300500 201406010500 /d4/gerry/simulation_dir/simulation_may_mn_less_dense_no_winds\n\tsimulator.py -d 1 /d4/gerry/simulation_dir/simulation_config/mi_simulation.cfg 300 201405300500 201406010500 /d4/gerry/simulation_dir/simulation_may_mi_more_dense_no_winds\n\tsimulator.py /d4/gerry/simulation_dir/simulation_config/mi_simulation.cfg 300 201405300500 201406010500 /d4/gerry/simulation_dir/simulation_may_mi_less_dense_no_winds\n\tsimulator.py -d 1 /d4/gerry/simulation_dir/simulation_config/mn_simulation.cfg 300 201404050000 201404070000 /d4/gerry/simulation_dir/simulation_april_mn_more_dense_no_winds\n\tsimulator.py /d4/gerry/simulation_dir/simulation_config/mn_simulation.cfg 300 201404050000 201404070000 /d4/gerry/simulation_dir/simulation_april_mn_less_dense_no_winds\n\tsimulator.py -d 1 /d4/gerry/simulation_dir/simulation_config/mi_simulation.cfg 300 201404050000 201404070000 /d4/gerry/simulation_dir/simulation_april_mi_more_dense_no_winds\n\tsimulator.py /d4/gerry/simulation_dir/simulation_config/mi_simulation.cfg 300 201404050000 201404070000 /d4/gerry/simulation_dir/simulation_april_mi_less_dense_no_winds\n\tsimulator.py -d 1 /d4/gerry/simulation_dir/simulation_config/mn_simulation.cfg 300 201403301200 201404011200 /d4/gerry/simulation_dir/simulation_march_mn_more_dense_no_winds\n\tsimulator.py /d4/gerry/simulation_dir/simulation_config/mn_simulation.cfg 300 201403301200 201404011200 /d4/gerry/simulation_dir/simulation_march_mn_less_dense_no_winds\n\tsimulator.py -d 1 /d4/gerry/simulation_dir/simulation_config/mi_simulation.cfg 300 201403301200 201404011200 /d4/gerry/simulation_dir/simulation_march_mi_more_dense_no_winds\n\tsimulator.py /d4/gerry/simulation_dir/simulation_config/mi_simulation.cfg 300 201403301200 201404011200 /d4/gerry/simulation_dir/simulation_march_mi_less_dense_no_winds\n\tsimulator.py -d 1 /d4/gerry/simulation_dir/simulation_config/mn_simulation.cfg 300 201402240000 201402260000 /d4/gerry/simulation_dir/simulation_feb_mn_more_dense_no_winds\n\tsimulator.py /d4/gerry/simulation_dir/simulation_config/mn_simulation.cfg 300 201402240000 201402260000 /d4/gerry/simulation_dir/simulation_feb_mn_less_dense_no_winds\n\tsimulator.py -d 1 /d4/gerry/simulation_dir/simulation_config/mi_simulation.cfg 300 201402240000 201402260000 /d4/gerry/simulation_dir/simulation_feb_mi_more_dense_no_winds\n\tsimulator.py /d4/gerry/simulation_dir/simulation_config/mi_simulation.cfg 300 201402240000 201402260000 /d4/gerry/simulation_dir/simulation_feb_mi_less_dense_no_winds\n\tsimulator.py -d 1 /d4/gerry/simulation_dir/simulation_config/mn_simulation.cfg 300 201401040500 201401060500 /d4/gerry/simulation_dir/simulation_jan_mn_more_dense_no_winds\n\tsimulator.py /d4/gerry/simulation_dir/simulation_config/mn_simulation.cfg 300 201401040500 201401060500 /d4/gerry/simulation_dir/simulation_jan_mn_less_dense_no_winds\n\tsimulator.py -d 1 /d4/gerry/simulation_dir/simulation_config/mi_simulation.cfg 300 201401040500 201401060500 /d4/gerry/simulation_dir/simulation_jan_mi_more_dense_no_winds\n\tsimulator.py /d4/gerry/simulation_dir/simulation_config/mi_simulation.cfg 300 201401040500 201401060500 /d4/gerry/simulation_dir/simulation_jan_mi_less_dense_no_winds"
usage2 = "\n\n\tWipers only examples:\n\tsimulator.py -W /d4/gerry/wiper_simulation_dir/simulation_config/mn_simulation_wipers.cfg 3600 201405310500 201406010500 /d4/gerry/wiper_simulation_dir/simulation_may_mn_wipers_only\n\tsimulator.py -W /d4/gerry/wiper_simulation_dir/simulation_config/mi_simulation_wipers.cfg 3600 201405310500 201406010500 /d4/gerry/wiper_simulation_dir/simulation_may_mi_wipers_only\n\tsimulator.py -W /d4/gerry/wiper_simulation_dir/simulation_config/mn_simulation_wipers.cfg 3600 201404060000 201404070000 /d4/gerry/wiper_simulation_dir/simulation_april_mn_wipers_only\n\tsimulator.py -W /d4/gerry/wiper_simulation_dir/simulation_config/mi_simulation_wipers.cfg 3600 201404060000 201404070000 /d4/gerry/wiper_simulation_dir/simulation_april_mi_wipers_only\n\tsimulator.py -W /d4/gerry/wiper_simulation_dir/simulation_config/mn_simulation_wipers.cfg 3600 201403311200 201404011200 /d4/gerry/wiper_simulation_dir/simulation_march_mn_wipers_only\n\tsimulator.py -W /d4/gerry/wiper_simulation_dir/simulation_config/mi_simulation_wipers.cfg 3600 201403311200 201404011200 /d4/gerry/wiper_simulation_dir/simulation_march_mi_wipers_only\n\tsimulator.py -W /d4/gerry/wiper_simulation_dir/simulation_config/mn_simulation_wipers.cfg 3600 201402250000 201402260000 /d4/gerry/wiper_simulation_dir/simulation_feb_mn_wipers_only\n\tsimulator.py -W /d4/gerry/wiper_simulation_dir/simulation_config/mi_simulation_wipers.cfg 3600 201402250000 201402260000 /d4/gerry/wiper_simulation_dir/simulation_feb_mi_wipers_only\n\tsimulator.py -W /d4/gerry/wiper_simulation_dir/simulation_config/mn_simulation_wipers.cfg 3600 201401050500 201401060500 /d4/gerry/wiper_simulation_dir/simulation_jan_mn_wipers_only\n\tsimulator.py -W /d4/gerry/wiper_simulation_dir/simulation_config/mi_simulation_wipers.cfg 3600 201401050500 201401060500 /d4/gerry/wiper_simulation_dir/simulation_jan_mi_wipers_only"
usage_str = "%prog config_file time_delta begin_date end_date output_dir\n\t" + usage1 + usage2
parser = OptionParser(usage=usage_str)
parser.add_option("-d",
"--density",
dest="density",
help="less dense - 0, more dense - 1")
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
parser.add_option("-W",
"--wipers_only",
dest="wipers_only",
action="store_true",
help="only do wipers processing")
(options, args) = parser.parse_args()
if len(args) < 5:
parser.print_help()
sys.exit(2)
density = 0
if options.density != None:
density = int(options.density)
config_file = args[0]
time_delta = int(args[1])
begin_date = args[2]
end_date = args[3]
output_dir = args[4]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting()
ret = simulation(options.wipers_only, density, config_file, Met_data_dict,
time_delta, begin_date, end_date, output_dir, logg)
logg.write_ending(ret)
def main():
usage_str = "%prog"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if len(args) < 0:
parser.print_help()
sys.exit(2)
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.set_suffix(".pyl")
logg.write_starting("run_create_probe_msg_history.py")
end_time = time.mktime(time.gmtime()) - 900
end_time = end_time - (end_time % 900)
begin_time = end_time - 3600
end_time_tup = time.gmtime(end_time)
day = time.strftime("%Y%m%d", end_time_tup)
hhmm = time.strftime("%H%M", end_time_tup)
begin_time_tup = time.gmtime(begin_time)
begin_day = time.strftime("%Y%m%d", begin_time_tup)
begin_hhmm = time.strftime("%H%M", begin_time_tup)
end_time = day + hhmm + "00"
begin_time = begin_day + begin_hhmm + "00"
cmd = "create_probe_msg_history.py -l /d2/vii/data/log/create_probe_msg_history_nv vii_config /d2/vii/data/processed/nv_vdt_output/ /d2/vii/data/nv_history/ -b %s -e %s" % (
begin_time, end_time)
logg.write_info("executing: %s" % cmd)
ret = os.system(cmd)
if ret != 0:
logg.write_error("create_probe_msg_history.py returned %d" % ret)
logg.write_ending(1, "run_create_probe_msg_history.py")
else:
logg.write_ending(0, "run_create_probe_msg_history.py")
def main():
usage_str = "%prog output_file"
parser = OptionParser(usage = usage_str)
parser.add_option("-c", "--count", dest="count", help="number of times to run")
parser.add_option("-l", "--log", dest="log", help="write log messages to specified file")
parser.add_option("-n", "--number", dest="number", help="number of queries")
parser.add_option("-t", "--time_offset", dest="time_offset", help="time offset")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_help()
sys.exit(2)
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
number = 1
time_offset = 15
if options.time_offset:
time_offset = int(options.time_offset)
if options.number:
number = int(options.number)
count = 10
if options.count:
count = int(options.count)
output_file = args[0]
offset = ""
logg.write_starting()
for ind in range(count):
current_time = time.time()
start_time = current_time - time_offset
begin_yyyymmddhhmmss = time.strftime("%Y%m%d%H%M%S", time.gmtime(start_time))
end_yyyymmddhhmmss = time.strftime("%Y%m%d%H%M%S", time.gmtime(current_time))
get_wxt_json.get_wxt_data(number, offset, begin_yyyymmddhhmmss, end_yyyymmddhhmmss, output_file, logg)
time.sleep(time_offset)
logg.write_ending()
def main():
logg = log_msg.LogMessage('/d2/vii/data/log/ep_run_mn_processes')
ingest_data(logg)
raw_2_probe(logg)
# combine_wxt(logg)
run_vdt4(logg)
run_rwh(logg)
run_probe_to_json(logg)
run_rwa_to_json(logg)
def main():
usage_str = "%prog time_delta site base_dir\nFor example:\n\t%prog 1800 72644078 /d2/vii/data/raw/minnesota_images\n\t%prog 1800 74341091 /d2/vii/data/raw/minnesota_images\n\t%prog 1800 74341090 /d2/vii/data/raw/minnesota_images\n\t%prog -n 1800 72489030 /d2/vii/data/raw/nevada_images"
parser = OptionParser(usage=usage_str)
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
parser.add_option("-n",
"--nevada",
dest="nevada",
action="store_true",
help="process nevada example site")
(options, args) = parser.parse_args()
if len(args) < 2:
parser.print_help()
sys.exit(2)
time_delta = int(args[0])
site = args[1]
base_dir = args[2]
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting()
if not options.nevada:
minnesota_latest_image_file = LatestImageFile(Minnesota_site_dict)
image_file = minnesota_latest_image_file.get_latest_image_file(
time_delta, int(site), base_dir)
print "image_file: ", image_file
else:
nevada_latest_image_file = LatestImageFile(Nevada_site_dict)
image_file = nevada_latest_image_file.get_latest_image_file(
time_delta, int(site), base_dir)
print "image_file: ", image_file
logg.write_ending()