def find_file(data_dir, file_substr, idx=0, print_list=0, print_debug=1):
if (print_debug):
icing_message.debug('looking for file in ' + data_dir + \
' matching: ' + file_substr)
if not path.isdir(data_dir):
msg = '\tinput data directory %s does not exist.' % data_dir
icing_message.error(msg)
return ''
#
# get a list of files in data directory
#
filelist = listdir(data_dir)
filelist.sort()
test_files = []
for filename in filelist:
if not filename.find(file_substr):
test_files.append(filename)
if len(test_files) and print_list:
icing_message.debug(str(test_files))
#
# the newest file will be last in list
#
if len(test_files):
file_path = path.join(data_dir, test_files[idx])
if get_file(file_path):
return file_path
return ''
def get_valid_path(run_time, base_dir, fn_tmplate, fcast='', add_date=False):
gm_tuple = gmtime(run_time)
#
# The 6, 9 and 12 hour forecasts are generated on 3 hour
# intervals
#
if fcast > 3:
run_shift = gm_tuple[3]%3
gm_tuple = gmtime((run_time - run_shift*3600) + (fcast * 3600))
else:
gm_tuple = gmtime(run_time + (fcast * 3600))
fcast_dir = 'f%02d' % fcast
date_dir = '%04d%02d%02d' % \
(gm_tuple[0], gm_tuple[1], gm_tuple[2])
if fcast > 0:
fcast_dir = 'f%02d' % fcast
pathname = path.join(base_dir, fcast_dir)
else:
pathname = base_dir
if add_date:
pathname = path.join(pathname, date_dir, strftime(fn_tmplate, gm_tuple))
else:
pathname = path.join(pathname, strftime(fn_tmplate, gm_tuple))
icing_message.debug('looking for ' + pathname)
return pathname
def get_valid_path(run_time, base_dir, fn_tmplate, fcast='', add_date=False):
gm_tuple = gmtime(run_time)
#
# The 6, 9 and 12 hour forecasts are generated on 3 hour
# intervals
#
if fcast > 3:
run_shift = gm_tuple[3]%3
gm_tuple = gmtime((run_time - run_shift*3600) + (fcast * 3600))
else:
gm_tuple = gmtime(run_time + (fcast * 3600))
fcast_dir = 'f%02d' % fcast
date_dir = '%04d%02d%02d' % \
(gm_tuple[0], gm_tuple[1], gm_tuple[2])
if fcast > 0:
fcast_dir = 'f%02d' % fcast
pathname = path.join(base_dir, fcast_dir)
else:
pathname = base_dir
if add_date:
pathname = path.join(pathname, date_dir, strftime(fn_tmplate, gm_tuple))
else:
pathname = path.join(pathname, strftime(fn_tmplate, gm_tuple))
icing_message.debug('looking for ' + pathname)
return pathname
def find_file(data_dir, file_substr, idx=0, print_list=0, print_debug=1):
if(print_debug) :
icing_message.debug('looking for file in ' + data_dir + \
' matching: ' + file_substr)
if not path.isdir(data_dir):
msg = '\tinput data directory %s does not exist.' % data_dir
icing_message.error(msg)
return ''
#
# get a list of files in data directory
#
filelist = listdir(data_dir)
filelist.sort()
test_files = []
for filename in filelist:
if not filename.find(file_substr):
test_files.append(filename)
if len(test_files) and print_list:
icing_message.debug(str(test_files))
#
# the newest file will be last in list
#
if len(test_files):
file_path = path.join(data_dir, test_files[idx])
if get_file(file_path):
return file_path
return ''
def remove_files(file_list):
for a_file in file_list:
icing_message.debug('removing ' + a_file)
try:
remove(a_file)
except OSError, e:
print e
icing_message.warning('unable to remove ' + a_file)
def insert_files(file_list, prodIDSuffix=""):
for a_file in file_list:
ldm_header = path.splitext(path.basename(a_file))[0]
insert_cmd = 'pqinsert -l /home/ldm/logs/icing_ldm_notify.log -p "' + ldm_header + \
prodIDSuffix + '" -f EXP ' + a_file
icing_message.debug('inserting ' + a_file + ' on LDM via: ' +
insert_cmd)
ret = system(insert_cmd)
if ret:
icing_message.warning('pqinsert failed on' + a_file)
def check_for_file(the_dir):
ldata = Ldata('icing_input', ldata_debug)
ret = ldata.info_read(the_dir, max_valid_age)
if ret != 0:
msg = '\tCould not find file in ' + the_dir
icing_message.error(msg)
return ''
file_path = ldata.data_path(the_dir)
icing_message.debug('file_path = ' + file_path)
return file_path
def check_for_file(the_dir):
ldata = Ldata('icing_input', ldata_debug)
ret = ldata.info_read(the_dir, max_valid_age)
if ret != 0:
msg = '\tCould not find file in ' + the_dir
icing_message.error(msg)
return ''
file_path = ldata.data_path(the_dir)
icing_message.debug('file_path = ' + file_path)
return file_path
def get_archive_time(run_time):
'creates a unix time from from time string in the form YYYYMMDDHH'
#
# separate out the time components
#
year = int(run_time[:4])
month = int(run_time[4:6])
day = int(run_time[6:8])
hour = int(run_time[8:10])
minutes = int(run_time[10:12])
gmt_tpl = (year, month, day, hour, minutes, 0, 0, 0, 0)
archive_time = mktime(gmt_tpl) - timezone
icing_message.debug('archive time is: ' + asctime(gmtime(archive_time)))
return archive_time
def get_archive_time(run_time):
'creates a unix time from from time string in the form YYYYMMDDHH'
#
# separate out the time components
#
year = int(run_time[:4])
month = int(run_time[4:6])
day = int(run_time[6:8])
hour = int(run_time[8:10])
minutes = int(run_time[10:12])
gmt_tpl = (year, month, day, hour, minutes, 0, 0, 0, 0)
archive_time = mktime(gmt_tpl) - timezone
icing_message.debug('archive time is: ' + asctime(gmtime(archive_time)))
return archive_time
def get_gen_path(run_time, base_dir, fn_tmplate, fcast=0, add_date=False):
gm_tuple = gmtime(run_time)
date_dir = '%04d%02d%02d' % \
(gm_tuple[0], gm_tuple[1], gm_tuple[2])
if fcast > 0:
fcast_dir = 'f%02d' % fcast
pathname = path.join(base_dir, fcast_dir)
else:
pathname = base_dir
if add_date:
pathname = path.join(pathname, date_dir, strftime(fn_tmplate, gm_tuple))
else:
pathname = path.join(pathname, strftime(fn_tmplate, gm_tuple))
icing_message.debug('looking for ' + pathname)
return pathname
def get_gen_path(run_time, base_dir, fn_tmplate, fcast=0, add_date=False):
gm_tuple = gmtime(run_time)
date_dir = '%04d%02d%02d' % \
(gm_tuple[0], gm_tuple[1], gm_tuple[2])
if fcast > 0:
fcast_dir = 'f%02d' % fcast
pathname = path.join(base_dir, fcast_dir)
else:
pathname = base_dir
if add_date:
pathname = path.join(pathname, date_dir, strftime(fn_tmplate, gm_tuple))
else:
pathname = path.join(pathname, strftime(fn_tmplate, gm_tuple))
icing_message.debug('looking for ' + pathname)
return pathname
def build_input_path(the_dir, the_file, run_time):
try:
dir_template = environ[the_dir]
except KeyError:
print 'key error'
icing_message.error('Unknown variable: ' + the_dir)
exit()
try:
file_template = environ[the_file]
except KeyError:
print 'key error'
icing_message.error('Unknown variable: ' + the_file)
exit()
the_path = path.join(fill_template(dir_template, run_time), \
fill_template(file_template, run_time))
icing_message.debug('the_path = ' + the_path)
return the_path
def build_input_path(the_dir, the_file, run_time):
try:
dir_template = environ[the_dir]
except KeyError:
print 'key error'
icing_message.error('Unknown variable: ' + the_dir)
exit()
try:
file_template = environ[the_file]
except KeyError:
print 'key error'
icing_message.error('Unknown variable: ' + the_file)
exit()
the_path = path.join(fill_template(dir_template, run_time), \
fill_template(file_template, run_time))
icing_message.debug('the_path = ' + the_path)
return the_path
def run_app(app_name, instance, check_outdir=True, start_time='',
stop_time='', in_file='', opt_in_file='', out_dir='', diag_dir='', rt='', additional_args='', time_center='', time_radius=''):
startTime = datetime.now()
icing_message.debug('\tStarting ' + app_name + ' : ' + instance + " at time " + startTime.strftime("%Y-%m-%d %H:%M"))
# build command line
if len(time_center) > 0 and len(time_radius) > 0 and len(out_dir) > 0:
icing_message.debug('time_center = ' + time_center)
icing_message.debug('time_radius = ' + time_radius)
icing_message.debug('out_dir = ' + out_dir)
cmd = 'run_' + app_name + '.%s.sh %s %s %s' % (instance, time_center, time_radius, out_dir)
elif len(start_time) > 0 and len(stop_time) > 0 and len(out_dir) > 0:
icing_message.debug('start_time = ' + start_time)
icing_message.debug('stop_time = ' + stop_time)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh \'%s\' \'%s\' %s' % \
(instance, start_time, stop_time, instance)
elif len(in_file) > 0 and len(out_dir) > 0 and len(diag_dir) > 0 and len(opt_in_file) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('out_dir = ' + out_dir)
icing_message.debug('diag_dir = ' + diag_dir)
command = 'run_' + app_name + '.%s.sh %s %s %s %s' % \
(instance, in_file, out_dir, diag_dir, instance)
elif len(in_file) > 0 and len(out_dir) > 0 and len(opt_in_file) == 0 and len(diag_dir) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh %s %s %s' % \
(instance, in_file, out_dir, instance)
elif len(start_time) > 0 and len(out_dir) > 0:
icing_message.debug('start_time = ' + start_time)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh \'%s\' %s' % \
(instance, start_time, instance)
elif len(in_file) > 0 and len(opt_in_file) > 0 and len(out_dir) > 0 and len(rt) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('opt_in_file = ' + opt_in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh %s %s %s %s' % \
(instance, in_file, opt_in_file, out_dir, instance)
elif len(in_file) > 0 and len(opt_in_file) > 0 and len(out_dir) > 0 and len(rt) > 0:
icing_message.debug('run_time = ' + rt)
icing_message.debug('in_file = ' + in_file)
icing_message.debug('opt_in_file = ' + opt_in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh %s %s %s %s %s' % \
(instance, rt, in_file, opt_in_file, out_dir, instance)
elif len(in_file) == 0 and len(opt_in_file) == 0 and len(out_dir) > 0 and len(rt) > 0:
icing_message.debug('run_time = ' + rt)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh %s %s \'%s\'' % (instance, out_dir, instance, rt)
elif len(in_file) > 0:
icing_message.debug('in_file = ' + in_file)
command = 'run_' + app_name + '.%s.sh %s %s' % (instance, in_file, instance)
elif len(in_file) == 0 and len(opt_in_file) == 0 and len(out_dir) > 0 and \
len(start_time) == 0 and len(stop_time) == 0:
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh %s' % (instance, instance)
else:
icing_message.error('command not set ... exiting.')
return ''
if additional_args != '':
command += " "+str(additional_args)
icing_message.debug('additional_args = ' + str(additional_args))
ret = system(command)
if ret:
icing_message.error(app_name + ' failed for instance: ' + instance)
endTime = datetime.now()
icing_message.debug('\t'+app_name + ' finished run at ' + endTime.strftime("%Y-%m-%d %H:%M"))
if check_outdir and len(out_dir) > 0:
return icing_input.check_for_file(out_dir)
return ''
def run_app(app_name, instance, check_outdir=True, start_time='',
stop_time='', in_file='', opt_in_file='', out_dir='', diag_dir='', rt=''):
startTime = datetime.now()
icing_message.debug('\tStarting ' + app_name + ' : ' + instance + " at time " + startTime.strftime("%Y-%m-%d %H:%M"))
# build command line
if len(start_time) > 0 and len(stop_time) > 0 and len(out_dir) > 0:
icing_message.debug('start_time = ' + start_time)
icing_message.debug('stop_time = ' + stop_time)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s \'%s\' \'%s\' %s' % \
(instance, start_time, stop_time, instance)
elif len(in_file) > 0 and len(out_dir) > 0 and len(diag_dir) > 0 and len(opt_in_file) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('out_dir = ' + out_dir)
icing_message.debug('diag_dir = ' + diag_dir)
command = 'start_' + app_name + '.%s %s %s %s %s' % \
(instance, in_file, out_dir, diag_dir, instance)
elif len(in_file) > 0 and len(out_dir) > 0 and len(opt_in_file) == 0 and len(diag_dir) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s %s %s %s' % \
(instance, in_file, out_dir, instance)
elif len(start_time) > 0 and len(out_dir) > 0:
icing_message.debug('start_time = ' + start_time)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s \'%s\' %s' % \
(instance, start_time, instance)
elif len(in_file) > 0 and len(opt_in_file) > 0 and len(out_dir) > 0 and len(rt) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('opt_in_file = ' + opt_in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s %s %s %s %s' % \
(instance, in_file, opt_in_file, out_dir, instance)
elif len(in_file) > 0 and len(opt_in_file) > 0 and len(out_dir) > 0 and len(rt) > 0:
icing_message.debug('run_time = ' + rt)
icing_message.debug('in_file = ' + in_file)
icing_message.debug('opt_in_file = ' + opt_in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s %s %s %s %s %s' % \
(instance, rt, in_file, opt_in_file, out_dir, instance)
elif len(in_file) == 0 and len(opt_in_file) == 0 and len(out_dir) > 0 and len(rt) > 0:
icing_message.debug('run_time = ' + rt)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s %s %s \'%s\'' % (instance, out_dir, instance, rt)
elif len(in_file) > 0:
icing_message.debug('in_file = ' + in_file)
command = 'start_' + app_name + '.%s %s %s' % (instance, in_file, instance)
elif len(in_file) == 0 and len(opt_in_file) == 0 and len(out_dir) > 0 and \
len(start_time) == 0 and len(stop_time) == 0:
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s %s' % (instance, instance)
else:
icing_message.error('command not set ... exiting.')
return ''
ret = system(command)
if ret:
icing_message.error(app_name + ' failed for instance: ' + instance)
endTime = datetime.now()
icing_message.debug('\t'+app_name + ' finished run at ' + endTime.strftime("%Y-%m-%d %H:%M"))
if check_outdir:
return icing_input.check_for_file(out_dir)
return ''