def get_file(file, timeout=10):
sleep_interval = 2
wait_interval = timeout
while not path.exists(file):
sleep(sleep_interval)
wait_interval = wait_interval - sleep_interval
if wait_interval <= 0:
return 0
last_size = 0
while wait_interval > 0:
try:
size = stat(file)[ST_SIZE]
except OSError, e:
msg = 'stat failed --', e
icing_message.error(msg)
continue
if size == last_size:
return 1
last_size = size;
sleep(sleep_interval)
wait_interval = wait_interval - sleep_interval
def find_file(data_dir, file_substr, idx=0, print_list=0, print_debug=1):
if (print_debug):
icing_message.debug('looking for file in ' + data_dir + \
' matching: ' + file_substr)
if not path.isdir(data_dir):
msg = '\tinput data directory %s does not exist.' % data_dir
icing_message.error(msg)
return ''
#
# get a list of files in data directory
#
filelist = listdir(data_dir)
filelist.sort()
test_files = []
for filename in filelist:
if not filename.find(file_substr):
test_files.append(filename)
if len(test_files) and print_list:
icing_message.debug(str(test_files))
#
# the newest file will be last in list
#
if len(test_files):
file_path = path.join(data_dir, test_files[idx])
if get_file(file_path):
return file_path
return ''
def find_file(data_dir, file_substr, idx=0, print_list=0, print_debug=1):
if(print_debug) :
icing_message.debug('looking for file in ' + data_dir + \
' matching: ' + file_substr)
if not path.isdir(data_dir):
msg = '\tinput data directory %s does not exist.' % data_dir
icing_message.error(msg)
return ''
#
# get a list of files in data directory
#
filelist = listdir(data_dir)
filelist.sort()
test_files = []
for filename in filelist:
if not filename.find(file_substr):
test_files.append(filename)
if len(test_files) and print_list:
icing_message.debug(str(test_files))
#
# the newest file will be last in list
#
if len(test_files):
file_path = path.join(data_dir, test_files[idx])
if get_file(file_path):
return file_path
return ''
def find_file_env_var(data_dir,
template,
run_time,
idx=0,
print_list=0,
print_debug=1):
try:
dir_template = environ[data_dir]
except KeyError:
icing_message.error('Unknown variable: ' + data_dir)
exit()
try:
file_template = environ[template]
except KeyError:
icing_message.error('Unknown variable: ' + template)
exit()
the_dir = fill_template(dir_template, run_time)
the_file = fill_template(file_template, run_time)
file_path = find_file(the_dir, the_file, idx, print_list, print_debug)
return file_path
def get_file(file, timeout=10):
sleep_interval = 2
wait_interval = timeout
while not path.exists(file):
sleep(sleep_interval)
wait_interval = wait_interval - sleep_interval
if wait_interval <= 0:
return 0
last_size = 0
while wait_interval > 0:
try:
size = stat(file)[ST_SIZE]
except OSError, e:
msg = 'stat failed --', e
icing_message.error(msg)
continue
if size == last_size:
return 1
last_size = size;
sleep(sleep_interval)
wait_interval = wait_interval - sleep_interval
def check_environ_dir(env_var, run_time=-1,tileNum=-1, yesterdayOK=False):
try:
env_dir = environ[env_var]
except KeyError, e:
print e
icing_message.error('environment variable $' + env_var + ' does not exist ... exiting ...')
return False
def check_environ_dir(env_var, run_time=-1,tileNum=-1, yesterdayOK=False):
try:
env_dir = environ[env_var]
except KeyError, e:
print e
icing_message.error('environment variable $' + env_var + ' does not exist ... exiting ...')
return False
def check_for_file(the_dir):
ldata = Ldata('icing_input', ldata_debug)
ret = ldata.info_read(the_dir, max_valid_age)
if ret != 0:
msg = '\tCould not find file in ' + the_dir
icing_message.error(msg)
return ''
file_path = ldata.data_path(the_dir)
icing_message.debug('file_path = ' + file_path)
return file_path
def check_for_file(the_dir):
ldata = Ldata('icing_input', ldata_debug)
ret = ldata.info_read(the_dir, max_valid_age)
if ret != 0:
msg = '\tCould not find file in ' + the_dir
icing_message.error(msg)
return ''
file_path = ldata.data_path(the_dir)
icing_message.debug('file_path = ' + file_path)
return file_path
def find_file_env_var(data_dir, template, run_time, idx=0, print_list=0, print_debug=1):
try:
dir_template = environ[data_dir]
except KeyError:
icing_message.error('Unknown variable: ' + data_dir)
exit()
try:
file_template = environ[template]
except KeyError:
icing_message.error('Unknown variable: ' + template)
exit()
the_dir = fill_template(dir_template, run_time)
the_file = fill_template(file_template, run_time)
file_path = find_file(the_dir, the_file, idx, print_list, print_debug)
return file_path
def build_input_path(the_dir, the_file, run_time):
try:
dir_template = environ[the_dir]
except KeyError:
print 'key error'
icing_message.error('Unknown variable: ' + the_dir)
exit()
try:
file_template = environ[the_file]
except KeyError:
print 'key error'
icing_message.error('Unknown variable: ' + the_file)
exit()
the_path = path.join(fill_template(dir_template, run_time), \
fill_template(file_template, run_time))
icing_message.debug('the_path = ' + the_path)
return the_path
def build_input_path(the_dir, the_file, run_time):
try:
dir_template = environ[the_dir]
except KeyError:
print 'key error'
icing_message.error('Unknown variable: ' + the_dir)
exit()
try:
file_template = environ[the_file]
except KeyError:
print 'key error'
icing_message.error('Unknown variable: ' + the_file)
exit()
the_path = path.join(fill_template(dir_template, run_time), \
fill_template(file_template, run_time))
icing_message.debug('the_path = ' + the_path)
return the_path
def run_app(app_name, instance, check_outdir=True, start_time='',
stop_time='', in_file='', opt_in_file='', out_dir='', diag_dir='', rt='', additional_args='', time_center='', time_radius=''):
startTime = datetime.now()
icing_message.debug('\tStarting ' + app_name + ' : ' + instance + " at time " + startTime.strftime("%Y-%m-%d %H:%M"))
# build command line
if len(time_center) > 0 and len(time_radius) > 0 and len(out_dir) > 0:
icing_message.debug('time_center = ' + time_center)
icing_message.debug('time_radius = ' + time_radius)
icing_message.debug('out_dir = ' + out_dir)
cmd = 'run_' + app_name + '.%s.sh %s %s %s' % (instance, time_center, time_radius, out_dir)
elif len(start_time) > 0 and len(stop_time) > 0 and len(out_dir) > 0:
icing_message.debug('start_time = ' + start_time)
icing_message.debug('stop_time = ' + stop_time)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh \'%s\' \'%s\' %s' % \
(instance, start_time, stop_time, instance)
elif len(in_file) > 0 and len(out_dir) > 0 and len(diag_dir) > 0 and len(opt_in_file) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('out_dir = ' + out_dir)
icing_message.debug('diag_dir = ' + diag_dir)
command = 'run_' + app_name + '.%s.sh %s %s %s %s' % \
(instance, in_file, out_dir, diag_dir, instance)
elif len(in_file) > 0 and len(out_dir) > 0 and len(opt_in_file) == 0 and len(diag_dir) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh %s %s %s' % \
(instance, in_file, out_dir, instance)
elif len(start_time) > 0 and len(out_dir) > 0:
icing_message.debug('start_time = ' + start_time)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh \'%s\' %s' % \
(instance, start_time, instance)
elif len(in_file) > 0 and len(opt_in_file) > 0 and len(out_dir) > 0 and len(rt) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('opt_in_file = ' + opt_in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh %s %s %s %s' % \
(instance, in_file, opt_in_file, out_dir, instance)
elif len(in_file) > 0 and len(opt_in_file) > 0 and len(out_dir) > 0 and len(rt) > 0:
icing_message.debug('run_time = ' + rt)
icing_message.debug('in_file = ' + in_file)
icing_message.debug('opt_in_file = ' + opt_in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh %s %s %s %s %s' % \
(instance, rt, in_file, opt_in_file, out_dir, instance)
elif len(in_file) == 0 and len(opt_in_file) == 0 and len(out_dir) > 0 and len(rt) > 0:
icing_message.debug('run_time = ' + rt)
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh %s %s \'%s\'' % (instance, out_dir, instance, rt)
elif len(in_file) > 0:
icing_message.debug('in_file = ' + in_file)
command = 'run_' + app_name + '.%s.sh %s %s' % (instance, in_file, instance)
elif len(in_file) == 0 and len(opt_in_file) == 0 and len(out_dir) > 0 and \
len(start_time) == 0 and len(stop_time) == 0:
icing_message.debug('out_dir = ' + out_dir)
command = 'run_' + app_name + '.%s.sh %s' % (instance, instance)
else:
icing_message.error('command not set ... exiting.')
return ''
if additional_args != '':
command += " "+str(additional_args)
icing_message.debug('additional_args = ' + str(additional_args))
ret = system(command)
if ret:
icing_message.error(app_name + ' failed for instance: ' + instance)
endTime = datetime.now()
icing_message.debug('\t'+app_name + ' finished run at ' + endTime.strftime("%Y-%m-%d %H:%M"))
if check_outdir and len(out_dir) > 0:
return icing_input.check_for_file(out_dir)
return ''
print e
icing_message.error('environment variable $' + env_var + ' does not exist ... exiting ...')
return False
if run_time > 0:
if tileNum > 0:
test_dir = fill_template(env_dir,run_time,tile_num=tileNum)
else:
test_dir = strftime(env_dir, gmtime(run_time))
else:
test_dir = env_dir
if not path.isdir(test_dir):
if not yesterdayOK:
icing_message.error('Directory (' + test_dir + ') from $' + env_var + ' does not exist ... exiting ...')
return False
else:
test_dir = strftime(env_dir, gmtime(run_time-24*60*60))
if not path.isdir(test_dir):
icing_message.error('Directory (' + test_dir + ') from $' + env_var + ' does not exist ... exiting ...')
return False
return True
#######################################################################
# get_valid_path(run_time, base_dir, fn_tmplate, fcast='', add_date=False)
#
# gets the pathname based on valid/forecast time
#
print e
icing_message.error('environment variable $' + env_var + ' does not exist ... exiting ...')
return False
if run_time > 0:
if tileNum > 0:
test_dir = fill_template(env_dir,run_time,tile_num=tileNum)
else:
test_dir = strftime(env_dir, gmtime(run_time))
else:
test_dir = env_dir
if not path.isdir(test_dir):
if not yesterdayOK:
icing_message.error('Directory (' + test_dir + ') from $' + env_var + ' does not exist ... exiting ...')
return False
else:
test_dir = strftime(env_dir, gmtime(run_time-24*60*60))
if not path.isdir(test_dir):
icing_message.error('Directory (' + test_dir + ') from $' + env_var + ' does not exist ... exiting ...')
return False
return True
#######################################################################
# get_valid_path(run_time, base_dir, fn_tmplate, fcast='', add_date=False)
#
# gets the pathname based on valid/forecast time
#
def run_app(app_name, instance, check_outdir=True, start_time='',
stop_time='', in_file='', opt_in_file='', out_dir='', diag_dir='', rt=''):
startTime = datetime.now()
icing_message.debug('\tStarting ' + app_name + ' : ' + instance + " at time " + startTime.strftime("%Y-%m-%d %H:%M"))
# build command line
if len(start_time) > 0 and len(stop_time) > 0 and len(out_dir) > 0:
icing_message.debug('start_time = ' + start_time)
icing_message.debug('stop_time = ' + stop_time)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s \'%s\' \'%s\' %s' % \
(instance, start_time, stop_time, instance)
elif len(in_file) > 0 and len(out_dir) > 0 and len(diag_dir) > 0 and len(opt_in_file) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('out_dir = ' + out_dir)
icing_message.debug('diag_dir = ' + diag_dir)
command = 'start_' + app_name + '.%s %s %s %s %s' % \
(instance, in_file, out_dir, diag_dir, instance)
elif len(in_file) > 0 and len(out_dir) > 0 and len(opt_in_file) == 0 and len(diag_dir) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s %s %s %s' % \
(instance, in_file, out_dir, instance)
elif len(start_time) > 0 and len(out_dir) > 0:
icing_message.debug('start_time = ' + start_time)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s \'%s\' %s' % \
(instance, start_time, instance)
elif len(in_file) > 0 and len(opt_in_file) > 0 and len(out_dir) > 0 and len(rt) == 0:
icing_message.debug('in_file = ' + in_file)
icing_message.debug('opt_in_file = ' + opt_in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s %s %s %s %s' % \
(instance, in_file, opt_in_file, out_dir, instance)
elif len(in_file) > 0 and len(opt_in_file) > 0 and len(out_dir) > 0 and len(rt) > 0:
icing_message.debug('run_time = ' + rt)
icing_message.debug('in_file = ' + in_file)
icing_message.debug('opt_in_file = ' + opt_in_file)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s %s %s %s %s %s' % \
(instance, rt, in_file, opt_in_file, out_dir, instance)
elif len(in_file) == 0 and len(opt_in_file) == 0 and len(out_dir) > 0 and len(rt) > 0:
icing_message.debug('run_time = ' + rt)
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s %s %s \'%s\'' % (instance, out_dir, instance, rt)
elif len(in_file) > 0:
icing_message.debug('in_file = ' + in_file)
command = 'start_' + app_name + '.%s %s %s' % (instance, in_file, instance)
elif len(in_file) == 0 and len(opt_in_file) == 0 and len(out_dir) > 0 and \
len(start_time) == 0 and len(stop_time) == 0:
icing_message.debug('out_dir = ' + out_dir)
command = 'start_' + app_name + '.%s %s' % (instance, instance)
else:
icing_message.error('command not set ... exiting.')
return ''
ret = system(command)
if ret:
icing_message.error(app_name + ' failed for instance: ' + instance)
endTime = datetime.now()
icing_message.debug('\t'+app_name + ' finished run at ' + endTime.strftime("%Y-%m-%d %H:%M"))
if check_outdir:
return icing_input.check_for_file(out_dir)
return ''