def process_runfiles(inps):
""" Calls the script to execute stackSentinel runfiles."""
if inps.flag_process:
command = 'execute_stacksentinel_run_files.py ' + inps.custom_template_file
messageRsmas.log(command)
#Check the performance, change in subprocess
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
logger.log(loglevel.ERROR,
'ERROR in execute_stacksentinel_run_files.py')
raise Exception('ERROR in execute_stacksentinel_run_files.py')
if os.path.isdir('PYSAR'):
shutil.rmtree('PYSAR')
if int(inps.custom_template['cleanopt']) >= 1:
cleanlist = clean_list()
_remove_directories(cleanlist[1])
if inps.stopprocess:
logger.log(loglevel.INFO, 'Exit as planned after processing')
sys.exit(0)
return inps
def process_time_series(inps):
""" runs time series analysis based on squeesar."""
if inps.flag_pysar:
if not os.path.isdir(inps.workingDir + '/run_files_SQ'):
create_squeesar_runfiles(inps)
command = 'execute_squeesar_run_files.py ' + inps.custom_template_file
messageRsmas.log(command)
# Check the performance, change in subprocess
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
logger.log(loglevel.ERROR,
'ERROR in execute_squeesar_run_files.py')
raise Exception('ERROR in execute_squeesar_run_files.py')
if int(inps.custom_template['cleanopt']) >= 1:
cleanlist = clean_list()
_remove_directories(cleanlist[1])
if inps.stoppysar:
logger.log(loglevel.DEBUG,
'Exit as planned after time series analysis')
sys.exit(0)
return
def create_squeesar_runfiles(inps):
os.chdir(inps.work_dir)
command = 'create_squeesar_run_files.py ' + inps.custom_template_file
messageRsmas.log(command)
# Check the performance, change in subprocess
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
logger.log(loglevel.ERROR, 'ERROR in create_squeesar_run_files.py')
raise Exception('ERROR in create_squeesar_run_files.py')
if inps.stopmakerun:
logger.log(loglevel.INFO,
'Exit as planned after making squeesar run files ')
sys.exit(0)
return
def main(iargs=None):
"""Downloads data with ssara and asfserial scripts."""
command = 'download_rsmas.py ' + iargs[0]
messageRsmas.log(command)
inps = command_line_parse(iargs)
#########################################
# Submit job
#########################################
if inps.submit_flag:
job_file_name = 'download_rsmas'
work_dir = os.getcwd()
job_name = inps.template_file.split(os.sep)[-1].split('.')[0]
wall_time = '24:00'
cb.submit_script(job_name, job_file_name, sys.argv[:], work_dir,
wall_time)
project_name = putils.get_project_name(
custom_template_file=inps.template_file)
work_dir = putils.get_work_directory(None, project_name)
slc_dir = os.path.join(work_dir, 'SLC')
if not os.path.isdir(work_dir):
os.makedirs(work_dir)
if not os.path.isdir(slc_dir):
os.makedirs(slc_dir)
os.chdir(work_dir)
# if satellite is not Sentinel (not tried yet)
if 'SenDT' not in project_name and 'SenAT' not in project_name:
dataset_template = Template(inps.template_file)
ssaraopt = dataset_template.generate_ssaraopt_string()
ssara_call = ['ssara_federated_query.py'
] + ssaraopt + ['--print', '--download']
ssara_process = subprocess.Popen(ssara_call, shell=True).wait()
completion_status = ssara_process.poll()
return
download('ssara', inps.template_file, slc_dir, outnum=1)
#download('ssara', inps.template_file, slc_dir, outnum = 2)
download('asfserial', inps.template_file, slc_dir, outnum=1)
def create_runfiles(inps):
""" Calls the script to create stackSentinel runfiles and configs."""
if inps.flag_makerun:
os.chdir(inps.work_dir)
command = 'create_stacksentinel_run_files.py ' + inps.custom_template_file
messageRsmas.log(command)
# Check the performance, change in subprocess
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
logger.log(loglevel.ERROR,
'ERROR in create_stacksentinel_run_files.py')
raise Exception('ERROR in create_stacksentinel_run_files.py')
if inps.stopmakerun:
logger.log(loglevel.INFO,
'Exit as planned after making sentinel run files ')
sys.exit(0)
return inps
def call_isce_dem(custom_template):
bbox = custom_template['sentinelStack.boundingBox']
bbox = bbox.strip("'")
south = round(float(bbox.split()[0]) -
0.5) # assumes quotes '-1 0.15 -91.3 -91.0'
north = round(float(bbox.split()[1]) + 0.5)
west = round(float(bbox.split()[2]) - 0.5)
east = round(float(bbox.split()[3]) + 0.5)
dembbox = str(int(south)) + ' ' + str(int(north)) + ' ' + str(
int(west)) + ' ' + str(int(east))
# cmd = 'dem.py -a stitch -b '+demBbox+' -c -u https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/'
cmd = 'dem_rsmas_kawan.py -a stitch -b ' + dembbox + ' -c -u https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/'
messageRsmas.log(cmd)
cwd = os.getcwd()
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
shell=True,
universal_newlines=True)
except subprocess.CalledProcessError as exc:
print("Command failed. Exit code, StdErr:", exc.returncode, exc.output)
sys.exit('Error produced by dem.py')
else:
# print("Success. StdOut \n{}\n".format(output))
if 'Could not create a stitched DEM. Some tiles are missing' in output:
os.chdir('..')
shutil.rmtree('DEM')
sys.exit('Error in dem.py: Tiles are missing. Ocean???')
xmlfile = glob.glob('demLat_*.wgs84.xml')[0]
fin = open(xmlfile, 'r')
fout = open("tmp.txt", "wt")
for line in fin:
fout.write(line.replace('demLat', cwd + '/demLat'))
fin.close()
fout.close()
os.rename('tmp.txt', xmlfile)
def main(argv):
# import pdb; pdb.set_trace()
messageRsmas.log(' '.join(argv))
inps = command_line_parse()
# moved below to parse methods
# parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
# epilog=EXAMPLE)
# parser.add_argument('custom_template_file', nargs='?',
# help='custom template with option settings.\n')
# inps = parser.parse_args()
custom_template = readfile.read_template(inps.custom_template_file)
# import pdb;
# pdb.set_trace()
if os.path.isdir('DEM'):
shutil.rmtree('DEM')
os.mkdir('DEM')
os.chdir('DEM')
# cwd = os.getcwd()
if 'sentinelStack.demMethod' not in custom_template.keys():
custom_template['sentinelStack.demMethod'] = 'bbox'
if custom_template['sentinelStack.demMethod'] == 'bbox' and inps.ssara:
call_ssara_dem()
if inps.new:
print('nice job kawan! You aren\' dumb!')
if custom_template['sentinelStack.demMethod'] == 'bbox' and inps.isce:
print('you started isce')
call_isce_dem(custom_template)
print('you finished isce')
else:
sys.exit('Error unspported demMethod option: ' +
custom_template['sentinelStack.demMethod'])
print('\n###############################################')
print('End of dem_rsmas.py')
print('################################################\n')
def run_ingest_insarmaps(inps):
""" Calls the script of ingest insarmaps and emails the results."""
if inps.flag_insarmaps:
command = 'ingest_insarmaps.py ' + inps.custom_template_file
messageRsmas.log(command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
logger.log(loglevel.ERROR, 'ERROR in ingest_insarmaps.py')
raise Exception('ERROR in ingest_insarmaps.py')
putils.email_insarmaps_results(inps.custom_template)
# clean
if int(inps.custom_template['cleanopt']) == 4:
cleanlist = clean_list()
_remove_directories(cleanlist[4])
if inps.stopinsarmaps:
logger.log(loglevel.DEBUG, 'Exit as planned after insarmaps')
def create_or_copy_dem(inps, work_dir, template, custom_template_file):
""" Downloads a DEM file or copies an existing one."""
#if inps.flag_dem:
dem_dir = work_dir + '/DEM'
if os.path.isdir(dem_dir) and len(os.listdir(dem_dir)) == 0:
os.rmdir(dem_dir)
if not os.path.isdir(dem_dir):
if 'sentinelStack.demDir' in list(template.keys(
)) and template['sentinelStack.demDir'] != str('auto'):
shutil.copytree(template['sentinelStack.demDir'], dem_dir)
else:
# TODO: Change subprocess call to get back error code and send error code to logger
command = 'dem_rsmas.py ' + custom_template_file
print(command)
messageRsmas.log(command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
logger.log(loglevel.ERROR, 'ERROR while making DEM')
raise Exception('ERROR while making DEM')
def call_pysar(custom_template, custom_template_file, flag_load_and_stop):
""" Calls pysarAPP to load and process data. """
# TODO: Change subprocess call to get back error code and send error code to logger
logger.log(loglevel.DEBUG,
'\n*************** running pysar ****************')
command = 'pysarApp.py ' + custom_template_file + ' --end load_data |& tee out_pysar.log'
out_file = 'out_pysar_load'
logger.log(loglevel.INFO, command)
messageRsmas.log(command)
command = '(' + command + ' | tee ' + out_file + '.o) 3>&1 1>&2 2>&3 | tee ' + out_file + '.e'
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
logger.log(loglevel.ERROR, 'ERROR in pysarApp.py --end load_data')
raise Exception('ERROR in pysarApp.py --end load_data')
if flag_load_and_stop:
logger.log(loglevel.DEBUG, 'Exit as planned after loading into pysar')
return
# clean after loading data for cleanopt >= 2
if 'All data needed found' in open('out_pysar.log').read():
if int(custom_template['cleanopt']) >= 2:
print('Cleaning files: cleanopt= ' +
str(custom_template['cleanopt']))
cleanlist = clean_list()
_remove_directories(cleanlist[2])
if int(custom_template['cleanopt']) >= 3:
_remove_directories(cleanlist[3])
command = 'pysarApp.py ' + custom_template_file
out_file = 'out_pysar'
logger.log(loglevel.INFO, command)
messageRsmas.log(command)
command = '(' + command + ' | tee ' + out_file + '.o) 3>&1 1>&2 2>&3 | tee ' + out_file + '.e'
messageRsmas.log(command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
logger.log(loglevel.ERROR, 'ERROR in pysarApp.py')
raise Exception('ERROR in pysarApp.py')
return None
def process(inps):
#########################################
# Initiation
#########################################
start_time = time.time()
inps.project_name = get_project_name(inps.custom_template_file)
inps.work_dir = get_work_directory(None, inps.project_name)
inps.slc_dir = os.path.join(inps.work_dir, 'SLC')
if inps.remove_project_dir:
_remove_directories(directories_to_delete=[inps.work_dir])
if not os.path.isdir(inps.work_dir):
os.makedirs(inps.work_dir)
os.chdir(inps.work_dir)
# Read and update template file:
inps = prs.create_or_update_template(inps)
print(inps)
if not inps.processingMethod or inps.workflow == 'interferogram':
inps.processingMethod = 'sbas'
if not os.path.isdir(inps.slc_dir):
os.makedirs(inps.slc_dir)
command_line = os.path.basename(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])
logger_process_rsmas.log(loglevel.INFO, '##### NEW RUN #####')
logger_process_rsmas.log(loglevel.INFO, 'process_rsmas.py ' + command_line)
messageRsmas.log('##### NEW RUN #####')
messageRsmas.log(command_line)
#########################################
# startssara: Getting Data
#########################################
# running download scripts:
# download_ssara_rsmas.py $TE/template
# downlaod_asfserial_rsmas.py $TE/template
prs.call_ssara(inps.flag_ssara, inps.custom_template_file, inps.slc_dir)
#########################################
# startmakerun: create run files
#########################################
# create or copy DEM
# running the script:
# dem_rsmas.py $TE/template
prs.create_or_copy_dem(inps, inps.work_dir, inps.template,
inps.custom_template_file)
# Check for DEM and create sentinel run files
# running the script:
# create_stacksentinel_run_files.py $TE/template
prs.create_runfiles(inps)
#########################################
# startprocess: Execute run files
#########################################
# Running the script:
# execute_stacksentinel_run_files.py $TE/template starting_run stopping_run
# Example for running run 1 to 4:
# execute_stacksentinel_run_files.py $TE/template 1 4
prs.process_runfiles(inps)
#########################################
# startpysar: running PySAR and email results
#########################################
if inps.processingMethod == 'squeesar':
# Run squeesar script:
# create_squeesar_run_files.py $TE/template
# execute_squeesar_run_files.py $TE/template
prs.process_time_series(inps)
else:
# Run PySAR script:
# pysarApp.py $TE/template
prs.run_pysar(inps, start_time)
# Run ingest insarmaps script and email results
# ingest_insarmaps.py $TE/template
prs.run_ingest_insarmaps(inps)
logger_process_rsmas.log(loglevel.INFO, 'End of process_rsmas')
def main(argv):
messageRsmas.log(' '.join(argv))
inps = dem_parser()
custom_template = readfile.read_template(inps.custom_template_file)
cwd = make_dem_dir()
# can sentinelStack.demMethod be removed? I think parser is the replacement
if 'sentinelStack.demMethod' not in custom_template.keys():
custom_template['sentinelStack.demMethod'] = 'bbox'
if custom_template['sentinelStack.demMethod'] == 'bbox':
bbox = custom_template['sentinelStack.boundingBox']
south = bbox.split(' ')[0].split('\'')[
1] # assumes quotes '-1 0.15 -91.3 -91.0'
north = bbox.split(' ')[1]
west = bbox.split(' ')[2]
east = bbox.split(' ')[3].split('\'')[0]
elif custom_template['sentinelStack.demMethod'] == 'ssara':
call_ssara_dem(custom_template, inps)
cmd = 'ssara_federated_query.py ' + custom_template[
'ssaraopt'] + ' --dem'
output = subprocess.check_output(cmd, shell=True)
output = output.split("\n")
for line in output:
if line.startswith("wget"):
coordList = line.split("?")[1].split("&")[0:4]
for item in coordList:
if "north" in item:
north = item.split("=")[1]
if "south" in item:
south = item.split("=")[1]
if "east" in item:
east = item.split("=")[1]
if "west" in item:
west = item.split("=")[1]
else:
sys.ext('Error unspported demMethod option: ' +
custom_template['sentinelStack.demMethod'])
if inps.ssara:
call_ssara_dem(custom_template, inps)
print('####### CONTINUED')
else:
print('not ssara')
south = round(float(south) - 0.5)
north = round(float(north) + 0.5)
west = round(float(west) - 0.5)
east = round(float(east) + 0.5)
demBbox = str(int(south)) + ' ' + str(int(north)) + ' ' + str(
int(west)) + ' ' + str(int(east))
cmd = 'dem.py -a stitch -b ' + demBbox + ' -c -u https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/'
messageRsmas.log(cmd)
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
shell=True,
universal_newlines=True)
except subprocess.CalledProcessError as exc:
print("Command failed. Exit code, StdErr:", exc.returncode, exc.output)
sys.exit('Error produced by dem.py')
else:
#print("Success. StdOut \n{}\n".format(output))
if 'Could not create a stitched DEM. Some tiles are missing' in output:
os.chdir('..')
shutil.rmtree('DEM')
sys.exit('Error in dem.py: Tiles are missing. Ocean???')
xmlFile = glob.glob('demLat_*.wgs84.xml')[0]
fin = open(xmlFile, 'r')
fout = open("tmp.txt", "wt")
for line in fin:
fout.write(line.replace('demLat', cwd + '/demLat'))
fin.close()
fout.close()
os.rename('tmp.txt', xmlFile)
print '\n###############################################'
print 'End of dem_rsmas.py'
print '################################################\n'
Submits a single script as a job.
:param job_name: Name of job.
:param job_file_name: Name of job file.
:param argv: Command line arguments for running job.
:param work_dir: Work directory in which to write job, output, and error files.
:param walltime: Input parameter of walltime for the job.
:param email_notif: If email notifications should be on or not. Defaults to true.
"""
command_line = os.path.basename(argv[0]) + " "
command_line += " ".join(flag for flag in argv[1:] if flag != "--submit")
write_single_job_file(job_name,
job_file_name,
command_line,
work_dir,
email_notif,
walltime=walltime,
queue=os.getenv("QUEUENAME"))
submit_single_job("{0}.job".format(job_file_name), work_dir)
if __name__ == "__main__":
PARAMS = parse_arguments(sys.argv[1::])
messageRsmas.log(
os.path.basename(sys.argv[0]) + " " + " ".join(sys.argv[1::]))
JOBS = write_batch_job_files(PARAMS.file,
PARAMS.outdir,
memory=PARAMS.memory,
walltime=PARAMS.wall,
queue=PARAMS.queue)
submit_batch_jobs(JOBS, PARAMS.file, PARAMS.outdir)
'plmethod', 'range_window', 'azimuth_window', 'cropbox', 'excludeDate',
'azimuthLooks', 'rangeLooks', 'unwMethod', 'textCmd'
]
for value, pref in zip(inpsvalue, prefixletters):
keyvalue = eval('inps.' + value)
if keyvalue is not None:
command = command + ' -' + str(pref) + ' ' + str(keyvalue)
print(command)
out_file = 'out_squeesar_create_runfiles'
command = '(' + command + ' | tee ' + out_file + '.o) 3>&1 1>&2 2>&3 | tee ' + out_file + '.e'
logger.log(loglevel.INFO, command)
messageRsmas.log(command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
logger.log(loglevel.ERROR,
'ERROR making run_files using {}'.format(script))
raise Exception('ERROR making run_files using {}'.format(script))
run_file_list = glob.glob(inps.work_dir + '/run_files_SQ/run_*')
with open(inps.work_dir + '/run_files_list_sq', 'w') as run_file:
for item in run_file_list:
run_file.writelines(item + '\n')
logger.log(loglevel.INFO,
"-----------------Done making Run files-------------------")
if os.path.isdir(json_folder):
logger.log(loglevel.INFO, 'Removing directory: {}'.format(json_folder))
shutil.rmtree(json_folder)
command1 = 'hdfeos5_2json_mbtiles.py ' + hdfeos_file + ' ' + json_folder + ' |& tee out_insarmaps.log'
command2 = 'json_mbtiles2insarmaps.py -u ' + password.insaruser + ' -p ' + password.insarpass + ' --host ' + \
'insarmaps.miami.edu -P rsmastest -U rsmas\@gmail.com --json_folder ' + \
json_folder + ' --mbtiles_file ' + mbtiles_file + ' |& tee -a out_insarmaps.log'
with open(inps.work_dir + '/PYSAR/run_insarmaps', 'w') as f:
f.write(command1 + '\n')
f.write(command2 + '\n')
out_file = 'out_insarmaps'
logger.log(loglevel.INFO, command1)
messageRsmas.log(command1)
command1 = '(' + command1 + ' | tee ' + out_file + '.o) 3>&1 1>&2 2>&3 | tee ' + out_file + '.e'
status = subprocess.Popen(command1, shell=True).wait()
if status is not 0:
logger.log(loglevel.ERROR, 'ERROR in hdfeos5_2json_mbtiles.py')
raise Exception('ERROR in hdfeos5_2json_mbtiles.py')
# TODO: Change subprocess call to get back error code and send error code to logger
logger.log(loglevel.INFO, command2)
messageRsmas.log(command2)
command2 = '(' + command2 + ' | tee -a ' + out_file + '.o) 3>&1 1>&2 2>&3 | tee -a ' + out_file + '.e'
status = subprocess.Popen(command2, shell=True).wait()
if status is not 0:
logger.log(loglevel.ERROR, 'ERROR in json_mbtiles2insarmaps.py')
raise Exception('ERROR in json_mbtiles2insarmaps.py')
def main(argv):
##### Inputs
coreNum = 1
memory = 3700
projectID = 'insarlab'
parallel = 'no'
queue = 'general'
walltime = '5:00'
waitJob = True
if len(sys.argv) > 2:
try: opts, args = getopt.getopt(argv,'h:f:n:w:r:q:p:e:l:',\
['help','file=','number=','walltime=',\
'memory=','queue=','project=','email=',\
'parallel','line=','nowait'])
except getopt.GetoptError:
Usage()
sys.exit(1)
if opts == []:
Usage()
sys.exit(1)
for opt, arg in opts:
if opt in ['-h', '--help']:
Usage()
sys.exit()
elif opt in ['-f', '--file']:
run_file = arg
elif opt in ['-n', '--number']:
coreNum = int(arg)
elif opt in ['-w', '--walltime']:
walltime = arg
elif opt in ['-r', '--memory']:
memory = int(arg)
elif opt in ['-q', '--queue']:
queue = arg
elif opt in ['-p', '--project']:
projectID = arg
elif opt in ['-e', '--email']:
email = arg
elif opt in ['-l', '--line']:
lineNum = int(arg)
elif opt in ['--parallel']:
parallel = 'yes'
elif opt in ['--nowait']:
waitJob = False
elif len(sys.argv) == 2 and os.path.isfile(argv[0]):
run_file = argv[0]
else:
Usage()
sys.exit(1)
run_file = os.path.realpath(run_file)
run_name = os.path.basename(run_file)
workDir = os.path.dirname(run_file)
os.chdir(workDir)
log(os.path.basename(sys.argv[0]) + ' ' + run_name)
#if int(coreNum) >= 16 and queue == 'general':
# msg('More than 16 processors is setted, change job queue to parallel.')
# queue = 'parallel'
try:
lineNum
except:
lineNum = coreNum
memory *= lineNum
#if int(memory) >= 25000:
# msg('Setted memory exceed the maximum, change it to the limit [25 GB].')
# memory = '25000'
##### Clean
prefix_in = 'z_input_' + run_name + '.'
prefix_out = 'z_output_' + run_name + '.'
cleanCmd = 'rm ' + prefix_in + '* ' + prefix_out + '*'
msg(cleanCmd)
os.system(cleanCmd)
##### Input file info
msg('Input file: ' + run_file)
with open(run_file) as f_run:
cmdNum = sum(1 for _ in f_run)
f_run.close()
msg('Number of lines: ' + str(cmdNum))
##### Split run_file according to the job/processor number
jobNum = int((cmdNum - 0.5) / lineNum) + 1
digitNum = len(str(jobNum))
splitCmd = 'split -a ' + str(digitNum) + ' -l ' + str(
lineNum) + ' -d ' + run_file + ' ' + prefix_in
msg(splitCmd)
os.system(splitCmd)
z_inList = glob.glob(prefix_in + '*')
##### Job Info
z_inNum = len(z_inList)
msg('Number of jobs to submit: ' + str(z_inNum))
msg('Queue: ' + queue)
msg('Walltime: ' + walltime)
msg('Memory:' + str(memory))
msg('Processors:' + str(coreNum))
try:
email
msg('Email: ' + email)
except:
pass
##### Write and Submit job file
for z_in in z_inList:
##### Write job setting
count = z_in.split(prefix_in)[1]
job = z_in + '.sh'
f = open(job, 'w')
f.write('#! /bin/bash')
f.write('\n#BSUB -J ' + run_name + '.' + count)
f.write('\n#BSUB -P ' + projectID)
f.write('\n#BSUB -o ' + prefix_out + count + '.%J.o')
f.write('\n#BSUB -e ' + prefix_out + count + '.%J.e')
f.write('\n#BSUB -W ' + walltime)
f.write('\n#BSUB -q ' + queue)
f.write('\n#BSUB -n ' + str(coreNum))
try:
f.write('\n#BSUB -R "rusage[mem=' + str(memory) + ']"')
except:
pass
if queue == 'parallel':
f.write('\n#BSUB -R "span[ptile=16]"')
try:
f.write('\n#BSUB -u ' + email)
f.write('\n#BSUB -N')
except:
pass
##### Write job excutable commands
f.write('\ncd ' + workDir + '\n')
fz_in = open(z_in, 'r')
## parallel computing
if parallel == 'yes' and lineNum > 1:
parallelFile = z_in + '.popen.py'
f_parallel = open(parallelFile, 'w')
f_parallel.write('#! /usr/bin/env python\n')
f_parallel.write('\nfrom subprocess import Popen\n')
for line in fz_in:
f_parallel.write('\nPopen(' + str(line.split()) + ')')
f_parallel.close()
os.system('chmod 755 ' + parallelFile)
f.write('./' + parallelFile)
## sequential computing
else:
for line in fz_in:
f.write(line)
fz_in.close()
f.close()
submitCmd = 'bsub < ' + job
msg('\n' + submitCmd)
os.system(submitCmd)
cleanCmd = 'rm ' + z_in
os.system(cleanCmd)
##### Loop waiting until all jobs are done to exit script
if not waitJob:
msg('Job submission completed, exit without waiting.')
return
msg('\nSleeping until ' + str(z_inNum) + ' jobs are done for ' + run_name)
im = 0
z_outNum = len(glob.glob(prefix_out + '*.o'))
while z_outNum < z_inNum:
time.sleep(1)
if im % 60 == 0: #every minute
msg('Current # of '+prefix_out+'*.o files in '+os.getcwd()+\
': <'+str(z_outNum)+'> out of <'+str(z_inNum)+'> after <'+str(im/60)+'> minutes')
z_outNum = len(glob.glob(prefix_out + '*.o'))
im = im + 1
msg('All ' + str(z_inNum) + ' jobs are done for ' + run_name)
m, s = divmod(im, 60)
h, m = divmod(m, 60)
msg('Total used time: %02d hours %02d mins %02d secs' % (h, m, s))
##### move z_* files into dedicated directory
jobDir = run_file + '_jobs'
msg('Moving z_* files into ' + jobDir)
rmCmd = 'rm -rf ' + jobDir
os.system(rmCmd)
mkCmd = 'mkdir ' + jobDir
os.system(mkCmd)
mvCmd = 'mv ' + prefix_in + '* ' + prefix_out + '* ' + jobDir
os.system(mvCmd)
msg('Finished at ' + str(datetime.datetime.now()))
def run_ssara(inps, run_number=1):
""" Runs ssara_federated_query-cj.py and checks for download issues.
Runs ssara_federated_query-cj.py and checks continuously for whether the data download has hung without
comleting or exited with an error code. If either of the above occur, the function is run again, for a
maxiumum of 10 times.
Parameters: run_number: int, the current iteration the wrapper is on (maxiumum 10 before quitting)
Returns: status_cod: int, the status of the donwload (0 for failed, 1 for success)
"""
logger.log(loglevel.INFO, "RUN NUMBER: %s", str(run_number))
if run_number > 10:
return 0
logger.log(loglevel.INFO, "PASSED RUN NUMBER > 10")
# Compute SSARA options to use
dataset_template = Template(inps.template)
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
logger.log(loglevel.INFO, "GENERATED SSARAOPT STRING")
# Runs ssara_federated_query-cj.py with proper options
ssara_call = ['ssara_federated_query-cj.py'
] + ssaraopt + ['--print', '--download']
print(' '.join(ssara_call))
messageRsmas.log(' '.join(ssara_call))
ssara_process = subprocess.Popen(ssara_call)
logger.log(loglevel.INFO, "STARTED PROCESS")
completion_status = ssara_process.poll(
) # the completion status of the process
hang_status = False # whether or not the download has hung
wait_time = 2 # 10 wait time in 'minutes' to determine hang status
prev_size = -1 # initial download directory size
i = 0 # index for waiting periods (for calculation of total time only)
logger.log(loglevel.INFO, "INITIAL COMPLETION STATUS: %s",
str(completion_status))
# while the process has not completed
#import pdb; pdb.set_trace()
while completion_status is None:
i = i + 1
# Computer the current download directory size
curr_size = int(
subprocess.check_output(['du', '-s',
os.getcwd()]).split()[0].decode('utf-8'))
# Compare the current and previous directory sizes to determine determine hang status
if prev_size == curr_size:
hang_status = True
logger.log(loglevel.WARNING, "SSARA Hung")
ssara_process.terminate(
) # teminate the process beacause download hung
break
# break the completion loop
prev_size = curr_size # store current size for comparison after waiting
time.sleep(
60 * wait_time
) # wait 'wait_time' minutes before continuing (checking for completion)
completion_status = ssara_process.poll()
logger.log(
loglevel.INFO, "{} minutes: {:.1f}GB, completion_status {}".format(
i * wait_time, curr_size / 1024 / 1024, completion_status))
exit_code = completion_status # get the exit code of the command
ssara_process.terminate()
logger.log(loglevel.INFO, "EXIT CODE: %s", str(exit_code))
bad_codes = [137, -9]
# If the exit code is one that signifies an error, rerun the entire command
if exit_code in bad_codes or hang_status:
if exit_code in bad_codes:
logger.log(loglevel.WARNING,
"Exited with bad exit code, running again")
if hang_status:
logger.log(loglevel.WARNING, "Hanging, running again")
run_ssara(inps, run_number=run_number + 1)
return 0
