def get_datasets_to_process(template_files, dataset=None):
"""
Extracts project names from the list of tenplate files
:param template_files: the list of template files to parse the dataset name from
:param dataset: a single dataset to return
:return: a list of dataset names
"""
# Obtains list of datasets to run processSentinel on
if dataset:
datasets = [dataset]
else:
datasets = [putils.get_project_name(template) for template in template_files]
return datasets
def main(iargs=None):
inps = putils.cmd_line_parse(iargs, script='upload_data_products')
if inps.image_products_flag:
inps.mintpy_products_flag = False
os.chdir(inps.work_dir)
if not iargs is None:
input_arguments = iargs
else:
input_arguments = sys.argv[1::]
message_rsmas.log(inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(input_arguments))
# get DATA_SERVER and return if it does not exist
DATA_SERVER = '[email protected]'
#try:
# DATA_SERVER = os.getenv('DATA_SERVER')
#except:
# return
project_name = putils.get_project_name(inps.custom_template_file)
if inps.mintpy_products_flag:
REMOTE_DIR = '/data/HDF5EOS/'
destination = DATA_SERVER + ':' + REMOTE_DIR
scp_list = [
'/mintpy/pic',
'/mintpy/*.he5',
'/mintpy/inputs',
'/remora_*'
]
if inps.mintpy_products_all_flag:
scp_list = [ '/mintpy' ]
command = 'ssh ' + DATA_SERVER + ' mkdir -p ' + REMOTE_DIR + project_name + '/mintpy'
print (command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
raise Exception('ERROR in upload_data_products.py')
for pattern in scp_list:
if ( len(glob.glob(inps.work_dir + '/' + pattern)) >= 1 ):
command = 'scp -r ' + inps.work_dir + pattern + ' ' + destination + project_name + '/'.join(pattern.split('/')[0:-1])
print (command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
raise Exception('ERROR in upload_data_products.py')
print ('\nAdjusting permissions:')
command = 'ssh ' + DATA_SERVER + ' chmod -R u=rwX,go=rX ' + REMOTE_DIR + project_name
print (command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
raise Exception('ERROR in upload_data_products.py')
if inps.image_products_flag:
REMOTE_DIR = '/data/image_products/'
destination = DATA_SERVER + ':' + REMOTE_DIR
rsync_list = [
'/image_products/*',
]
command = 'ssh ' + DATA_SERVER + ' mkdir -p ' + REMOTE_DIR + project_name
print (command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
raise Exception('ERROR in upload_data_products.py')
for pattern in rsync_list:
command = 'rsync -avuz -e ssh --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r ' + inps.work_dir + pattern + ' ' + destination + project_name + '/'.join(pattern.split('/')[0:-1])
print (command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
raise Exception('ERROR in upload_data_products.py')
return None
return None
polygon = '--intersectsWith=\'Polygon(({:.2f} {:.2f}, {:.2f} {:.2f}, {:.2f} {:.2f}, {:.2f} {:.2f}, {:.2f} {:.2f}))\''.format(
min_lon, min_lat, min_lon, max_lat, max_lon, max_lat, max_lon, min_lat,
min_lon, min_lat)
# add --polygon and remove --frame option
ssaraopt.insert(2, polygon)
ssaraopt = [x for x in ssaraopt if not x[0:7] == '--frame']
return ssaraopt
if __name__ == "__main__":
inps = command_line_parse(sys.argv[1:])
inps.project_name = putils.get_project_name(
custom_template_file=inps.template)
inps.work_dir = putils.get_work_directory(None, inps.project_name)
inps.slc_dir = inps.work_dir + "/SLC"
#########################################
# Submit job
#########################################
if inps.submit_flag:
job_file_name = 'download_ssara_rsmas'
job_name = inps.template.split(os.sep)[-1].split('.')[0]
work_dir = inps.work_dir
wall_time = '24:00'
js.submit_script(job_name, job_file_name, sys.argv[:], work_dir,
wall_time)
sys.exit(0)
def run_operations(args):
"""
Runs the entire data processing routing from start to finish. Steps as follows:
1. Generates the template files for all of the datasets in the provivded CSV or Google Sheet file
2. Gets the dataset names for each template files
for each dataset:
3. Gets the newest available image date from `ssara_federated_query.py`
4. Gets the last image date downloaded from the `stored_date.date` file
5. Runs `process_rsmas.py` if their is new data available
6. Waits for all of the output files from submitted `process_rsmas` calls to exist
:param args: command line arguments to use
"""
inps = command_line_parse(args)
# Remove and reinitiate $OPERATIONS directory
if inps.restart:
shutil.rmtree(OPERATIONS_DIRECTORY)
initiate_operations()
template_files = []
# inps.sheet_ids is an array of sheets.
# Each use of the --sheet_id command line parameter adds another array to the inps.sheet_id variable.
for sheet in inps.sheet_ids:
template_files += generate_templates_with_options(
inps.template_csv, inps.dataset, sheet[0])
datasets = get_datasets_to_process(template_files, inps.dataset)
logger_run_operations.log(loglevel.INFO,
"Datasets to Process: {}".format(datasets))
output_files = []
job_to_dset = {}
for dset in datasets:
template_file = "{}/{}.template".format(TEMPLATE_DIRECTORY, dset)
logger_run_operations.log(loglevel.INFO,
"{}: {}".format(dset, template_file))
newest_date = get_newest_data_date(template_file)
last_date = get_last_downloaded_date(dset)
print(newest_date)
print(last_date)
if newest_date > last_date:
print("Submitting minsar_wrapper.bash for {}".format(
putils.get_project_name(template_file)))
subprocess.Popen(["minsar_wrapper.bash", template_file],
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
else:
print("SKIPPING")
print(
"-------------- run_operations.py has completed. Exiting now. -------------- \n\n\n\n\n\n\n"
)
sys.exit(0)
return
def run_generate_chunk_template_files(inps):
""" create e*chunk*.template files """
project_name = putils.get_project_name(inps.custom_template_file)
location_name, sat_direction, sat_track = putils.split_project_name(
project_name)
location_name = location_name.split('Big')[0]
chunk_templates_dir = inps.work_dir + '/chunk_templates'
chunk_templates_dir_string = '$SCRATCHDIR/' + project_name + '/chunk_templates'
os.makedirs(chunk_templates_dir, exist_ok=True)
command_list = []
sleep_time = 0
command_options = ''
if inps.start_step is None and inps.do_step is None:
inps.start_step = 'jobfiles'
if inps.do_step is not None:
command_options = command_options + ' --dostep ' + inps.do_step
else:
if inps.start_step is not None:
command_options = command_options + ' --start ' + inps.start_step
if inps.end_step is not None:
command_options = command_options + ' --end ' + inps.end_step
prefix = 'tops'
bbox_list = inps.template[prefix + 'Stack.boundingBox'].split(' ')
bbox_list[0] = bbox_list[0].replace(
"\'", ''
) # this does ["'-8.75", '-7.8', '115.0', "115.7'"] (needed for run_operations.py, run_operations
bbox_list[1] = bbox_list[1].replace(
"\'", ''
) # --> ['-8.75', '-7.8', '115.0', '115.7'] (should be modified so that this is not needed)
bbox_list[2] = bbox_list[2].replace("\'", '')
bbox_list[3] = bbox_list[3].replace("\'", '')
tmp_min_lat = float(bbox_list[0])
tmp_max_lat = float(bbox_list[1])
min_lat = math.ceil(tmp_min_lat)
max_lat = math.floor(tmp_max_lat)
lat = min_lat
chunk_number = 0
chunk1_option = ''
while lat < max_lat:
tmp_min_lat = lat
tmp_max_lat = lat + inps.lat_step
chunk_name = [
location_name + 'Chunk' + str(int(lat)) + sat_direction + sat_track
]
chunk_template_file = chunk_templates_dir + '/' + chunk_name[
0] + '.template'
chunk_template_file_base = chunk_name[0] + '.template'
shutil.copy(inps.custom_template_file, chunk_template_file)
chunk_bbox_list = bbox_list
chunk_bbox_list[0] = str(float(tmp_min_lat - inps.lat_margin))
chunk_bbox_list[1] = str(float(tmp_max_lat + inps.lat_margin))
print(chunk_name, tmp_min_lat, tmp_max_lat, chunk_bbox_list)
custom_tempObj = Template(inps.custom_template_file)
custom_tempObj.options['topsStack.boundingBox'] = ' '.join(
chunk_bbox_list)
slcDir = '$SCRATCHDIR/' + project_name + '/SLC'
demDir = '$SCRATCHDIR/' + project_name + '/DEM'
custom_tempObj.options['topsStack.slcDir'] = slcDir
custom_tempObj.options['topsStack.demDir'] = demDir
#if inps.download_flag in [ True , 'True']:
# del(custom_tempObj.options['topsStack.slcDir'])
if 'download' in command_options:
del (custom_tempObj.options['topsStack.slcDir'])
putils.write_template_file(chunk_template_file, custom_tempObj)
putils.beautify_template_file(chunk_template_file)
chunk_number = chunk_number + 1
if chunk_number > 1 and inps.bash_script == 'minsarApp.bash':
chunk1_option = ' --no_download_ECMWF '
command = inps.bash_script + ' ' + chunk_templates_dir_string + '/' + chunk_template_file_base + command_options + chunk1_option + ' --sleep ' + str(
sleep_time) + ' &'
command_list.append(command)
lat = lat + inps.lat_step
sleep_time = sleep_time + inps.wait_time
chunk1_option = ''
commands_file = inps.work_dir + '/minsar_commands.txt'
f = open(commands_file, "w")
print()
for item in command_list:
print(item)
f.write(item + '\n')
print()
f.write('\n')
return
def main(iargs=None):
inps = putils.cmd_line_parse(iargs, script='upload_data_products')
os.chdir(inps.work_dir)
if not iargs is None:
message_rsmas.log(
inps.work_dir,
os.path.basename(__file__) + ' ' + ' '.join(iargs[:]))
else:
message_rsmas.log(
inps.work_dir,
os.path.basename(__file__) + ' ' + ' '.join(sys.argv[1::]))
# get DATA_SERVER and return if it does not exist
DATA_SERVER = '[email protected]'
#try:
# DATA_SERVER = os.getenv('DATA_SERVER')
#except:
# return
project_name = putils.get_project_name(inps.custom_template_file)
if inps.flag_mintpy_products:
REMOTE_DIR = '/data/HDF5EOS/'
destination = DATA_SERVER + ':' + REMOTE_DIR
rsync_list = ['/mintpy/inputs', '/mintpy/pic', '/mintpy/*.he5']
command = 'ssh ' + DATA_SERVER + ' mkdir -p ' + REMOTE_DIR + project_name
print(command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
raise Exception('ERROR in upload_data_products.py')
for pattern in rsync_list:
command = 'rsync -avuz -e ssh --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r ' + inps.work_dir + pattern + ' ' + destination + project_name + '/'.join(
pattern.split('/')[0:-1])
print(command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
raise Exception('ERROR in upload_data_products.py')
# temporary rsync of full mintpy folder
# text for test PR 2
command = 'rsync -avuz -e ssh --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r ' + inps.work_dir + '/mintpy ' + destination + project_name + '/full_mintpy'
print(command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
raise Exception('ERROR in upload_data_products.py')
return None
if inps.flag_image_products:
REMOTE_DIR = '/data/image_products/'
destination = DATA_SERVER + ':' + REMOTE_DIR
rsync_list = [
'/image_products/*',
]
command = 'ssh ' + DATA_SERVER + ' mkdir -p ' + REMOTE_DIR + project_name
print(command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
raise Exception('ERROR in upload_data_products.py')
for pattern in rsync_list:
command = 'rsync -avuz -e ssh --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r ' + inps.work_dir + pattern + ' ' + destination + project_name + '/'.join(
pattern.split('/')[0:-1])
print(command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
raise Exception('ERROR in upload_data_products.py')
return None
return None
def run_generate_chunk_template_files(inps):
""" create e*chunk*.template files """
project_name = putils.get_project_name(inps.custom_template_file)
location_name, sat_direction, sat_track = putils.split_project_name(
project_name)
chunk_templates_dir = inps.work_dir + '/chunk_templates'
os.makedirs(chunk_templates_dir, exist_ok=True)
commands_file = inps.work_dir + '/minsar_commands.txt'
f = open(commands_file, "w")
if inps.download_flag == True:
minsarApp_option = '--start download'
else:
minsarApp_option = '--start dem'
prefix = 'tops'
bbox_list = inps.template[prefix + 'Stack.boundingBox'].split(' ')
bbox_list[0] = bbox_list[0].replace(
"\'", ''
) # this does ["'-8.75", '-7.8', '115.0', "115.7'"] (needed for run_operations.py, run_operations
bbox_list[1] = bbox_list[1].replace(
"\'", ''
) # --> ['-8.75', '-7.8', '115.0', '115.7'] (should be modified so that this is not needed)
bbox_list[2] = bbox_list[2].replace("\'", '')
bbox_list[3] = bbox_list[3].replace("\'", '')
tmp_min_lat = float(bbox_list[0])
tmp_max_lat = float(bbox_list[1])
min_lat = math.ceil(tmp_min_lat)
max_lat = math.floor(tmp_max_lat)
lat = min_lat
while lat < max_lat:
tmp_min_lat = lat
tmp_max_lat = lat + inps.lat_step
chunk_name = [
location_name + 'Chunk' + str(int(lat)) + sat_direction + sat_track
]
chunk_template_file = chunk_templates_dir + '/' + chunk_name[
0] + '.template'
shutil.copy(inps.custom_template_file, chunk_template_file)
chunk_bbox_list = bbox_list
chunk_bbox_list[0] = str(float(tmp_min_lat - inps.lat_margin))
chunk_bbox_list[1] = str(float(tmp_max_lat + inps.lat_margin))
print(chunk_name, tmp_min_lat, tmp_max_lat, chunk_bbox_list)
custom_tempObj = Template(inps.custom_template_file)
custom_tempObj.options['topsStack.boundingBox'] = ' '.join(
chunk_bbox_list)
if inps.download_flag in [True, 'True']:
del (custom_tempObj.options['topsStack.slcDir'])
putils.write_template_file(chunk_template_file, custom_tempObj)
putils.beautify_template_file(chunk_template_file)
minsar_command = 'minsarApp.bash ' + chunk_template_file + ' ' + minsarApp_option
f.write(minsar_command + '\n')
lat = lat + inps.lat_step
return
