def list_jobs(config_file, step):
tiles_full_info = initialization.init_tiles_full_info(config_file)
filename = str(step) + ".jobs"
if not (os.path.exists(cfg['out_dir'])):
os.mkdir(cfg['out_dir'])
if step in [2,4]: #preprocessing, processing
f = open(os.path.join(cfg['out_dir'],filename),'w')
for tile in tiles_full_info:
tile_dir = tile['directory']
f.write(tile_dir + ' ' + str(step) + '\n')
f.close()
elif step in [3,5,7]: # global values, global extent, finalization
f = open(os.path.join(cfg['out_dir'],filename),'w')
f.write('all_tiles ' + str(step) + '\n')
f.close()
elif step ==6 : # compute dsm
f = open(os.path.join(cfg['out_dir'],filename),'w')
for i in range(cfg['dsm_nb_tiles']):
f.write('dsm_'+ str(i) + ' ' + str(step) + '\n')
f.close()
else:
print "Unkown step required: %s" % str(step)
def execute_job(config_file,params):
"""
Execute a job.
Args:
- json config file
- params ( <==> [tile_dir,step,...])
"""
tile_dir = params[0]
step = int(params[1])
tiles_full_info = initialization.init_tiles_full_info(config_file)
if not (tile_dir == 'all_tiles' or 'dsm' in tile_dir ):
for tile in tiles_full_info:
if tile_dir == tile['directory']:
tile_to_process = tile
break
try:
if step == 2:#"preprocess_tiles":
print 'preprocess_tiles on %s ...' % tile_to_process
preprocess_tile(tile_to_process)
if step == 3:#"global_values":
print 'global values ...'
global_values(tiles_full_info)
if step == 4:#"process_tiles" :
print 'process_tiles on %s ...' % tile_to_process
process_tile(tile_to_process)
if step == 5:#"global extent" :
print 'global extent ...'
global_extent(tiles_full_info)
if step == 6:#"compute_dsm" :
print 'compute_dsm ...'
current_tile=int(tile_dir.split('_')[1]) # for instance, dsm_2 becomes 2
compute_dsm([config_file,cfg['dsm_nb_tiles'],current_tile])
if step == 7:#"global_finalization":
print 'global finalization...'
global_finalization(tiles_full_info)
except KeyboardInterrupt:
pool.terminate()
sys.exit(1)
except common.RunFailure as e:
print "FAILED call: ", e.args[0]["command"]
print "\toutput: ", e.args[0]["output"]
def main(config_file):
"""
Launch the entire s2p pipeline with the parameters given by a json file.
It is a succession of five steps:
initialization
preprocessing
global_values
processing
global_finalization
Args:
config_file: path to a json configuration file
"""
t0 = time.time()
# initialization
initialization.init_dirs_srtm_roi(config_file)
tiles_full_info = initialization.init_tiles_full_info(config_file)
show_progress.total = len(tiles_full_info)
# multiprocessing setup
nb_workers = multiprocessing.cpu_count() # nb of available cores
if cfg['max_nb_threads']:
nb_workers = min(nb_workers, cfg['max_nb_threads'])
# omp_num_threads: should not exceed nb_workers when multiplied by the
# number of tiles
cfg['omp_num_threads'] = max(1, int(nb_workers / len(tiles_full_info)))
# do the job
print '\npreprocessing tiles...'
launch_parallel_calls(preprocess_tile, tiles_full_info, nb_workers)
print "Elapsed time:", datetime.timedelta(seconds=int(time.time() - t0))
print '\ncomputing global values...'
global_values(tiles_full_info)
print "Elapsed time:", datetime.timedelta(seconds=int(time.time() - t0))
print '\nprocessing tiles...'
launch_parallel_calls(process_tile, tiles_full_info, nb_workers)
print "Elapsed time:", datetime.timedelta(seconds=int(time.time() - t0))
print '\nglobal finalization...'
global_finalization(tiles_full_info)
print "Total runtime:", datetime.timedelta(seconds=int(time.time() - t0))
# cleanup
common.garbage_cleanup()
def main(config_file):
"""
Launch the entire s2p pipeline with the parameters given by a json file.
It is a succession of five steps:
initialization
preprocessing
global_values
processing
global_finalization
Args:
config_file: path to a json configuration file
"""
t0 = time.time()
# initialization
initialization.init_dirs_srtm_roi(config_file)
tiles_full_info = initialization.init_tiles_full_info(config_file)
show_progress.total = len(tiles_full_info)
# multiprocessing setup
nb_workers = multiprocessing.cpu_count() # nb of available cores
if cfg['max_nb_threads']:
nb_workers = min(nb_workers, cfg['max_nb_threads'])
# omp_num_threads: should not exceed nb_workers when multiplied by the
# number of tiles
cfg['omp_num_threads'] = max(1, int(nb_workers / len(tiles_full_info)))
# do the job
print '\npreprocessing tiles...'
launch_parallel_calls(preprocess_tile, tiles_full_info, nb_workers)
print "Elapsed time:", datetime.timedelta(seconds=int(time.time() - t0))
print '\ncomputing global values...'
global_values(tiles_full_info)
print "Elapsed time:", datetime.timedelta(seconds=int(time.time() - t0))
print '\nprocessing tiles...'
launch_parallel_calls(process_tile, tiles_full_info, nb_workers)
print "Elapsed time:", datetime.timedelta(seconds=int(time.time() - t0))
print '\nglobal finalization...'
global_finalization(tiles_full_info)
print "Total runtime:", datetime.timedelta(seconds=int(time.time() - t0))
# cleanup
common.garbage_cleanup()
def list_jobs(config_file, step):
"""
"""
tiles_full_info = initialization.init_tiles_full_info(config_file)
filename = str(step) + ".jobs"
if not (os.path.exists(cfg['out_dir'])):
os.mkdir(cfg['out_dir'])
if step in [2, 4]:
f = open(os.path.join(cfg['out_dir'], filename), 'w')
for tile in tiles_full_info:
tile_dir = tile['directory']
f.write(tile_dir + ' ' + str(step) + '\n')
f.close()
elif step in [3, 5]:
f = open(os.path.join(cfg['out_dir'], filename), 'w')
f.write('all_tiles ' + str(step) + '\n')
f.close()
else:
print "Unkown step required: %s" % str(step)
def execute_job(config_file, tile_dir, step):
"""
Execute a job.
Args:
- json config file
- tile_dir
- step
"""
tiles_full_info = initialization.init_tiles_full_info(config_file)
if not tile_dir == 'all_tiles':
for tile in tiles_full_info:
if tile_dir == tile['directory']:
tile_to_process = tile
print tile_to_process
break
try:
if step == 2:
print 'preprocess_tiles on %s ...' % tile_to_process
preprocess_tile(tile_to_process)
if step == 3:
print 'global values...'
global_values(tiles_full_info)
if step == 4:
print 'process_tiles on %s ...' % tile_to_process
process_tile(tile_to_process)
if step == 5:
print 'global finalization...'
global_finalization(tiles_full_info)
except common.RunFailure as e:
print "FAILED call: ", e.args[0]["command"]
print "\toutput: ", e.args[0]["output"]
def main(config_file, step=None, clusterMode=None, misc=None):
"""
Launch the entire s2p pipeline with the parameters given in a json file.
It is a succession of six steps:
initialization
preprocessing
global_values
processing
compute dsms
global_finalization
Args:
config_file: path to a json configuration file
step: integer between 1 and 5 specifying which step to run. Default
value is None. In that case all the steps are run.
"""
print_elapsed_time.t0 = datetime.datetime.now()
if clusterMode == 'list_jobs':
list_jobs(config_file, step)
elif clusterMode == 'job':
cfg['omp_num_threads'] = 1
execute_job(config_file,misc)
else:
# determine which steps to run
steps = [step] if step else [1, 2, 3, 4, 5, 6, 7]
# initialization (has to be done whatever the queried steps)
initialization.init_dirs_srtm(config_file)
tiles_full_info = initialization.init_tiles_full_info(config_file)
# multiprocessing setup
nb_workers = multiprocessing.cpu_count() # nb of available cores
if cfg['max_nb_threads']:
nb_workers = min(nb_workers, cfg['max_nb_threads'])
# omp_num_threads: should not exceed nb_workers when multiplied by the
# number of tiles
cfg['omp_num_threads'] = max(1, int(nb_workers / len(tiles_full_info)))
# do the job
if 2 in steps:
print '\npreprocessing tiles...'
show_progress.total = len(tiles_full_info)
launch_parallel_calls(preprocess_tile, tiles_full_info, nb_workers)
print_elapsed_time()
if 3 in steps:
print '\ncomputing global values...'
global_values(tiles_full_info)
print_elapsed_time()
if 4 in steps:
print '\nprocessing tiles...'
show_progress.total = len(tiles_full_info)
launch_parallel_calls(process_tile, tiles_full_info, nb_workers)
print_elapsed_time()
if 5 in steps:
print '\ncomputing global extent...'
global_extent(tiles_full_info)
print_elapsed_time()
if 6 in steps:
print '\ncompute dsm...'
args = []
for i in range(cfg['dsm_nb_tiles']):
args.append([config_file, cfg['dsm_nb_tiles'], i])
show_progress.total = cfg['dsm_nb_tiles']
launch_parallel_calls(compute_dsm, args, nb_workers)
print_elapsed_time()
if 7 in steps:
print '\nglobal finalization...'
global_finalization(tiles_full_info)
print_elapsed_time()
# cleanup
print_elapsed_time(since_first_call=True)
common.garbage_cleanup()
def main(config_file, step=None, clusterMode=None, misc=None):
"""
Launch the entire s2p pipeline with the parameters given in a json file.
It is a succession of five steps:
initialization
preprocessing
global_values
processing
global_finalization
Args:
config_file: path to a json configuration file
step: integer between 1 and 5 specifying which step to run. Default
value is None. In that case all the steps are run.
"""
if clusterMode == 'list_jobs':
list_jobs(config_file, step)
elif clusterMode == 'job':
cfg['omp_num_threads'] = 1
execute_job(config_file, misc[0], int(misc[1]))
else:
# determine which steps to run
steps = [step] if step else [1, 2, 3, 4, 5]
# initialization (has to be done whatever the queried steps)
initialization.init_dirs_srtm(config_file)
tiles_full_info = initialization.init_tiles_full_info(config_file)
show_progress.total = len(tiles_full_info)
print_elapsed_time.t0 = datetime.datetime.now()
# multiprocessing setup
nb_workers = multiprocessing.cpu_count() # nb of available cores
if cfg['max_nb_threads']:
nb_workers = min(nb_workers, cfg['max_nb_threads'])
# omp_num_threads: should not exceed nb_workers when multiplied by the
# number of tiles
cfg['omp_num_threads'] = max(1, int(nb_workers / len(tiles_full_info)))
# do the job
if 2 in steps:
print '\npreprocessing tiles...'
launch_parallel_calls(preprocess_tile, tiles_full_info, nb_workers)
print_elapsed_time()
if 3 in steps:
print '\ncomputing global values...'
global_values(tiles_full_info)
print_elapsed_time()
if 4 in steps:
print '\nprocessing tiles...'
launch_parallel_calls(process_tile, tiles_full_info, nb_workers)
print_elapsed_time()
if 5 in steps:
print '\nglobal finalization...'
global_finalization(tiles_full_info)
print_elapsed_time()
# cleanup
common.garbage_cleanup()
