def process_synthetics(params, first_job, last_job):
"""
Processes the synthetic seismograms in parallel.
"""
event_list = params['event_list']
forward_run_dir = params['forward_run_dir']
lasif_path = params['lasif_path']
lasif_scratch_path = params['lasif_scratch_path']
iteration_name = params['iteration_name']
chosen_event = event_list[first_job:last_job + 1]
source_dirs = [
os.path.join(forward_run_dir, event, 'OUTPUT_FILES')
for event in chosen_event
]
dest_dirs = [
os.path.join(lasif_path, 'SYNTHETICS', event,
'ITERATION_%s' % (iteration_name))
for event in chosen_event
]
dest_dirs_scratch = [
os.path.join(lasif_scratch_path, 'SYNTHETICS', event,
'ITERATION_%s' % (iteration_name))
for event in chosen_event
]
if __name__ == 'specfem_control.control':
pool = Pool(processes=cpu_count())
print "Using %d cpus..." % (cpu_count())
#pool.map(_copy_files_and_tar, zip(source_dirs, dest_dirs, repeat(True)))
pool.map(_copy_files_and_tar,
zip(source_dirs, dest_dirs_scratch, repeat(False)))
utils.print_blu('Done.')
def clean_event_kernels(params):
"""
Goes through the simulation directories for an iteration, and cleans out
the individual event kernels (make sure you've summed them already!).
"""
forward_run_dir = params['forward_run_dir']
event_list = params['event_list']
for dir in sorted(os.listdir(forward_run_dir)):
if dir in event_list:
utils.print_ylw("Cleaning " + dir + "...")
databases_mpi = os.path.join(forward_run_dir, dir, 'DATABASES_MPI')
for file in os.listdir(databases_mpi):
if file.endswith('kernel.bin'):
os.remove(os.path.join(databases_mpi, file))
utils.print_blu('Done.')
def clean_attenuation_dumps(params):
"""
Goes through the simulation directories for an iteration, and cleans out
the massive adios attenuation snapshot files.
"""
forward_run_dir = params['forward_run_dir']
event_list = params['event_list']
for dir in sorted(os.listdir(forward_run_dir)):
if dir in event_list:
utils.print_ylw("Cleaning " + dir + "...")
databases_mpi = os.path.join(forward_run_dir, dir, 'DATABASES_MPI')
for file in os.listdir(databases_mpi):
if file.startswith('save_frame_at'):
os.remove(os.path.join(databases_mpi, file))
utils.print_blu('Done.')
def clean_failed(params):
"""
Deletes error files after a failed run.
"""
forward_run_dir = params['forward_run_dir']
for dir in os.listdir(forward_run_dir):
utils.print_ylw("Cleaning " + dir + "...")
if os.path.exists(os.path.join(forward_run_dir, dir, 'OUTPUT_FILES')):
output_files = os.path.join(forward_run_dir, dir, 'OUTPUT_FILES')
for file in os.listdir(output_files):
if 'error' in file:
os.remove(os.path.join(output_files, file))
if 'sac' in file:
os.remove(os.path.join(output_files, file))
if os.path.exists(os.path.join(forward_run_dir, dir, 'core')):
os.remove(os.path.join(forward_run_dir, dir, 'core'))
utils.print_blu('Done.')
def distribute_adjoint_sources(params):
"""
Searches through the OUTPUT directory of the lasif project, and distributes
the adjoint sources to their appropriate simulated directories. Also writes
the STATIONS_ADJOINT file.
"""
lasif_output_dir = os.path.join(params['lasif_path'], 'OUTPUT')
forward_run_dir = params['forward_run_dir']
for dir in sorted(os.listdir(lasif_output_dir))[::-1]:
if 'adjoint' not in dir:
continue
adjoint_source_dir = os.path.join(lasif_output_dir, dir)
event_name = adjoint_source_dir[adjoint_source_dir.find('GCMT'):]
solver_data_dir = os.path.join(forward_run_dir, event_name, 'DATA')
solver_sem_dir = os.path.join(forward_run_dir, event_name, 'SEM')
adjoint_stations = os.listdir(adjoint_source_dir)
utils.print_ylw("Copying adjoint sources for " + event_name + "...")
utils.mkdir_p(solver_sem_dir)
# This is necessary because the adjoint source names expected by
# specfem3d_globe are opposite of what lasif puts out. Could just fix
# this in lasif instead. This line would then reduce to a
# 'copy_directory'.
for old_name in os.listdir(adjoint_source_dir):
fields = old_name.split('.')
new_name = fields[1] + '.' + fields[0] + '.' + fields[2] + '.' + \
fields[3]
utils.safe_copy_file(os.path.join(adjoint_source_dir, old_name),
os.path.join(solver_sem_dir, new_name))
adjoint_stations = sorted(
list(
set([
fields0.split('.')[0] + '.' + fields0.split('.')[1]
for fields0 in adjoint_stations
])))
with open(os.path.join(solver_data_dir, 'STATIONS_ADJOINT'), 'w') as \
sta_adj:
sta = open(os.path.join(solver_data_dir, 'STATIONS'), 'r')
for line in sta:
if line.split()[0] + '.' + line.split()[1] in adjoint_stations:
sta_adj.write(line)
utils.print_blu('Done.')
def setup_new_iteration(params, old_iteration, new_iteration):
"""
Sets up a new iteration, and links the mesh files from the old iteration to
the new one.
"""
forward_stage_dir = params['forward_stage_dir']
event_list = params['event_list']
params.update({'iteration_name': new_iteration})
new_forward_run_dir = os.path.join(forward_stage_dir, new_iteration)
params.update({'forward_run_dir': new_forward_run_dir})
setup_solver(params)
old_database_dir = os.path.join(forward_stage_dir, old_iteration, 'mesh',
'DATABASES_MPI')
new_database_dir = os.path.join(forward_stage_dir, new_iteration, 'mesh',
'DATABASES_MPI')
old_optimization_dir = os.path.join(forward_stage_dir, old_iteration,
'OPTIMIZATION', 'PROCESSED_KERNELS')
new_optimization_dir = os.path.join(forward_stage_dir, new_iteration,
'OPTIMIZATION', 'PROCESSED_KERNELS')
utils.print_ylw("Copying mesh information...")
utils.copy_directory(old_database_dir, new_database_dir)
utils.print_ylw("Copying kernels...")
utils.copy_directory(old_optimization_dir,
new_optimization_dir,
ends='smooth.bin')
utils.print_ylw("Copying DATA files...")
for event in event_list:
old_data_dir = os.path.join(forward_stage_dir, old_iteration, event,
'DATA')
new_data_dir = os.path.join(forward_stage_dir, new_iteration, event,
'DATA')
utils.copy_directory(old_data_dir,
new_data_dir,
only=['Par_file', 'CMTSOLUTION', 'STATIONS'])
old_mesh_dir = os.path.join(forward_stage_dir, old_iteration, 'mesh',
'DATA')
new_mesh_dir = os.path.join(forward_stage_dir, new_iteration, 'mesh',
'DATA')
utils.copy_directory(old_mesh_dir,
new_mesh_dir,
only=['Par_file', 'CMTSOLUTION', 'STATIONS'])
utils.print_blu('Done.')
def delete_adjoint_sources_for_iteration(params):
"""
Deletes the directories on both /scratch and /project which contain the
adjoint sources for this iteration. As well, cleans the SEM and
STATIONS_ADJOINT files for the solver.
"""
forward_run_dir = params['forward_run_dir']
lasif_scratch_dir_output = os.path.join(params['lasif_scratch_path'],
'OUTPUT')
lasif_dir_output = os.path.join(params['lasif_path'], 'OUTPUT')
lasif_scratch_dir_adjoint = os.path.join(params['lasif_scratch_path'],
'ADJOINT_SOURCES_AND_WINDOWS')
lasif_project_dir_adjoint = os.path.join(params['lasif_path'],
'ADJOINT_SOURCES_AND_WINDOWS')
iteration_name = params['iteration_name']
utils.print_ylw("Cleaning forward runs...")
for dir in os.listdir(forward_run_dir):
if not os.path.exists(os.path.join(forward_run_dir, dir, 'SEM')):
continue
for file in os.listdir(os.path.join(forward_run_dir, dir, 'SEM')):
os.remove(os.path.join(forward_run_dir, dir, 'SEM', file))
utils.print_ylw("Cleaning LASIF scratch...")
for dir in os.listdir(lasif_scratch_dir_output):
if iteration_name and 'adjoint_sources' in dir:
shutil.rmtree(os.path.join(lasif_scratch_dir_output, dir))
for dir in os.listdir(lasif_scratch_dir_adjoint):
for sub_dir in os.listdir(os.path.join(lasif_scratch_dir_adjoint,
dir)):
shutil.rmtree(os.path.join(lasif_scratch_dir_adjoint, dir,
sub_dir))
utils.print_ylw("Cleaning LASIF project...")
for dir in os.listdir(lasif_dir_output):
if iteration_name and 'adjoint_sources' in dir:
shutil.rmtree(os.path.join(lasif_dir_output, dir))
for dir in os.listdir(lasif_project_dir_adjoint):
for sub_dir in os.listdir(os.path.join(lasif_project_dir_adjoint,
dir)):
shutil.rmtree(os.path.join(lasif_project_dir_adjoint, dir,
sub_dir))
utils.print_blu('Done.')
def prepare_solver(params):
"""
Sets up symbolic link to generated mesh files.
"""
forward_run_dir = params['forward_run_dir']
event_list = params['event_list']
utils.print_ylw("Preparing solver directories...")
databases_mpi = os.path.join(params['forward_run_dir'], 'mesh',
'DATABASES_MPI')
output_files = os.path.join(params['forward_run_dir'], 'mesh',
'OUTPUT_FILES')
for dir in sorted(os.listdir(params['forward_run_dir'])):
if dir not in event_list:
continue
utils.print_ylw('Linking ' + dir + '...')
dest_mpi = os.path.join(forward_run_dir, dir, 'DATABASES_MPI')
dest_out = os.path.join(forward_run_dir, dir, 'OUTPUT_FILES')
utils.sym_link_directory(databases_mpi, dest_mpi)
utils.sym_link_directory(output_files, dest_out)
utils.print_blu('Done.')
def setup_solver(params):
"""
It's dylan, you know the drill.
"""
# Setup local parameters.
forward_run_dir = params['forward_run_dir']
forward_stage_dir = params['forward_stage_dir']
event_list = params['event_list']
lasif_path = params['lasif_path']
iteration_name = params['iteration_name']
specfem_root = params['specfem_root']
compiler_suite = params['compiler_suite']
project_name = params['project_name']
# Set up the mesh directory.
_setup_dir_tree('mesh', forward_run_dir)
# Set up the optimization directory.
optimization_base = os.path.join(forward_run_dir, 'OPTIMIZATION')
utils.mkdir_p(optimization_base)
utils.mkdir_p(os.path.join(optimization_base, 'bin'))
utils.mkdir_p(os.path.join(optimization_base, 'PROCESSED_KERNELS'))
utils.mkdir_p(os.path.join(optimization_base, 'GRADIENT_INFO'))
utils.mkdir_p(os.path.join(optimization_base, 'LOGS'))
utils.mkdir_p(os.path.join(optimization_base, 'DATA'))
utils.mkdir_p(os.path.join(optimization_base, 'VTK_FILES'))
# Create the forward modelling directories. Also copy relevant parameter
# files from the LASIF project. _copy_input_files also copies the input
# files to the specfem_root directory if mesh == True.
utils.print_ylw("Creating forward modelling directories...")
mesh = True
for i, event in enumerate(event_list):
_setup_dir_tree(event, forward_run_dir)
_copy_input_files(event,
forward_run_dir,
lasif_path,
iteration_name,
specfem_root,
mesh=mesh)
mesh = False
# Copy the files in SUBMISSION to the specfem root directory.
par_file = os.path.join(lasif_path, 'SUBMISSION', project_name, 'Par_file')
dest = os.path.join(specfem_root, 'DATA')
utils.safe_copy(par_file, dest)
# Change to specfem root directory and compile.
utils.print_ylw("Compiling...")
os.chdir(specfem_root)
with open('compilation_log.txt', 'w') as output:
proc = subprocess.Popen(['./mk_daint.sh', compiler_suite, 'adjoint'],
stdout=output,
stderr=output)
proc.communicate()
proc.wait()
# Distribute binaries and Par_file to directories.
utils.print_ylw('Copying compiled binaries...')
bin_directory = os.path.join('./bin')
opt_bin_directory = os.path.join(optimization_base, 'bin')
opt_dat_directory = os.path.join(optimization_base, 'DATA')
utils.copy_directory(bin_directory, opt_bin_directory)
for event in os.listdir(forward_run_dir):
event_bin = os.path.join(forward_run_dir, event, 'bin')
event_dat = os.path.join(forward_run_dir, event, 'DATA')
compile_par = os.path.join(specfem_root, 'DATA', 'Par_file')
utils.safe_copy(compile_par, event_dat)
utils.copy_directory(bin_directory,
event_bin,
only=['xspecfem3D', 'xmeshfem3D'])
# Also copy to the optimization directory. Recompile with vectorized cray
# compiler.
utils.print_ylw("Recompiling for vectorized smoother CRAY smoother...")
with open('compilation_log_tomo.txt', 'w') as output:
proc = subprocess.Popen(['./mk_daint.sh', 'cray.tomo', 'adjoint'],
stdout=output,
stderr=output)
proc.communicate()
proc.wait()
utils.copy_directory(bin_directory, opt_bin_directory)
compile_par = os.path.join(specfem_root, 'DATA', 'Par_file')
utils.safe_copy(compile_par, opt_dat_directory)
# Copy jobarray script to base directory.
utils.print_ylw('Copying jobarray sbatch script...')
source = os.path.join(lasif_path, 'SUBMISSION', project_name,
'jobArray_solver_daint.sbatch')
utils.safe_copy(source, forward_stage_dir)
utils.mkdir_p(os.path.join(forward_stage_dir, 'logs'))
# Copy mesh submission script.
source = os.path.join(lasif_path, 'SUBMISSION', project_name,
'job_mesher_daint.sbatch')
dest = os.path.join(forward_run_dir, 'mesh')
utils.safe_copy(source, dest)
# Copy topography information to mesh directory.
utils.print_ylw('Copying topography information...')
master_topo_path = os.path.join(specfem_root, 'DATA', 'topo_bathy')
mesh_topo_path = os.path.join(forward_run_dir, 'mesh', 'DATA',
'topo_bathy')
utils.mkdir_p(mesh_topo_path)
utils.copy_directory(master_topo_path, mesh_topo_path)
utils.print_blu('Done.')