def stage_data(self):
# Write script
remote_script = os.path.join(self.config.solver_dir,
"process_synthetics.py")
with io.open(utilities.get_script_file("process_synthetics"),
"r") as fh:
script_string = fh.readlines()
script_string.insert(0, "#!{}\n".format(self.config.python_exec))
self.remote_machine.write_file(remote_script, "".join(script_string))
# Copy over pickle file.
info = {
"lowpass": self.iteration_info["lowpass"],
"highpass": self.iteration_info["highpass"],
"event_list": self.event_info.keys()
}
tmp_pickle = "tmp_pickle.p"
remote_pickle = os.path.join(self.config.solver_dir, "info.p")
with io.open(tmp_pickle, "wb") as fh:
cPickle.dump(info, fh)
self.remote_machine.put_file(tmp_pickle, remote_pickle)
os.remove(tmp_pickle)
# Copy sbatch file.
remote_sbatch = os.path.join(self.config.solver_dir,
"process_synthetics.sbatch")
with io.open(utilities.get_template_file("sbatch"), "r") as fh:
sbatch_string = fh.read().format(**self.sbatch_dict)
self.remote_machine.write_file(remote_sbatch, sbatch_string)
def stage_data(self):
# Write script
remote_script = os.path.join(self.config.solver_dir, "process_synthetics.py")
with io.open(utilities.get_script_file("process_synthetics"), "r") as fh:
script_string = fh.readlines()
script_string.insert(0, "#!{}\n".format(self.config.python_exec))
self.remote_machine.write_file(remote_script, "".join(script_string))
# Copy over pickle file.
info = {"lowpass": self.iteration_info["lowpass"],
"highpass": self.iteration_info["highpass"],
"event_list": self.event_info.keys()}
tmp_pickle = "tmp_pickle.p"
remote_pickle = os.path.join(self.config.solver_dir, "info.p")
with io.open(tmp_pickle, "wb") as fh:
cPickle.dump(info, fh)
self.remote_machine.put_file(tmp_pickle, remote_pickle)
os.remove(tmp_pickle)
# Copy sbatch file.
remote_sbatch = os.path.join(self.config.solver_dir, "process_synthetics.sbatch")
with io.open(utilities.get_template_file("sbatch"), "r") as fh:
sbatch_string = fh.read().format(**self.sbatch_dict)
self.remote_machine.write_file(remote_sbatch, sbatch_string)
def stage_data(self):
self.remote_machine.makedir(self.config.preprocessing_dir)
click.secho('Copying stations...')
self.remote_machine.execute_command('rsync -a {} {}'.format(
os.path.join(self.config.lasif_project_path, 'STATIONS', 'StationXML'),
self.config.preprocessing_dir))
with click.progressbar(self.all_events, label="Copying data...") as events:
for event in events:
raw_dir = os.path.join(self.config.lasif_project_path, 'DATA', event, 'raw', 'data.mseed')
event_dir = os.path.join(self.config.preprocessing_dir, event)
self.remote_machine.makedir(event_dir)
self.remote_machine.execute_command('rsync {} {}'.format(raw_dir, event_dir))
self.remote_machine.put_file('lasif_data.p',
os.path.join(self.config.preprocessing_dir, 'lasif_data.p'))
remote_script = os.path.join(self.config.preprocessing_dir, "preprocess_data.py")
with io.open(utilities.get_script_file('preprocess_data'), 'r') as fh:
script_string = fh.readlines()
script_string.insert(0, '#!{}\n'.format(self.config.python_exec))
self.remote_machine.write_file(remote_script, ''.join(script_string))
remote_sbatch = os.path.join(self.config.preprocessing_dir, 'preprocess_data.sbatch')
with io.open(utilities.get_template_file('sbatch'), 'r') as fh:
sbatch_string = fh.read().format(**self.sbatch_dict)
self.remote_machine.write_file(remote_sbatch, sbatch_string)
def stage_data(self):
# Get adjoint_sources.p
raw_dir = os.path.join(self.config.lasif_project_path,
'ADJOINT_SOURCES_AND_WINDOWS/ADJOINT_SOURCES',
self.config.base_iteration, 'adjoint_sources.p')
event_dir = os.path.join(self.config.solver_dir)
self.remote_machine.execute_command('rsync {} {}'.format(
raw_dir, event_dir))
# Get lasif_data.p
raw_dir = os.path.join(self.config.lasif_project_path, 'lasif_data.p')
event_dir = os.path.join(self.config.solver_dir)
self.remote_machine.execute_command('rsync {} {}'.format(
raw_dir, event_dir))
remote_script = os.path.join(self.config.solver_dir,
"write_adjoint_sources.py")
with io.open(utilities.get_script_file('write_adjoint_sources'),
'r') as fh:
script_string = fh.readlines()
script_string.insert(0, '#!{}\n'.format(self.config.python_exec))
self.remote_machine.write_file(remote_script, ''.join(script_string))
remote_sbatch = os.path.join(self.config.solver_dir,
'write_adjoint_sources.sbatch')
with io.open(utilities.get_template_file('sbatch'), 'r') as fh:
sbatch_string = fh.read().format(**self.sbatch_dict)
self.remote_machine.write_file(remote_sbatch, sbatch_string)
def stage_data(self):
self.remote_machine.makedir(self.config.adjoint_dir)
hpass = 1 / self.iteration_info['highpass']
lpass = 1 / self.iteration_info['lowpass']
with open('./lasif_data.p', 'rb') as fh:
f = cPickle.load(fh)
if self.config.input_data_type == 'noise':
f.append({'input_data_type': 'noise'})
elif self.config.input_data_type == 'earthquake':
f.append({'input_data_type': 'earthquake'})
with open('./lasif_data.p', 'wb') as fh:
cPickle.dump(f,fh)
with click.progressbar(self.all_events, label="Copying preprocessed data...") as events:
for event in events:
try:
raw_dir = os.path.join(self.config.lasif_project_path, 'DATA', event, 'preprocessed_{:.1f}_{:.1f}'.format(lpass, hpass), 'preprocessed_data.mseed')
event_dir = os.path.join(self.config.adjoint_dir, event)
self.remote_machine.makedir(event_dir)
self.remote_machine.execute_command('rsync {} {}'.format(raw_dir, event_dir))
except:
print "\nCould not sync preprocessed_data.mseed for: " + event
with click.progressbar(self.all_events, label="Copying synthetics...") as events:
for event in events:
try:
raw_dir = os.path.join(self.config.lasif_project_path, 'SYNTHETICS', event, self.config.base_iteration, 'synthetics.mseed')
event_dir = os.path.join(self.config.adjoint_dir, event)
self.remote_machine.execute_command('rsync {} {}'.format(raw_dir, event_dir))
except:
print "\nCould not sync synthetics.mseed for: " + event
with click.progressbar(self.all_events, label="Copying windows...") as events:
for event in events:
try:
raw_dir = os.path.join(self.config.lasif_project_path, 'ADJOINT_SOURCES_AND_WINDOWS/WINDOWS', self.config.first_iteration ,event, 'windows.p')
event_dir = os.path.join(self.config.adjoint_dir, event)
self.remote_machine.execute_command('rsync {} {}'.format(raw_dir, event_dir))
except:
print "\nCould not sync window.p for: " + event
self.remote_machine.put_file('lasif_data.p',
os.path.join(self.config.adjoint_dir, 'lasif_data.p'))
remote_script = os.path.join(self.config.adjoint_dir, "create_adjoint_sources.py")
with io.open(utilities.get_script_file('create_adjoint_sources'), 'r') as fh:
script_string = fh.readlines()
script_string.insert(0, '#!{}\n'.format(self.config.python_exec))
self.remote_machine.write_file(remote_script, ''.join(script_string))
remote_sbatch = os.path.join(self.config.adjoint_dir, 'create_adjoint_sources.sbatch')
with io.open(utilities.get_template_file('sbatch'), 'r') as fh:
sbatch_string = fh.read().format(**self.sbatch_dict)
self.remote_machine.write_file(remote_sbatch, sbatch_string)
def stage_data(self):
with open('./lasif_data.p', 'rb') as fh:
f = cPickle.load(fh)
if self.config.input_data_type == 'noise':
f.append({'input_data_type': 'noise'})
elif self.config.input_data_type == 'earthquake':
f.append({'input_data_type': 'earthquake'})
with open('./lasif_data.p', 'wb') as fh:
cPickle.dump(f,fh)
hpass = 1 / self.iteration_info['highpass']
lpass = 1 / self.iteration_info['lowpass']
pre_dir_string = 'preprocessed_{:.1f}_{:.1f}'.format(lpass, hpass)
# Copy remote data.
all_events = sorted(self.event_info.keys())
self.remote_machine.makedir(self.config.window_dir)
syn_base = os.path.join(self.config.lasif_project_path, "SYNTHETICS")
dat_base = os.path.join(self.config.lasif_project_path, 'DATA')
with click.progressbar(all_events, label="Copying data...") as events:
for event in events:
event_dir = os.path.join(self.config.window_dir, event)
self.remote_machine.makedir(event_dir)
syn_dat = os.path.join(syn_base, event,
self.config.base_iteration, "synthetics.mseed")
self.remote_machine.execute_command(
"rsync {} {}".format(syn_dat, event_dir))
pro_dat = os.path.join(dat_base, event, pre_dir_string, 'preprocessed_data.mseed')
self.remote_machine.execute_command(
'rsync {} {}'.format(pro_dat, event_dir))
# Put Stations
click.secho('Copying stations...')
self.remote_machine.execute_command('rsync -a {} {}'.format(
os.path.join(self.config.lasif_project_path, 'STATIONS', 'StationXML'),
self.config.window_dir))
# Put data
file = "lasif_data.p"
self.remote_machine.put_file(file, os.path.join(self.config.window_dir, file))
# Put python script.
remote_script = os.path.join(self.config.window_dir, 'select_windows.py')
with io.open(utilities.get_script_file('select_windows'), 'r') as fh:
script_string = fh.readlines()
script_string.insert(0, '#!{}\n'.format(self.config.python_exec))
self.remote_machine.write_file(remote_script, ''.join(script_string))
# Sbatch
self.sbatch_dict["python_exec"] = os.path.dirname(self.config.python_exec)
remote_sbatch = os.path.join(self.config.window_dir, 'select_windows.sbatch')
with io.open(utilities.get_template_file('sbatch_python_parallel'), 'r') as fh:
sbatch_string = fh.read().format(**self.sbatch_dict)
self.remote_machine.write_file(remote_sbatch, sbatch_string)
def stage_data(self):
hpass = 1 / self.iteration_info['highpass']
lpass = 1 / self.iteration_info['lowpass']
pre_dir_string = 'preprocessed_{:.1f}_{:.1f}'.format(lpass, hpass)
# Copy remote data.
all_events = sorted(self.event_info.keys())
self.remote_machine.makedir(self.config.window_dir)
syn_base = os.path.join(self.config.lasif_project_path, "SYNTHETICS")
dat_base = os.path.join(self.config.lasif_project_path, 'DATA')
with click.progressbar(all_events, label="Copying data...") as events:
for event in events:
event_dir = os.path.join(self.config.window_dir, event)
self.remote_machine.makedir(event_dir)
syn_dat = os.path.join(syn_base, event,
self.config.base_iteration,
"synthetics.mseed")
self.remote_machine.execute_command("rsync {} {}".format(
syn_dat, event_dir))
pro_dat = os.path.join(dat_base, event, pre_dir_string,
'preprocessed_data.mseed')
self.remote_machine.execute_command('rsync {} {}'.format(
pro_dat, event_dir))
# Put Stations
click.secho('Copying stations...')
self.remote_machine.execute_command('rsync -a {} {}'.format(
os.path.join(self.config.lasif_project_path, 'STATIONS',
'StationXML'), self.config.window_dir))
# Put data
file = "lasif_data.p"
self.remote_machine.put_file(
file, os.path.join(self.config.window_dir, file))
# Put python script.
remote_script = os.path.join(self.config.window_dir,
'select_windows.py')
with io.open(utilities.get_script_file('select_windows'), 'r') as fh:
script_string = fh.readlines()
script_string.insert(0, '#!{}\n'.format(self.config.python_exec))
self.remote_machine.write_file(remote_script, ''.join(script_string))
# Sbatch
self.sbatch_dict["python_exec"] = os.path.dirname(
self.config.python_exec)
remote_sbatch = os.path.join(self.config.window_dir,
'select_windows.sbatch')
with io.open(utilities.get_template_file('sbatch_python_parallel'),
'r') as fh:
sbatch_string = fh.read().format(**self.sbatch_dict)
self.remote_machine.write_file(remote_sbatch, sbatch_string)
def stage_data(self):
# Get data summary
remote_script = os.path.join(self.config.lasif_project_path,
'pickle_lasif_data.py')
self.remote_machine.put_file(
utilities.get_script_file("pickle_lasif_data"), remote_script)
# Run script
if self.regenerate_data_cache:
click.secho("Generating data cache...")
command = '{} {} {}'.format(self.remote_machine.python_exec,
remote_script,
self.config.base_iteration)
print command
self.remote_machine.execute_command(
command, self.config.lasif_project_path)
# Read data cache.
data_cache = os.path.join(self.config.lasif_project_path,
"data_pickle.p")
self.remote_machine.get_file(data_cache, "./data_pickle.p")
def stage_data(self):
# Get data summary
remote_script = os.path.join(self.config.lasif_project_path,
'pickle_lasif_data.py')
self.remote_machine.put_file(
utilities.get_script_file("pickle_lasif_data"), remote_script)
# Run script
if self.regenerate_data_cache:
click.secho("Generating data cache...")
command = '{} {} {}'.format(self.remote_machine.python_exec,
remote_script,
self.config.base_iteration)
print command
self.remote_machine.execute_command(
command, self.config.lasif_project_path)
# Read data cache.
data_cache = os.path.join(self.config.lasif_project_path,
"data_pickle.p")
self.remote_machine.get_file(data_cache, "./data_pickle.p")
def stage_data(self):
# Get adjoint_sources.p
raw_dir = os.path.join(self.config.lasif_project_path, 'ADJOINT_SOURCES_AND_WINDOWS/ADJOINT_SOURCES',
self.config.base_iteration, 'adjoint_sources.p')
event_dir = os.path.join(self.config.solver_dir)
self.remote_machine.execute_command('rsync {} {}'.format(raw_dir, event_dir))
# Get lasif_data.p
raw_dir = os.path.join(self.config.lasif_project_path, 'lasif_data.p')
event_dir = os.path.join(self.config.solver_dir)
self.remote_machine.execute_command('rsync {} {}'.format(raw_dir, event_dir))
remote_script = os.path.join(self.config.solver_dir, "write_adjoint_sources.py")
with io.open(utilities.get_script_file('write_adjoint_sources'), 'r') as fh:
script_string = fh.readlines()
script_string.insert(0, '#!{}\n'.format(self.config.python_exec))
self.remote_machine.write_file(remote_script, ''.join(script_string))
remote_sbatch = os.path.join(self.config.solver_dir, 'write_adjoint_sources.sbatch')
with io.open(utilities.get_template_file('sbatch'), 'r') as fh:
sbatch_string = fh.read().format(**self.sbatch_dict)
self.remote_machine.write_file(remote_sbatch, sbatch_string)
def stage_data(self):
self.remote_machine.makedir(self.config.adjoint_dir)
hpass = 1 / self.iteration_info['highpass']
lpass = 1 / self.iteration_info['lowpass']
with open('./lasif_data.p', 'rb') as fh:
f = cPickle.load(fh)
if self.config.input_data_type == 'noise':
f.append({'input_data_type': 'noise'})
elif self.config.input_data_type == 'earthquake':
f.append({'input_data_type': 'earthquake'})
with open('./lasif_data.p', 'wb') as fh:
cPickle.dump(f,fh)\
with click.progressbar(self.all_events,
label="Copying preprocessed data...") as events:
for event in events:
try:
raw_dir = os.path.join(
self.config.lasif_project_path, 'DATA', event,
'preprocessed_{:.1f}_{:.1f}'.format(lpass, hpass),
'preprocessed_data.mseed')
event_dir = os.path.join(self.config.adjoint_dir, event)
self.remote_machine.makedir(event_dir)
self.remote_machine.execute_command('rsync {} {}'.format(
raw_dir, event_dir))
except:
print "\nCould not sync preprocessed_data.mseed for: " + event
with click.progressbar(self.all_events,
label="Copying synthetics...") as events:
for event in events:
try:
raw_dir = os.path.join(self.config.lasif_project_path,
'SYNTHETICS', event,
self.config.base_iteration,
'synthetics.mseed')
event_dir = os.path.join(self.config.adjoint_dir, event)
self.remote_machine.execute_command('rsync {} {}'.format(
raw_dir, event_dir))
except:
print "\nCould not sync synthetics.mseed for: " + event
with click.progressbar(self.all_events,
label="Copying windows...") as events:
for event in events:
try:
raw_dir = os.path.join(
self.config.lasif_project_path,
'ADJOINT_SOURCES_AND_WINDOWS/WINDOWS',
self.config.first_iteration, event, 'windows.p')
event_dir = os.path.join(self.config.adjoint_dir, event)
self.remote_machine.execute_command('rsync {} {}'.format(
raw_dir, event_dir))
except:
print "\nCould not sync window.p for: " + event
self.remote_machine.put_file(
'lasif_data.p',
os.path.join(self.config.adjoint_dir, 'lasif_data.p'))
remote_script = os.path.join(self.config.adjoint_dir,
"create_adjoint_sources.py")
with io.open(utilities.get_script_file('create_adjoint_sources'),
'r') as fh:
script_string = fh.readlines()
script_string.insert(0, '#!{}\n'.format(self.config.python_exec))
self.remote_machine.write_file(remote_script, ''.join(script_string))
remote_sbatch = os.path.join(self.config.adjoint_dir,
'create_adjoint_sources.sbatch')
with io.open(utilities.get_template_file('sbatch'), 'r') as fh:
sbatch_string = fh.read().format(**self.sbatch_dict)
self.remote_machine.write_file(remote_sbatch, sbatch_string)