def execute(self, dryrun=False):
"""
Perform the actual work
"""
# First check if the job has already been completed
if self.postvalidate():
self.status = JobStatus.COMPLETED
message = 'AMWG job already computed, skipping'
self.event_list.push(message=message)
logging.info(message)
return 0
# Create directory of regridded climos
regrid_path = os.path.join(
os.sep.join(self.config['test_path_diag'].split(os.sep)[:-2]),
'climo_regrid')
file_list = get_climo_output_files(input_path=regrid_path,
start_year=self.start_year,
end_year=self.end_year)
if not file_list or len(file_list) == 0:
print """
ERROR: AMWG: {start:04d}-{end:04d} could not find input climatologies at {path}\n
did you add ncclimo to this year_set?""".format(start=self.start_year,
end=self.end_year,
path=regrid_path)
self.status = JobStatus.FAILED
return 0
if not os.path.exists(self.config['test_path_climo']):
print 'creating temp directory for amwg'
os.makedirs(self.config['test_path_climo'])
create_symlink_dir(src_dir=regrid_path,
src_list=file_list,
dst=self.config['test_path_climo'])
# Rename the files to the format amwg expects
for item in os.listdir(self.config['test_path_climo']):
search = re.search(r'\_\d\d\d\d\d\d\_', item)
if not search:
continue
index = search.start()
os.rename(
os.path.join(self.config['test_path_climo'], item),
os.path.join(self.config['test_path_climo'],
item[:index] + '_climo.nc'))
# render the csh script into the output directory
self.output_path = self.config['output_path']
template_out = os.path.join(self.output_path, 'amwg.csh')
render(variables=self.config,
input_path=self.config.get('template_path'),
output_path=template_out)
expected_name = '{type}_{start:04d}-{end:04d}'.format(
start=self.config.get('start_year'),
end=self.config.get('end_year'),
type=self.type)
# Copy the rendered run script into the scripts directory
run_script_template_out = os.path.join(
self.config.get('run_scripts_path'), expected_name)
copyfile(src=template_out, dst=run_script_template_out)
# setup sbatch script
run_script = os.path.join(self.config.get('run_scripts_path'),
expected_name)
if os.path.exists(run_script):
os.remove(run_script)
self.slurm_args['output_file'] = '-o {output_file}'.format(
output_file=run_script + '.out')
cmd = '\ncsh {template}'.format(template=template_out)
slurm_args_str = [
'#SBATCH {value}'.format(value=v)
for k, v in self.slurm_args.items()
]
slurm_prefix = '\n'.join(slurm_args_str)
with open(run_script, 'w') as batchfile:
batchfile.write('#!/bin/bash\n')
batchfile.write(slurm_prefix)
batchfile.write(cmd)
if dryrun:
self.status = JobStatus.COMPLETED
return 0
slurm = Slurm()
print 'submitting to queue {type}: {start:04d}-{end:04d}'.format(
type=self.type, start=self.start_year, end=self.end_year)
self.job_id = slurm.batch(run_script, '--oversubscribe')
status = slurm.showjob(self.job_id)
self.status = StatusMap[status.get('JobState')]
message = '{type} id: {id} changed state to {state}'.format(
type=self.type, id=self.job_id, state=self.status)
logging.info(message)
self.event_list.push(message=message)
return self.job_id
def setup_data(self, config, filemanager, case):
"""
symlinks all data_types sepecified in the jobs _data_required field,
and puts a copy of the path for the links into the _input_file_paths field
"""
for datatype in self._data_required:
datainfo = config['data_types'].get(datatype)
if not datainfo:
print "ERROR: Unable to find config information for {}".format(
datatype)
sys.exit(1)
monthly = datainfo.get('monthly')
# first get the list of file paths to the data
if monthly == 'True' or monthly == True:
files = filemanager.get_file_paths_by_year(
datatype=datatype,
case=case,
start_year=self._start_year,
end_year=self._end_year)
else:
files = filemanager.get_file_paths_by_year(datatype=datatype,
case=case)
if not files or len(files) == 0:
msg = '{prefix}: filemanager cant find input files for datatype {datatype}'.format(
prefix=self.msg_prefix(), datatype=datatype)
logging.error(msg)
continue
# extract the file names
filesnames = list()
for file in files:
tail, head = os.path.split(file)
filesnames.append(head)
# setup the temp directory to hold symlinks
if self._run_type is not None:
temp_path = os.path.join(
config['global']['project_path'], 'output', 'temp',
self._short_name, '{}_{}'.format(self._job_type,
self._run_type),
'{:04d}_{:04d}'.format(self._start_year, self._end_year))
elif isinstance(self, Diag):
if self._comparison == 'obs':
comp = 'obs'
else:
comp = config['simulations'][self.comparison]['short_name']
temp_path = os.path.join(
config['global']['project_path'], 'output', 'temp',
self._short_name, self._job_type,
'{:04d}_{:04d}_vs_{}'.format(self._start_year,
self._end_year, comp))
else:
temp_path = os.path.join(
config['global']['project_path'], 'output', 'temp',
self._short_name, self._job_type,
'{:04d}_{:04d}'.format(self._start_year, self._end_year))
if not os.path.exists(temp_path):
os.makedirs(temp_path)
# keep a reference to the input data for later
self._input_file_paths.extend(
[os.path.join(temp_path, x) for x in filesnames])
# create the symlinks
create_symlink_dir(src_dir=tail,
src_list=filesnames,
dst=temp_path)
return
def execute(self, dryrun=False):
# Check if the output already exists
if self.postvalidate():
self.status = JobStatus.COMPLETED
message = 'ACME diags already computed, skipping'
self.event_list.push(message=message)
logging.info(message)
return 0
# render the parameters file
self.output_path = self.config['output_path']
template_out = os.path.join(self.output_path, 'params.py')
variables = {
'sets': self.config['sets'],
'backend': self.config['backend'],
'reference_data_path': self.config['reference_data_path'],
'test_data_path': self.config['regrided_climo_path'],
'test_name': self.config['test_name'],
'seasons': self.config['seasons'],
'results_dir': self.config['results_dir']
}
render(variables=variables,
input_path=self.config.get('template_path'),
output_path=template_out)
run_name = '{type}_{start:04d}_{end:04d}'.format(
start=self.config.get('start_year'),
end=self.config.get('end_year'),
type=self.type)
template_copy = os.path.join(self.config.get('run_scripts_path'),
run_name)
copyfile(src=template_out, dst=template_copy)
# Create directory of regridded climos
file_list = get_climo_output_files(
input_path=self.config['regrid_base_path'],
start_year=self.start_year,
end_year=self.end_year)
create_symlink_dir(src_dir=self.config['regrid_base_path'],
src_list=file_list,
dst=self.config['regrided_climo_path'])
# setup sbatch script
run_script = os.path.join(self.config.get('run_scripts_path'),
run_name)
if os.path.exists(run_script):
os.remove(run_script)
self.slurm_args['output_file'] = '-o {output_file}'.format(
output_file=run_script + '.out')
cmd = 'acme_diags_driver.py -p {template}'.format(
template=template_out)
slurm_args_str = [
'#SBATCH {value}\n'.format(value=v)
for k, v in self.slurm_args.items()
]
slurm_prefix = ''.join(slurm_args_str)
with open(run_script, 'w') as batchfile:
batchfile.write('#!/bin/bash\n')
batchfile.write(slurm_prefix)
batchfile.write(cmd)
slurm = Slurm()
print 'submitting to queue {type}: {start:04d}-{end:04d}'.format(
type=self.type, start=self.start_year, end=self.end_year)
self.job_id = slurm.batch(run_script, '--oversubscribe')
status = slurm.showjob(self.job_id)
self.status = StatusMap[status.get('JobState')]
message = '{type} id: {id} changed state to {state}'.format(
type=self.type, id=self.job_id, state=self.status)
logging.info(message)
self.event_list.push(message=message)
return self.job_id