def prevalidate(self, config=None):
"""
Validates transfer inputs
"""
if self.status == JobStatus.VALID:
return 0
if not check_logged_in():
self.status = JobStatus.INVALID
return 1
for i in config:
if i in self.inputs:
if i == 'recursive':
if config.get(i) == 'True':
self.config[i] = True
else:
self.config[i] = False
else:
self.config[i] = config.get(i)
for i in self.inputs:
if i not in self.config or self.config[i] is None:
if i == 'recursive':
self.config[i] = False
else:
print_message('Missing transfer argument {}'.format(i))
self.status = JobStatus.INVALID
return -1
self.status = JobStatus.VALID
return 0
def test_send_mail_invalid(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
m = Mailer(src='xxyyzz', dst='xxyyzz')
ret = m.send(status='THIS IS A TEST', msg='THIS IS ONLY A TEST')
self.assertFalse(ret)
def test_send_mail_valid(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
m = Mailer(src='*****@*****.**', dst='*****@*****.**')
ret = m.send(status='THIS IS A TEST', msg='THIS IS ONLY A TEST')
self.assertTrue(ret)
def test_e3sm_diags_prevalidate(self):
"""
test that the e3sm_diags prevalidate and prerun setup works correctly
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
_args = ['-c', self.config_path, '-r', 'resources/']
config, filemanager, runmanager = initialize(
argv=_args,
version="2.0.0",
branch="testing",
event_list=self.event_list,
kill_event=Event(),
testing=True)
self.assertFalse(config is None)
self.assertFalse(filemanager is None)
self.assertFalse(runmanager is None)
config['global']['dryrun'] = True
runmanager.check_data_ready()
runmanager.start_ready_jobs()
for case in runmanager.cases:
for job in case['jobs']:
if job.job_type == 'e3sm_diags':
job.setup_data(
config=config,
filemanager=filemanager,
case='20180129.DECKv1b_piControl.ne30_oEC.edison')
job.execute(config=config, dryrun=True)
self.assertEquals(job.status, JobStatus.COMPLETED)
def test_init_from_scratch_config_globus_bad_uuid(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = [
'-c',
'tests/test_configs/valid_config_from_scratch_globus_bad_uuid.cfg'
]
project_path = '/p/user_pub/e3sm/baldwin32/testing/empty/'
if os.path.exists(project_path):
shutil.rmtree(project_path)
config, filemanager, runmanager = initialize(
argv=pargv,
version=__version__,
branch=__branch__,
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
self.assertEqual(config, False)
self.assertEqual(filemanager, False)
self.assertEqual(runmanager, False)
self.assertEqual(os.path.exists(project_path), True)
if os.path.exists(project_path):
shutil.rmtree(project_path)
def test_invalid_bad_data_type(self):
print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
config_path = 'tests/test_configs/invalid_config_bad_img_hosting.cfg'
config = ConfigObj(config_path)
messages = verify_config(config)
self.assertTrue('image hosting is turned on, but no img_host_server specified' in messages)
self.assertTrue('image hosting is turned on, but no host_directory specified' in messages)
def split(self):
"""
Perform the requested variable extraction
"""
msg = f'starting extraction with nproc = {self._nproc}'
logging.info(msg)
print_message(msg, status='ok')
pool_res = list()
self._pool = Pool(self._nproc)
for var in var_list:
outfile = os.path.join(
self._output_path,
f'{var}_{self._start:04d}01_{self._end:04d}12.nc')
pool_res.append(
self._pool.apply_async(
_split_one, [var, self._file_list, outfile]))
out = None
err = None
for idx, res in enumerate(pool_res):
out, err = res.get(9999999)
if err:
print out, err
else:
out += f', {idx + 1}/{len(pool_res)} jobs complete'
print_message(out, 'ok')
self._pool.close()
self._pool.join()
os.chdir(self._prev_dir)
def test_transfer_file(self):
"""
test moving a single file
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
source_file = {
'remote_path':
'/global/homes/s/sbaldwin/test_directory/test_file.txt',
'local_path': os.path.join(project_path, 'test_file.txt')
}
source_path = '/global/homes/s/sbaldwin/test_directory'
transfer = Transfer(
{
'file_list': [source_file],
'recursive': False,
'source_endpoint': self.source_endpoint,
'destination_endpoint': self.destination_endpoint,
'source_path': source_path,
'destination_path': project_path,
'source_email': '*****@*****.**',
'display_event': threading.Event(),
'ui': False
},
event_list=EventList())
transfer.execute(event=threading.Event())
self.assertTrue(transfer.postvalidate())
self.assertEqual(transfer.status.name, 'COMPLETED')
self.assertTrue(os.path.exists(source_file['local_path']))
def test_timeseries_setup(self):
"""
test a valid timeseries setup
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
timeseries = Timeseries(
{
'regrid_output_directory':
os.getcwd(),
'filemanager':
None,
'year_set':
1,
'annual_mode':
'sdd',
'start_year':
50,
'end_year':
55,
'output_directory':
os.path.join(self.project_path, 'output', 'monthly'),
'var_list':
self.config['ncclimo']['var_list'],
'caseId':
self.config['global']['experiment'],
'run_scripts_path':
os.path.join(self.project_path, 'output', 'run_scripts'),
'regrid_map_path':
self.config['ncclimo']['regrid_map_path'],
'file_list': [],
}, EventList())
self.assertEqual(timeseries.status.name, 'VALID')
def test_filemanager_update_local(self):
"""
run filemanager set and populate, then create a dummy file in the
input directory and run update_local which should mark it as present
"""
"""
############# SETUP ################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = False
types = ['atm', 'ice', 'ocn', 'rest', 'streams.cice', 'streams.ocean']
database = '{}.db'.format(inspect.stack()[0][3])
simstart = 51
simend = 60
remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
mutex = threading.Lock()
"""
############### TEST #################
"""
filemanager = FileManager(event_list=EventList(),
mutex=mutex,
sta=sta,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=experiment)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
filemanager.mutex.acquire()
df = DataFile.select().limit(1)
filemanager.mutex.release()
name = df[0].name
head, tail = os.path.split(df[0].local_path)
if not os.path.exists(head):
os.makedirs(head)
dummy_file_path = df[0].local_path
print '----- writing out dummy file at {} -----'.format(
dummy_file_path)
with open(dummy_file_path, 'w') as fp:
fp.write('this is a test file')
filemanager.update_local_status()
filemanager.mutex.acquire()
df = DataFile.select().where(DataFile.name == name)[0]
filemanager.mutex.release()
self.assertEqual(df.local_status, 0)
self.assertTrue(df.local_size > 0)
"""
############### CLEANUP #################
"""
os.remove(database)
def test_filemanager_setup_no_sta(self):
"""
run filemansger setup with no sta
"""
"""
############## SETUP ################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = False
database = '{}.db'.format(inspect.stack()[0][3])
remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
mutex = threading.Lock()
experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
"""
############### TEST ################
"""
filemanager = FileManager(mutex=mutex,
event_list=EventList(),
sta=sta,
types=self.file_types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=experiment)
self.assertTrue(isinstance(filemanager, FileManager))
self.assertTrue(os.path.exists(database))
"""
############## CLEANUP ###############
"""
os.remove(database)
def test_invalid_missing_project(self):
print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
config_path = 'tests/test_configs/invalid_config_no_project_path.cfg'
config = ConfigObj(config_path)
messages = verify_config(config)
self.assertTrue('no project_path in global options' in messages)
self.assertTrue('No data_types section found in config' in messages)
def test_filemanager_populate_no_sta(self):
"""
run filemanager set and populate with sta turned off
"""
"""
############### SETUP ################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = False
database = '{}.db'.format(inspect.stack()[0][3])
simstart = 1
simend = 10
experiment = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
mutex = threading.Lock()
"""
############## TEST ###############
"""
filemanager = FileManager(event_list=EventList(),
mutex=mutex,
sta=sta,
types=self.file_types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=self.experiment)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
filemanager.mutex.acquire()
simlength = simend - simstart + 1
for _type in ['atm', 'lnd', 'ocn', 'ice']:
file_names = [
x.name
for x in DataFile.select().where(DataFile.datatype == _type)
]
if not len(file_names) == (simlength * 12):
print _type + ' does not have ' + str(
simlength * 12) + ' files'
self.assertEqual(len(file_names), (simlength * 12))
for year in range(simstart, simend + 1):
for month in range(1, 13):
name = (file_type_map[_type].replace(
'EXPERIMENT',
experiment).replace('YEAR',
'{:04d}'.format(year)).replace(
'MONTH',
'{:02}'.format(month)))
self.assertTrue(name in file_names)
filemanager.mutex.release()
"""
############## CLEANUP ###############
"""
os.remove(database)
def test_amwg_execution_completed_job(self):
"""
test that when run on a completed set of jobs, amwg recognizes that the run has already
taken place and doesnt start again
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
_args = ['-c', self.config_path, '-r', 'resources/']
config, filemanager, runmanager = initialize(
argv=_args,
version="2.0.0",
branch="testing",
event_list=self.event_list,
kill_event=Event(),
testing=True)
config['global']['dryrun'] = True
runmanager.check_data_ready()
runmanager.start_ready_jobs()
for case in runmanager.cases:
for job in case['jobs']:
if job.job_type == 'amwg':
job.setup_data(
config=config,
filemanager=filemanager,
case='20180129.DECKv1b_piControl.ne30_oEC.edison')
job.execute(config=config, dryrun=True)
self.assertEquals(job.status, JobStatus.COMPLETED)
def test_render_bad_input_file(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
render_target = os.path.join(os.getcwd(), 'tests', 'DOES_NOT_EXIST')
render_output = os.path.join(os.getcwd(), 'tests', 'render_output.txt')
self.assertFalse(render({}, render_target, render_output))
def test_processflow_from_scratch_yes_sta(self):
"""
End to end test from scrath with no sta
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
config_path = os.path.join(os.getcwd(), 'tests', 'test_configs',
'test_processflow_with_inplace_data.cfg')
output_path = os.path.join('/export/baldwin32/jenkins/workspace/',
inspect.stack()[0][3], 'output')
input_path = os.path.join('/export/baldwin32/jenkins/workspace/',
inspect.stack()[0][3], 'input')
if os.path.exists(output_path):
print "removing previous output directory"
shutil.rmtree(output_path)
if os.path.exists(input_path):
print "removing previous input directory"
shutil.rmtree(input_path)
print "---- project cleanup complete ----"
testargs = [
'-c', config_path, '-n', '-f', '-r', './resources', '-o',
output_path, '-i', input_path, '-d'
]
ret = main(test=True, testargs=testargs)
self.assertEqual(ret, 0)
def test_invalid_missing_global(self):
print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
config_path = 'tests/test_configs/invalid_config_no_global.cfg'
config = ConfigObj(config_path)
messages = verify_config(config)
self.assertTrue('No global section found in config' in messages)
self.assertTrue('No simulations section found in config' in messages)
def verify_remote_files(self, client, case):
"""
Check that the user supplied file paths are valid for remote files
Parameters:
client: either an ssh_client or a globus_client
case: the case to check remote paths for
"""
if not self._config['global']['verify']:
return True
msg = 'verifying remote file paths'
print_line(msg, self._event_list)
data_types_to_verify = []
q = (DataFile
.select()
.where(
(DataFile.case == case) &
(DataFile.local_status != FileStatus.PRESENT.value)))
for datafile in q.execute():
if datafile.datatype not in data_types_to_verify:
data_types_to_verify.append(datafile.datatype)
found_all = True
for datatype in data_types_to_verify:
q = (DataFile
.select()
.where(
(DataFile.case == case) &
(DataFile.datatype == datatype)))
files = q.execute()
remote_path, _ = os.path.split(files[0].remote_path)
msg = 'Checking {} files in {}'.format(datatype, remote_path)
print_line(msg, self._event_list)
if files[0].transfer_type == 'globus':
from lib.globus_interface import get_ls as globus_ls
remote_contents = globus_ls(
client=client,
path=remote_path,
endpoint=self._config['simulations'][case]['remote_uuid'])
elif files[0].transfer_type == 'sftp':
from lib.ssh_interface import get_ls as ssh_ls
remote_contents = ssh_ls(
client=client,
remote_path=remote_path)
remote_names = [x['name'] for x in remote_contents]
for df in files:
if df.name not in remote_names:
msg = 'Unable to find file {name} at {remote_path}'.format(
name=df.name,
remote_path=remote_path)
print_message(msg, 'error')
found_all = False
if not found_all:
return False
else:
msg = 'found all remote files for {}'.format(case)
print_message(msg, 'ok')
return True
def test_AMWG_execute_completed(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
start_year = 1
end_year = 10
self.config['global'][
'project_path'] = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_complete'
self.config['global'][
'exeriment'] = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
set_string = '{start:04d}-{end:04d}'.format(start=start_year,
end=end_year)
web_directory = os.path.join(
self.config.get('global').get('host_directory'),
os.environ['USER'],
self.config.get('global').get('experiment'),
self.config.get('amwg').get('host_directory'), set_string)
host_url = '/'.join([
self.config.get('global').get('img_host_server'),
os.environ['USER'],
self.config.get('global').get('experiment'),
self.config.get('amwg').get('host_directory'), set_string
])
self.config['global']['output_path'] = os.path.join(
self.config['global']['project_path'], 'output')
self.config['global']['run_scripts_path'] = os.path.join(
self.config['global']['project_path'], 'output', 'run_scripts')
self.config['global']['resource_path'] = os.path.join(
os.getcwd(), 'resources')
regrid_path = os.path.join(self.config['global']['output_path'], 'pp',
'fv129x256', 'climo', '10yr')
output_path = os.path.join(self.config['global']['output_path'],
'amwg_diag', set_string)
template_path = os.path.join(self.config['global']['resource_path'],
'amwg_template.csh')
diag_home = self.config['amwg']['diag_home']
temp_path = os.path.join(output_path, 'tmp', 'amwg', set_string)
config = {
'web_dir': web_directory,
'host_url': host_url,
'experiment': self.config.get('global').get('experiment'),
'run_scripts_path': self.config['global']['run_scripts_path'],
'output_path': self.config['global']['output_path'],
'test_casename': self.config.get('global').get('experiment'),
'test_path_history': regrid_path + os.sep,
'regrided_climo_path': regrid_path + os.sep,
'test_path_climo': temp_path,
'test_path_diag': output_path,
'start_year': start_year,
'end_year': end_year,
'year_set': 1,
'run_directory': output_path,
'template_path': template_path,
'diag_home': diag_home
}
amwg = AMWGDiagnostic(config=config, event_list=EventList())
self.assertEqual(amwg.status.name, 'VALID')
amwg.execute(dryrun=True)
self.assertEqual(amwg.status.name, 'COMPLETED')
def test_shownode(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
slurm = Slurm()
node = os.environ['HOSTNAME'].lower().split('.')[0]
node_info = slurm.shownode(node)
self.assertTrue(node_info['Arch'] == 'x86_64')
def test_render_bad_outout_file(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
render_target = os.path.join(os.getcwd(), 'tests',
'test_render_target.txt')
render_output = '/usr/local/NO_PERMISSIONS'
self.assertFalse(render({}, render_target, render_output))
def test_invalid_bad_data_type(self):
print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
config_path = 'tests/test_configs/invalid_config_data_job_type.cfg'
config = ConfigObj(config_path)
messages = verify_config(config)
self.assertTrue('atm has no file_format' in messages)
self.assertTrue('atm has no remote_path' in messages)
self.assertTrue('atm has no local_path' in messages)
def test_init_print_version(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
argv = ['-v']
with self.assertRaises(SystemExit) as exitexception:
a, b, c = initialize(argv=argv, version=__version__)
self.assertEqual(exitexception.exception.code, 0)
def test_init_no_config(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
argv = []
a, b, c = initialize(argv=argv)
self.assertEqual(a, False)
self.assertEqual(b, False)
self.assertEqual(c, False)
def test_invalid_bad_regrid(self):
print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
config_path = 'tests/test_configs/invalid_config_bad_regrid.cfg'
config = ConfigObj(config_path)
messages = verify_config(config)
self.assertTrue('no source_grid_path given for lnd regrid' in messages)
self.assertTrue('no destination_grid_path given for lnd regrid' in messages)
self.assertTrue('no destination_grid_name given for lnd regrid' in messages)
self.assertTrue('regrid is set to run on data_type ocn, but this type is not set in simulation 20180129.DECKv1b_piControl.ne30_oEC.edison' in messages)
def test_shownode(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
slurm = Slurm()
node = 'acme1'
node_info = slurm.shownode(node)
self.assertTrue(node_info['Arch'] == 'x86_64')
self.assertTrue(node_info['CoresPerSocket'] == '24')
def test_runmanager_setup(self):
"""
Run the runmanager setup
"""
print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
db_path = os.path.join(
self.project_path,
'{}.db'.format(inspect.stack()[0][3]))
if os.path.exists(self.project_path):
shutil.rmtree(self.project_path)
os.makedirs(self.project_path)
filemanager = FileManager(
ui=False,
event_list=EventList(),
database=db_path,
types=['atm'],
sta=False,
mutex=self.mutex,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=self.project_path,
experiment=self.experiment)
runmanager = RunManager(
short_name='testname',
account='',
resource_path='./resources',
ui=False,
event_list=EventList(),
output_path=self.output_path,
caseID=self.experiment,
scripts_path=self.run_scripts_path,
thread_list=[],
event=threading.Event(),
no_host=True,
url_prefix='',
always_copy=False)
runmanager.setup_job_sets(
set_frequency=[5, 10],
sim_start_year=int(self.config['global']['simulation_start_year']),
sim_end_year=int(self.config['global']['simulation_end_year']),
config=self.config,
filemanager=filemanager)
self.assertEqual(len(runmanager.job_sets), 1)
for job_set in runmanager.job_sets:
if job_set.set_number == 1:
self.assertEqual(job_set.length, 5)
self.assertEqual(job_set.set_start_year, 1)
self.assertEqual(job_set.set_end_year, 5)
job_names = job_set.get_job_names()
self.assertTrue('ncclimo' in job_names)
self.assertTrue('timeseries' in job_names)
self.assertTrue('aprime_diags' in job_names)
self.assertTrue('e3sm_diags' in job_names)
def get_ls(client, path, endpoint):
for fail_count in xrange(10):
try:
res = globus_ls(client, path, endpoint, False, 0, False)
except Exception as e:
sleep(fail_count)
print_message("Globus server error, retrying")
print_debug(e)
else:
return res
def test_init_valid_config_simple(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = ['-c', 'tests/test_configs/valid_config_simple.cfg']
config, filemanager, runmanager = initialize(
argv=pargv,
version=__version__,
branch=__branch__,
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
def test_event_list(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
elist = EventList()
elist.push('beep boop', data=42)
self.assertEquals(len(elist.list), 1)
rmessage = 'boop beed'
elist.replace(index=0, message=rmessage)
self.assertEquals(elist.list[0].message, rmessage)
