def test_runmanager_setup(self):
"""
Run the runmanager setup
"""
print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
db_path = os.path.join(
self.project_path,
'{}.db'.format(inspect.stack()[0][3]))
if os.path.exists(self.project_path):
shutil.rmtree(self.project_path)
os.makedirs(self.project_path)
filemanager = FileManager(
ui=False,
event_list=EventList(),
database=db_path,
types=['atm'],
sta=False,
mutex=self.mutex,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=self.project_path,
experiment=self.experiment)
runmanager = RunManager(
short_name='testname',
account='',
resource_path='./resources',
ui=False,
event_list=EventList(),
output_path=self.output_path,
caseID=self.experiment,
scripts_path=self.run_scripts_path,
thread_list=[],
event=threading.Event(),
no_host=True,
url_prefix='',
always_copy=False)
runmanager.setup_job_sets(
set_frequency=[5, 10],
sim_start_year=int(self.config['global']['simulation_start_year']),
sim_end_year=int(self.config['global']['simulation_end_year']),
config=self.config,
filemanager=filemanager)
self.assertEqual(len(runmanager.job_sets), 1)
for job_set in runmanager.job_sets:
if job_set.set_number == 1:
self.assertEqual(job_set.length, 5)
self.assertEqual(job_set.set_start_year, 1)
self.assertEqual(job_set.set_end_year, 5)
job_names = job_set.get_job_names()
self.assertTrue('ncclimo' in job_names)
self.assertTrue('timeseries' in job_names)
self.assertTrue('aprime_diags' in job_names)
self.assertTrue('e3sm_diags' in job_names)
def test_runmanager_write(self):
print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
db_path = os.path.join(
self.project_path,
'{}.db'.format(inspect.stack()[0][3]))
if os.path.exists(db_path):
os.remove(db_path)
local_path = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_complete'
database = '{}.db'.format(inspect.stack()[0][3])
local_path = os.path.join(
self.project_path,
'input')
types = ['atm']
mutex = threading.Lock()
filemanager = FileManager(
mutex=mutex,
event_list=EventList(),
sta=False,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=local_path,
experiment=self.experiment)
self.assertTrue(isinstance(filemanager, FileManager))
runmanager = RunManager(
short_name='testname',
account='',
resource_path='./resources',
ui=False,
event_list=EventList(),
output_path=self.output_path,
caseID=self.experiment,
scripts_path=self.run_scripts_path,
thread_list=[],
event=threading.Event(),
no_host=True,
url_prefix='',
always_copy=False)
runmanager.setup_job_sets(
set_frequency=[5, 10],
sim_start_year=int(self.config['global']['simulation_start_year']),
sim_end_year=int(self.config['global']['simulation_end_year']),
config=self.config,
filemanager=filemanager)
path = os.path.join(self.project_path, 'output', 'job_state.txt')
runmanager.write_job_sets(path)
self.assertTrue(os.path.exists(path))
os.remove(database)
def test_cmor_valid_completed(self):
"""
tests that a valid config on a completed case will mark itself as
already being run and not start
"""
print_message(
'\n---- Starting Test: {} ----'.format(inspect.stack()[0][3]),
'ok')
config = ConfigObj(self.valid_config_path)
config['post-processing']['cmor']['variable_list'] = [
config['post-processing']['cmor']['variable_list']
]
case_name = '20180129.DECKv1b_piControl.ne30_oEC.edison'
case = config['simulations'][case_name]
messages = verify_config(config)
self.assertEqual(len(messages), 0)
config['global']['resource_path'] = 'resources/'
filemanager = FileManager(config=config, event_list=EventList())
filemanager.populate_file_list()
filemanager.update_local_status()
timeseries = Timeseries(short_name=case['short_name'],
case=case_name,
start=config['simulations']['start_year'],
end=config['simulations']['end_year'],
config=config,
run_type='atm')
timeseries.check_data_ready(filemanager=filemanager)
timeseries.setup_data(config=config,
filemanager=filemanager,
case=case_name)
timeseries.execute(config=config, event_list=EventList())
timeseries.handle_completion(filemanager=filemanager,
config=config,
event_list=EventList())
cmor = Cmor(short_name=case['short_name'],
case=case_name,
start=config['simulations']['start_year'],
end=config['simulations']['end_year'],
config=config)
cmor.check_data_ready(filemanager=filemanager)
cmor.setup_data(config=config, filemanager=filemanager, case=case_name)
self.assertTrue(cmor.postvalidate(config=config))
self.assertTrue(cmor.execute(config=config, event_list=EventList()))
self.assertEquals(cmor.status, JobStatus.COMPLETED)
self.assertTrue(
cmor.handle_completion(filemanager=filemanager,
event_list=EventList(),
config=config))
def test_init_from_scratch_config_globus_bad_uuid(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = [
'-c',
'tests/test_configs/valid_config_from_scratch_globus_bad_uuid.cfg'
]
project_path = '/p/user_pub/e3sm/baldwin32/testing/empty/'
if os.path.exists(project_path):
shutil.rmtree(project_path)
config, filemanager, runmanager = initialize(
argv=pargv,
version=__version__,
branch=__branch__,
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
self.assertEqual(config, False)
self.assertEqual(filemanager, False)
self.assertEqual(runmanager, False)
self.assertEqual(os.path.exists(project_path), True)
if os.path.exists(project_path):
shutil.rmtree(project_path)
def test_transfer_file(self):
"""
test moving a single file
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
source_file = {
'remote_path':
'/global/homes/s/sbaldwin/test_directory/test_file.txt',
'local_path': os.path.join(project_path, 'test_file.txt')
}
source_path = '/global/homes/s/sbaldwin/test_directory'
transfer = Transfer(
{
'file_list': [source_file],
'recursive': False,
'source_endpoint': self.source_endpoint,
'destination_endpoint': self.destination_endpoint,
'source_path': source_path,
'destination_path': project_path,
'source_email': '*****@*****.**',
'display_event': threading.Event(),
'ui': False
},
event_list=EventList())
transfer.execute(event=threading.Event())
self.assertTrue(transfer.postvalidate())
self.assertEqual(transfer.status.name, 'COMPLETED')
self.assertTrue(os.path.exists(source_file['local_path']))
def test_filemanager_populate_no_sta(self):
"""
run filemanager set and populate with sta turned off
"""
"""
############### SETUP ################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = False
database = '{}.db'.format(inspect.stack()[0][3])
simstart = 1
simend = 10
experiment = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
mutex = threading.Lock()
"""
############## TEST ###############
"""
filemanager = FileManager(event_list=EventList(),
mutex=mutex,
sta=sta,
types=self.file_types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=self.experiment)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
filemanager.mutex.acquire()
simlength = simend - simstart + 1
for _type in ['atm', 'lnd', 'ocn', 'ice']:
file_names = [
x.name
for x in DataFile.select().where(DataFile.datatype == _type)
]
if not len(file_names) == (simlength * 12):
print _type + ' does not have ' + str(
simlength * 12) + ' files'
self.assertEqual(len(file_names), (simlength * 12))
for year in range(simstart, simend + 1):
for month in range(1, 13):
name = (file_type_map[_type].replace(
'EXPERIMENT',
experiment).replace('YEAR',
'{:04d}'.format(year)).replace(
'MONTH',
'{:02}'.format(month)))
self.assertTrue(name in file_names)
filemanager.mutex.release()
"""
############## CLEANUP ###############
"""
os.remove(database)
def test_finilize_complete(self):
pargv = ['-c', 'tests/test_configs/valid_config_simple.cfg']
event_list = EventList()
kill_event = threading.Event()
config, filemanager, runmanager = initialize(argv=pargv,
version="0.0.0",
branch="000",
event_list=event_list,
kill_event=kill_event,
mutex=threading.Lock(),
testing=True)
self.assertNotEqual(config, False)
self.assertNotEqual(filemanager, False)
self.assertNotEqual(runmanager, False)
# all jobs should be complete already
# this will mark them as such
runmanager.check_data_ready()
runmanager.start_ready_jobs()
runmanager.monitor_running_jobs()
finalize(config=config,
event_list=event_list,
kill_event=kill_event,
status=runmanager.is_all_done(),
runmanager=runmanager)
self.assertTrue(runmanager.is_all_done())
def test_filemanager_update_local(self):
"""
run filemanager set and populate, then create a dummy file in the
input directory and run update_local which should mark it as present
"""
"""
############# SETUP ################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = False
types = ['atm', 'ice', 'ocn', 'rest', 'streams.cice', 'streams.ocean']
database = '{}.db'.format(inspect.stack()[0][3])
simstart = 51
simend = 60
remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
mutex = threading.Lock()
"""
############### TEST #################
"""
filemanager = FileManager(event_list=EventList(),
mutex=mutex,
sta=sta,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=experiment)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
filemanager.mutex.acquire()
df = DataFile.select().limit(1)
filemanager.mutex.release()
name = df[0].name
head, tail = os.path.split(df[0].local_path)
if not os.path.exists(head):
os.makedirs(head)
dummy_file_path = df[0].local_path
print '----- writing out dummy file at {} -----'.format(
dummy_file_path)
with open(dummy_file_path, 'w') as fp:
fp.write('this is a test file')
filemanager.update_local_status()
filemanager.mutex.acquire()
df = DataFile.select().where(DataFile.name == name)[0]
filemanager.mutex.release()
self.assertEqual(df.local_status, 0)
self.assertTrue(df.local_size > 0)
"""
############### CLEANUP #################
"""
os.remove(database)
def test_filemanager_setup_no_sta(self):
"""
run filemansger setup with no sta
"""
"""
############## SETUP ################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = False
database = '{}.db'.format(inspect.stack()[0][3])
remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
mutex = threading.Lock()
experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
"""
############### TEST ################
"""
filemanager = FileManager(mutex=mutex,
event_list=EventList(),
sta=sta,
types=self.file_types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=experiment)
self.assertTrue(isinstance(filemanager, FileManager))
self.assertTrue(os.path.exists(database))
"""
############## CLEANUP ###############
"""
os.remove(database)
def test_timeseries_setup(self):
"""
test a valid timeseries setup
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
timeseries = Timeseries(
{
'regrid_output_directory':
os.getcwd(),
'filemanager':
None,
'year_set':
1,
'annual_mode':
'sdd',
'start_year':
50,
'end_year':
55,
'output_directory':
os.path.join(self.project_path, 'output', 'monthly'),
'var_list':
self.config['ncclimo']['var_list'],
'caseId':
self.config['global']['experiment'],
'run_scripts_path':
os.path.join(self.project_path, 'output', 'run_scripts'),
'regrid_map_path':
self.config['ncclimo']['regrid_map_path'],
'file_list': [],
}, EventList())
self.assertEqual(timeseries.status.name, 'VALID')
def test_AMWG_execute_completed(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
start_year = 1
end_year = 10
self.config['global'][
'project_path'] = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_complete'
self.config['global'][
'exeriment'] = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
set_string = '{start:04d}-{end:04d}'.format(start=start_year,
end=end_year)
web_directory = os.path.join(
self.config.get('global').get('host_directory'),
os.environ['USER'],
self.config.get('global').get('experiment'),
self.config.get('amwg').get('host_directory'), set_string)
host_url = '/'.join([
self.config.get('global').get('img_host_server'),
os.environ['USER'],
self.config.get('global').get('experiment'),
self.config.get('amwg').get('host_directory'), set_string
])
self.config['global']['output_path'] = os.path.join(
self.config['global']['project_path'], 'output')
self.config['global']['run_scripts_path'] = os.path.join(
self.config['global']['project_path'], 'output', 'run_scripts')
self.config['global']['resource_path'] = os.path.join(
os.getcwd(), 'resources')
regrid_path = os.path.join(self.config['global']['output_path'], 'pp',
'fv129x256', 'climo', '10yr')
output_path = os.path.join(self.config['global']['output_path'],
'amwg_diag', set_string)
template_path = os.path.join(self.config['global']['resource_path'],
'amwg_template.csh')
diag_home = self.config['amwg']['diag_home']
temp_path = os.path.join(output_path, 'tmp', 'amwg', set_string)
config = {
'web_dir': web_directory,
'host_url': host_url,
'experiment': self.config.get('global').get('experiment'),
'run_scripts_path': self.config['global']['run_scripts_path'],
'output_path': self.config['global']['output_path'],
'test_casename': self.config.get('global').get('experiment'),
'test_path_history': regrid_path + os.sep,
'regrided_climo_path': regrid_path + os.sep,
'test_path_climo': temp_path,
'test_path_diag': output_path,
'start_year': start_year,
'end_year': end_year,
'year_set': 1,
'run_directory': output_path,
'template_path': template_path,
'diag_home': diag_home
}
amwg = AMWGDiagnostic(config=config, event_list=EventList())
self.assertEqual(amwg.status.name, 'VALID')
amwg.execute(dryrun=True)
self.assertEqual(amwg.status.name, 'COMPLETED')
def __init__(self, *args, **kwargs):
super(TestAMWGDiagnostic, self).__init__(*args, **kwargs)
self.config_path = os.path.join(os.getcwd(), 'tests', 'test_configs',
'test_amwg_complete.cfg')
self.config = ConfigObj(self.config_path)
self.config['global']['run_scripts_path'] = os.path.join(
self.config['global']['project_path'], 'output', 'scripts')
self.event_list = EventList()
def test_init_valid_config_simple(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = ['-c', 'tests/test_configs/valid_config_simple.cfg']
config, filemanager, runmanager = initialize(
argv=pargv,
version=__version__,
branch=__branch__,
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
def test_filemanager_update_remote_no_sta(self):
"""
run filemanager setup and populate, then run update_remote_status
with 10 years of atm output, and finally run all_data_remote to show that
all the remote data has been recognized
"""
"""
############# SETUP ##################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
sta = False
types = ['atm']
database = '{}.db'.format(inspect.stack()[0][3])
simstart = 51
simend = 60
experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
mutex = threading.Lock()
"""
################ TEST ##################
"""
filemanager = FileManager(event_list=EventList(),
mutex=mutex,
sta=False,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=experiment)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
client = get_client()
filemanager.update_remote_status(client)
filemanager.mutex.acquire()
for datafile in DataFile.select():
if datafile.remote_status != 0:
print datafile.name, datafile.remote_path, datafile.remote_status, datafile.datatype
self.assertEqual(datafile.remote_status, 0)
if filemanager.mutex.locked():
filemanager.mutex.release()
self.assertTrue(filemanager.all_data_remote())
"""
############## CLEANUP ###############
"""
os.remove(database)
def test_filemanager_update_remote_yes_sta(self):
"""
run filemanager setup and populate, then run update_remote_status on a directory
that has been short term archived
"""
"""
############### SETUP #################
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = True
types = [
'atm', 'ice', 'ocn', 'rest', 'streams.ocean', 'streams.cice',
'mpas-o_in', 'mpas-cice_in', 'meridionalHeatTransport'
]
database = '{}.db'.format(inspect.stack()[0][3])
simstart = 51
source_path = '/global/cscratch1/sd/golaz/ACME_simulations/20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
simend = 60
experiment = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
mutex = threading.Lock()
"""
############### TEST #################
"""
filemanager = FileManager(event_list=EventList(),
mutex=mutex,
sta=sta,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=source_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=self.experiment)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
client = get_client()
filemanager.update_remote_status(client)
filemanager.mutex.acquire()
for datafile in DataFile.select():
if datafile.remote_status != 0:
print datafile.name, datafile.remote_path
self.assertEqual(datafile.remote_status, 0)
if filemanager.mutex.locked():
filemanager.mutex.release()
self.assertTrue(filemanager.all_data_remote())
"""
############## CLEANUP ###############
"""
os.remove(database)
def test_ncclimo_execute_not_completed(self):
"""
Test that ncclimo will do all proper setup in an incomplete run
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
start_year = 51
end_year = 55
self.config['global'][
'project_path'] = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_not_complete'
self.config['global'][
'exeriment'] = '20171011.beta2_FCT2-icedeep_branch.A_WCYCL1850S.ne30_oECv3_ICG.edison'
year_set_string = '{start:04d}-{end:04d}'.format(start=start_year,
end=end_year)
config = {
'account':
'',
'year_set':
1,
'start_year':
start_year,
'end_year':
end_year,
'caseId':
self.config['global']['experiment'],
'annual_mode':
'sdd',
'input_directory':
os.path.join(self.config['global']['project_path'], 'input',
'atm'),
'climo_output_directory':
os.path.join(self.config['global']['project_path'], 'output', 'pp',
'ne30', 'climo', '5yr'),
'regrid_output_directory':
os.path.join(self.config['global']['project_path'], 'output', 'pp',
'fv129x256', 'climo', '5yr'),
'regrid_map_path':
self.config['ncclimo']['regrid_map_path'],
'year_set':
1,
'run_scripts_path':
os.path.join(self.config['global']['project_path'], 'output',
'run_scripts')
}
ncclimo = Ncclimo(config=config, event_list=EventList())
self.assertEqual(ncclimo.status, JobStatus.VALID)
self.assertFalse(ncclimo.execute(dryrun=True))
self.assertEqual(ncclimo.status.name, 'COMPLETED')
def test_init_missing_lnd(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = ['-c', 'tests/test_configs/invalid_config_missing_lnd.cfg']
config, filemanager, runmanager = initialize(
argv=pargv,
version=__version__,
branch=__branch__,
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
self.assertEqual(config, False)
self.assertEqual(filemanager, False)
self.assertEqual(runmanager, False)
def test_ncclimo_execute_bad_year(self):
"""
test that if given the wrong input year ncclimo will exit correctly
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
start_year = 55
end_year = 60
self.config['global'][
'project_path'] = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_complete'
self.config['global'][
'exeriment'] = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
year_set_string = '{start:04d}-{end:04d}'.format(start=start_year,
end=end_year)
config = {
'account':
'',
'year_set':
1,
'start_year':
start_year,
'end_year':
end_year,
'caseId':
self.config['global']['experiment'],
'annual_mode':
'sdd',
'input_directory':
os.path.join(self.config['global']['project_path'], 'input',
'atm'),
'climo_output_directory':
os.path.join(self.config['global']['project_path'], 'output',
'climo', '5yr'),
'regrid_output_directory':
os.path.join(self.config['global']['project_path'], 'output',
'climo_regrid'),
'regrid_map_path':
self.config['ncclimo']['regrid_map_path'],
'year_set':
1,
'run_scripts_path':
os.path.join(self.config['global']['project_path'], 'output',
'run_scripts')
}
ncclimo = Ncclimo(config=config, event_list=EventList())
self.assertEqual(ncclimo.status, JobStatus.VALID)
self.assertFalse(ncclimo.postvalidate())
def test_ncclimo_execute_bad_climo_dir(self):
"""
test that ncclimo will correctly exit if given a non-existant climo dir
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
start_year = 56
end_year = 60
self.config['global'][
'project_path'] = '/p/cscratch/acme/baldwin32/20171016/'
self.config['global'][
'exeriment'] = '20171011.beta2_FCT2-icedeep_branch.A_WCYCL1850S.ne30_oECv3_ICG.edison'
year_set_string = '{start:04d}-{end:04d}'.format(start=start_year,
end=end_year)
config = {
'account':
'',
'year_set':
1,
'start_year':
start_year,
'end_year':
end_year,
'caseId':
self.config['global']['experiment'],
'annual_mode':
'sdd',
'input_directory':
os.path.join(self.config['global']['project_path'], 'input',
'atm'),
'climo_output_directory':
os.path.join(self.config['global']['project_path'], 'output',
'climo', 'XXYYZZ'),
'regrid_output_directory':
os.path.join(self.config['global']['project_path'], 'output',
'climo_regrid'),
'regrid_map_path':
self.config['ncclimo']['regrid_map_path'],
'year_set':
1,
'run_scripts_path':
os.path.join(self.config['global']['project_path'], 'output',
'run_scripts')
}
ncclimo = Ncclimo(config=config, event_list=EventList())
self.assertEqual(ncclimo.status, JobStatus.VALID)
self.assertFalse(ncclimo.postvalidate())
def test_ncclimo_execute_completed(self):
"""
test that if ncclimo is told to run on a project thats already completed ncclimo
for the given yearset it will varify that the output is present and not run again
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
start_year = 1
end_year = 10
# REAL DATA
project_path = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_complete'
year_set_string = '{start:04d}-{end:04d}'.format(start=start_year,
end=end_year)
config = {
'account':
'',
'year_set':
1,
'start_year':
start_year,
'end_year':
end_year,
'caseId':
'20180129.DECKv1b_piControl.ne30_oEC.edison',
'annual_mode':
'sdd',
'input_directory':
os.path.join(project_path, 'input', 'atm'),
'climo_output_directory':
os.path.join(project_path, 'output', 'pp', 'ne30', 'climo',
'10yr'),
'regrid_output_directory':
os.path.join(project_path, 'output', 'pp', 'fv129x256', 'climo',
'10yr'),
'regrid_map_path':
self.config['ncclimo']['regrid_map_path'],
'year_set':
1,
'run_scripts_path':
os.path.join(project_path, 'output', 'run_scripts')
}
ncclimo = Ncclimo(config=config, event_list=EventList())
self.assertEqual(ncclimo.status, JobStatus.VALID)
ncclimo.execute(dryrun=True)
self.assertTrue(ncclimo.postvalidate())
def test_finilize_complete(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = ['-c', 'tests/test_configs/test_amwg_complete.cfg']
config, filemanager, runmanager = initialize(
argv=pargv,
version="2.0.0",
branch="master",
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
finalize(config=config,
event_list=self.event_list,
status=1,
runmanager=runmanager)
def test_event_list(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
elist = EventList()
elist.push('beep boop', data=42)
self.assertEquals(len(elist.list), 1)
rmessage = 'boop beed'
elist.replace(index=0, message=rmessage)
self.assertEquals(elist.list[0].message, rmessage)
def test_filemanager_setup_valid_from_scratch(self):
"""
run filemansger setup with no sta
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = False
db = '{}.db'.format(inspect.stack()[0][3])
config_path = 'tests/test_configs/valid_config_from_scratch.cfg'
config = ConfigObj(config_path)
experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil'
filemanager = FileManager(database=db,
event_list=EventList(),
config=config)
self.assertTrue(isinstance(filemanager, FileManager))
self.assertTrue(os.path.exists(db))
os.remove(db)
def test_filemanager_setup_valid_with_inplace_data(self):
"""
run the filemanager setup with sta turned on
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
config_path = 'tests/test_configs/e3sm_diags_complete.cfg'
config = ConfigObj(config_path)
db = '{}.db'.format(inspect.stack()[0][3])
filemanager = FileManager(database=db,
event_list=EventList(),
config=config)
filemanager.populate_file_list()
filemanager.update_local_status()
self.assertTrue(isinstance(filemanager, FileManager))
self.assertTrue(os.path.exists(db))
self.assertTrue(filemanager.all_data_local())
os.remove(db)
def test_finilize_complete_marked_failed(self):
pargv = ['-c', 'tests/test_configs/valid_config_simple.cfg']
event_list = EventList()
kill_event = threading.Event()
config, filemanager, runmanager = initialize(argv=pargv,
version="0.0.0",
branch="000",
event_list=event_list,
kill_event=kill_event,
mutex=threading.Lock(),
testing=True)
self.assertNotEqual(config, False)
self.assertNotEqual(filemanager, False)
self.assertNotEqual(runmanager, False)
finalize(config=config,
event_list=event_list,
kill_event=kill_event,
status=-1,
runmanager=runmanager)
self.assertEqual(runmanager.is_all_done(), -1)
def test_ncclimo_valid_prevalidate(self):
"""
Test that valid input config will be marked as valid by the job
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
config = {
'account':
'',
'year_set':
1,
'start_year':
50,
'end_year':
55,
'caseId':
self.config['global']['experiment'],
'annual_mode':
'sdd',
'input_directory':
os.path.join(self.project_path, 'input'),
'climo_output_directory':
os.path.join(self.config['global']['project_path'], 'output', 'pp',
'ne30', 'climo', '5yr'),
'regrid_output_directory':
os.path.join(self.config['global']['project_path'], 'output', 'pp',
'fv129x256', 'climo', '5yr'),
'regrid_map_path':
self.config['ncclimo']['regrid_map_path'],
'year_set':
1,
'run_scripts_path':
os.path.join(self.project_path, 'output', 'run_scripts')
}
ncclimo = Ncclimo(config=config, event_list=EventList())
ncclimo.status = JobStatus.VALID
self.assertFalse(ncclimo.prevalidate(config))
def test_init_from_scratch_config_bad_project_dir(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = [
'-c',
'tests/test_configs/valid_config_from_scratch_bad_project_path.cfg'
]
project_path = '/usr/testing/'
with self.assertRaises(SystemExit) as exitexception:
config, filemanager, runmanager = initialize(
argv=pargv,
version=__version__,
branch=__branch__,
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
self.assertEqual(config, False)
self.assertEqual(filemanager, False)
self.assertEqual(runmanager, False)
self.assertEqual(os.path.exists(project_path), False)
self.assertEqual(exitexception.exception.code, 1)
def test_ncclimo_setup(self):
"""
Run ncclimo setup on valid config
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
config = {
'account':
'',
'year_set':
1,
'start_year':
50,
'end_year':
55,
'caseId':
self.config['global']['experiment'],
'annual_mode':
'sdd',
'input_directory':
os.path.join(self.project_path, 'input'),
'climo_output_directory':
os.path.join(self.config['global']['project_path'], 'output', 'pp',
'ne30', 'climo', '5yr'),
'regrid_output_directory':
os.path.join(self.config['global']['project_path'], 'output', 'pp',
'fv129x256', 'climo', '5yr'),
'regrid_map_path':
self.config['ncclimo']['regrid_map_path'],
'year_set':
1,
'run_scripts_path':
os.path.join(self.project_path, 'output', 'run_scripts')
}
ncclimo = Ncclimo(config=config, event_list=EventList())
self.assertEqual(ncclimo.status.name, 'VALID')
def test_filemanager_setup_with_sta(self):
"""
run the filemanager setup with sta turned on
"""
"""
############## SETUP ###############
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
sta = True
types = [
'atm', 'ice', 'ocn', 'rest', 'streams.ocean', 'streams.cice',
'mpas-o_in', 'mpas-cice_in', 'meridionalHeatTransport', 'lnd'
]
database = '{}.db'.format(inspect.stack()[0][3])
mutex = threading.Lock()
"""
############## TEST ###############
"""
filemanager = FileManager(mutex=mutex,
event_list=EventList(),
sta=sta,
types=self.file_types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path,
experiment=self.experiment)
self.assertTrue(isinstance(filemanager, FileManager))
self.assertTrue(os.path.exists(database))
"""
############## CLEANUP ###############
"""
os.remove(database)
def test_AMWG_execute_missing_output_path(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
start_year = 10
end_year = 100
self.config['global'][
'project_path'] = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_not_complete'
self.config['global'][
'exeriment'] = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison'
set_string = '{start:04d}-{end:04d}'.format(start=start_year,
end=end_year)
web_directory = os.path.join(
self.config.get('global').get('host_directory'),
os.environ['USER'],
self.config.get('global').get('experiment'),
self.config.get('amwg').get('host_directory'), set_string)
host_url = '/'.join([
self.config.get('global').get('img_host_server'),
os.environ['USER'],
self.config.get('global').get('experiment'),
self.config.get('amwg').get('host_directory'), set_string
])
self.config['global']['output_path'] = os.path.join(
self.config['global']['project_path'], 'output')
self.config['global']['run_scripts_path'] = os.path.join(
self.config['global']['project_path'], 'output', 'run_scripts')
self.config['global']['resource_path'] = os.path.join(
os.getcwd(), 'resources')
regrid_path = os.path.join(self.config['global']['output_path'], 'pp',
'fv129x256', 'climo', '10yr')
output_path = os.path.join(self.config['global']['output_path'],
'diags',
self.config['global']['remap_grid_name'],
self.config['amwg']['host_directory'],
set_string)
img_output_path = os.path.join(
'/p/cscratch/acme/baldwin32/20171016/output/amwg_diag/',
'{start:04d}-{end:04d}{experiment}-obs'.format(
start=start_year,
end=end_year,
experiment=self.config['global']['experiment']))
if os.path.exists(output_path):
shutil.rmtree(output_path)
if os.path.exists(img_output_path):
shutil.rmtree(img_output_path)
template_path = os.path.join(self.config['global']['resource_path'],
'amwg_template.csh')
diag_home = self.config['amwg']['diag_home']
temp_path = os.path.join(self.config['global']['output_path'], 'tmp',
'amwg', set_string)
config = {
'web_dir': web_directory,
'host_url': host_url,
'experiment': self.config.get('global').get('experiment'),
'run_scripts_path': self.config['global']['run_scripts_path'],
'output_path': self.config['global']['output_path'],
'test_casename': self.config.get('global').get('experiment'),
'test_path_history': regrid_path + os.sep,
'regrided_climo_path': regrid_path + os.sep,
'test_path_climo': temp_path,
'test_path_diag': output_path,
'start_year': start_year,
'end_year': end_year,
'year_set': 1,
'run_directory': output_path,
'template_path': template_path,
'diag_home': diag_home
}
amwg = AMWGDiagnostic(config=config, event_list=EventList())
self.assertEqual(amwg.status.name, 'INVALID')
self.assertFalse(amwg.postvalidate())
