def test_runmanager_setup(self): """ Run the runmanager setup """ print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') db_path = os.path.join( self.project_path, '{}.db'.format(inspect.stack()[0][3])) if os.path.exists(self.project_path): shutil.rmtree(self.project_path) os.makedirs(self.project_path) filemanager = FileManager( ui=False, event_list=EventList(), database=db_path, types=['atm'], sta=False, mutex=self.mutex, remote_endpoint=self.remote_endpoint, remote_path=self.remote_path, local_endpoint=self.local_endpoint, local_path=self.project_path, experiment=self.experiment) runmanager = RunManager( short_name='testname', account='', resource_path='./resources', ui=False, event_list=EventList(), output_path=self.output_path, caseID=self.experiment, scripts_path=self.run_scripts_path, thread_list=[], event=threading.Event(), no_host=True, url_prefix='', always_copy=False) runmanager.setup_job_sets( set_frequency=[5, 10], sim_start_year=int(self.config['global']['simulation_start_year']), sim_end_year=int(self.config['global']['simulation_end_year']), config=self.config, filemanager=filemanager) self.assertEqual(len(runmanager.job_sets), 1) for job_set in runmanager.job_sets: if job_set.set_number == 1: self.assertEqual(job_set.length, 5) self.assertEqual(job_set.set_start_year, 1) self.assertEqual(job_set.set_end_year, 5) job_names = job_set.get_job_names() self.assertTrue('ncclimo' in job_names) self.assertTrue('timeseries' in job_names) self.assertTrue('aprime_diags' in job_names) self.assertTrue('e3sm_diags' in job_names)
def test_runmanager_write(self): print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') db_path = os.path.join( self.project_path, '{}.db'.format(inspect.stack()[0][3])) if os.path.exists(db_path): os.remove(db_path) local_path = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_complete' database = '{}.db'.format(inspect.stack()[0][3]) local_path = os.path.join( self.project_path, 'input') types = ['atm'] mutex = threading.Lock() filemanager = FileManager( mutex=mutex, event_list=EventList(), sta=False, types=types, database=database, remote_endpoint=self.remote_endpoint, remote_path=self.remote_path, local_endpoint=self.local_endpoint, local_path=local_path, experiment=self.experiment) self.assertTrue(isinstance(filemanager, FileManager)) runmanager = RunManager( short_name='testname', account='', resource_path='./resources', ui=False, event_list=EventList(), output_path=self.output_path, caseID=self.experiment, scripts_path=self.run_scripts_path, thread_list=[], event=threading.Event(), no_host=True, url_prefix='', always_copy=False) runmanager.setup_job_sets( set_frequency=[5, 10], sim_start_year=int(self.config['global']['simulation_start_year']), sim_end_year=int(self.config['global']['simulation_end_year']), config=self.config, filemanager=filemanager) path = os.path.join(self.project_path, 'output', 'job_state.txt') runmanager.write_job_sets(path) self.assertTrue(os.path.exists(path)) os.remove(database)
def test_cmor_valid_completed(self): """ tests that a valid config on a completed case will mark itself as already being run and not start """ print_message( '\n---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') config = ConfigObj(self.valid_config_path) config['post-processing']['cmor']['variable_list'] = [ config['post-processing']['cmor']['variable_list'] ] case_name = '20180129.DECKv1b_piControl.ne30_oEC.edison' case = config['simulations'][case_name] messages = verify_config(config) self.assertEqual(len(messages), 0) config['global']['resource_path'] = 'resources/' filemanager = FileManager(config=config, event_list=EventList()) filemanager.populate_file_list() filemanager.update_local_status() timeseries = Timeseries(short_name=case['short_name'], case=case_name, start=config['simulations']['start_year'], end=config['simulations']['end_year'], config=config, run_type='atm') timeseries.check_data_ready(filemanager=filemanager) timeseries.setup_data(config=config, filemanager=filemanager, case=case_name) timeseries.execute(config=config, event_list=EventList()) timeseries.handle_completion(filemanager=filemanager, config=config, event_list=EventList()) cmor = Cmor(short_name=case['short_name'], case=case_name, start=config['simulations']['start_year'], end=config['simulations']['end_year'], config=config) cmor.check_data_ready(filemanager=filemanager) cmor.setup_data(config=config, filemanager=filemanager, case=case_name) self.assertTrue(cmor.postvalidate(config=config)) self.assertTrue(cmor.execute(config=config, event_list=EventList())) self.assertEquals(cmor.status, JobStatus.COMPLETED) self.assertTrue( cmor.handle_completion(filemanager=filemanager, event_list=EventList(), config=config))
def test_init_from_scratch_config_globus_bad_uuid(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') pargv = [ '-c', 'tests/test_configs/valid_config_from_scratch_globus_bad_uuid.cfg' ] project_path = '/p/user_pub/e3sm/baldwin32/testing/empty/' if os.path.exists(project_path): shutil.rmtree(project_path) config, filemanager, runmanager = initialize( argv=pargv, version=__version__, branch=__branch__, event_list=EventList(), kill_event=threading.Event(), testing=True) self.assertEqual(config, False) self.assertEqual(filemanager, False) self.assertEqual(runmanager, False) self.assertEqual(os.path.exists(project_path), True) if os.path.exists(project_path): shutil.rmtree(project_path)
def test_transfer_file(self): """ test moving a single file """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') source_file = { 'remote_path': '/global/homes/s/sbaldwin/test_directory/test_file.txt', 'local_path': os.path.join(project_path, 'test_file.txt') } source_path = '/global/homes/s/sbaldwin/test_directory' transfer = Transfer( { 'file_list': [source_file], 'recursive': False, 'source_endpoint': self.source_endpoint, 'destination_endpoint': self.destination_endpoint, 'source_path': source_path, 'destination_path': project_path, 'source_email': '*****@*****.**', 'display_event': threading.Event(), 'ui': False }, event_list=EventList()) transfer.execute(event=threading.Event()) self.assertTrue(transfer.postvalidate()) self.assertEqual(transfer.status.name, 'COMPLETED') self.assertTrue(os.path.exists(source_file['local_path']))
def test_filemanager_populate_no_sta(self): """ run filemanager set and populate with sta turned off """ """ ############### SETUP ################ """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') sta = False database = '{}.db'.format(inspect.stack()[0][3]) simstart = 1 simend = 10 experiment = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison' mutex = threading.Lock() """ ############## TEST ############### """ filemanager = FileManager(event_list=EventList(), mutex=mutex, sta=sta, types=self.file_types, database=database, remote_endpoint=self.remote_endpoint, remote_path=self.remote_path, local_endpoint=self.local_endpoint, local_path=self.local_path, experiment=self.experiment) filemanager.populate_file_list(simstart=simstart, simend=simend, experiment=experiment) filemanager.mutex.acquire() simlength = simend - simstart + 1 for _type in ['atm', 'lnd', 'ocn', 'ice']: file_names = [ x.name for x in DataFile.select().where(DataFile.datatype == _type) ] if not len(file_names) == (simlength * 12): print _type + ' does not have ' + str( simlength * 12) + ' files' self.assertEqual(len(file_names), (simlength * 12)) for year in range(simstart, simend + 1): for month in range(1, 13): name = (file_type_map[_type].replace( 'EXPERIMENT', experiment).replace('YEAR', '{:04d}'.format(year)).replace( 'MONTH', '{:02}'.format(month))) self.assertTrue(name in file_names) filemanager.mutex.release() """ ############## CLEANUP ############### """ os.remove(database)
def test_finilize_complete(self): pargv = ['-c', 'tests/test_configs/valid_config_simple.cfg'] event_list = EventList() kill_event = threading.Event() config, filemanager, runmanager = initialize(argv=pargv, version="0.0.0", branch="000", event_list=event_list, kill_event=kill_event, mutex=threading.Lock(), testing=True) self.assertNotEqual(config, False) self.assertNotEqual(filemanager, False) self.assertNotEqual(runmanager, False) # all jobs should be complete already # this will mark them as such runmanager.check_data_ready() runmanager.start_ready_jobs() runmanager.monitor_running_jobs() finalize(config=config, event_list=event_list, kill_event=kill_event, status=runmanager.is_all_done(), runmanager=runmanager) self.assertTrue(runmanager.is_all_done())
def test_filemanager_update_local(self): """ run filemanager set and populate, then create a dummy file in the input directory and run update_local which should mark it as present """ """ ############# SETUP ################ """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') sta = False types = ['atm', 'ice', 'ocn', 'rest', 'streams.cice', 'streams.ocean'] database = '{}.db'.format(inspect.stack()[0][3]) simstart = 51 simend = 60 remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil' experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil' mutex = threading.Lock() """ ############### TEST ################# """ filemanager = FileManager(event_list=EventList(), mutex=mutex, sta=sta, types=types, database=database, remote_endpoint=self.remote_endpoint, remote_path=remote_path, local_endpoint=self.local_endpoint, local_path=self.local_path, experiment=experiment) filemanager.populate_file_list(simstart=simstart, simend=simend, experiment=experiment) filemanager.mutex.acquire() df = DataFile.select().limit(1) filemanager.mutex.release() name = df[0].name head, tail = os.path.split(df[0].local_path) if not os.path.exists(head): os.makedirs(head) dummy_file_path = df[0].local_path print '----- writing out dummy file at {} -----'.format( dummy_file_path) with open(dummy_file_path, 'w') as fp: fp.write('this is a test file') filemanager.update_local_status() filemanager.mutex.acquire() df = DataFile.select().where(DataFile.name == name)[0] filemanager.mutex.release() self.assertEqual(df.local_status, 0) self.assertTrue(df.local_size > 0) """ ############### CLEANUP ################# """ os.remove(database)
def test_filemanager_setup_no_sta(self): """ run filemansger setup with no sta """ """ ############## SETUP ################ """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') sta = False database = '{}.db'.format(inspect.stack()[0][3]) remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil' mutex = threading.Lock() experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil' """ ############### TEST ################ """ filemanager = FileManager(mutex=mutex, event_list=EventList(), sta=sta, types=self.file_types, database=database, remote_endpoint=self.remote_endpoint, remote_path=remote_path, local_endpoint=self.local_endpoint, local_path=self.local_path, experiment=experiment) self.assertTrue(isinstance(filemanager, FileManager)) self.assertTrue(os.path.exists(database)) """ ############## CLEANUP ############### """ os.remove(database)
def test_timeseries_setup(self): """ test a valid timeseries setup """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') timeseries = Timeseries( { 'regrid_output_directory': os.getcwd(), 'filemanager': None, 'year_set': 1, 'annual_mode': 'sdd', 'start_year': 50, 'end_year': 55, 'output_directory': os.path.join(self.project_path, 'output', 'monthly'), 'var_list': self.config['ncclimo']['var_list'], 'caseId': self.config['global']['experiment'], 'run_scripts_path': os.path.join(self.project_path, 'output', 'run_scripts'), 'regrid_map_path': self.config['ncclimo']['regrid_map_path'], 'file_list': [], }, EventList()) self.assertEqual(timeseries.status.name, 'VALID')
def test_AMWG_execute_completed(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') start_year = 1 end_year = 10 self.config['global'][ 'project_path'] = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_complete' self.config['global'][ 'exeriment'] = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison' set_string = '{start:04d}-{end:04d}'.format(start=start_year, end=end_year) web_directory = os.path.join( self.config.get('global').get('host_directory'), os.environ['USER'], self.config.get('global').get('experiment'), self.config.get('amwg').get('host_directory'), set_string) host_url = '/'.join([ self.config.get('global').get('img_host_server'), os.environ['USER'], self.config.get('global').get('experiment'), self.config.get('amwg').get('host_directory'), set_string ]) self.config['global']['output_path'] = os.path.join( self.config['global']['project_path'], 'output') self.config['global']['run_scripts_path'] = os.path.join( self.config['global']['project_path'], 'output', 'run_scripts') self.config['global']['resource_path'] = os.path.join( os.getcwd(), 'resources') regrid_path = os.path.join(self.config['global']['output_path'], 'pp', 'fv129x256', 'climo', '10yr') output_path = os.path.join(self.config['global']['output_path'], 'amwg_diag', set_string) template_path = os.path.join(self.config['global']['resource_path'], 'amwg_template.csh') diag_home = self.config['amwg']['diag_home'] temp_path = os.path.join(output_path, 'tmp', 'amwg', set_string) config = { 'web_dir': web_directory, 'host_url': host_url, 'experiment': self.config.get('global').get('experiment'), 'run_scripts_path': self.config['global']['run_scripts_path'], 'output_path': self.config['global']['output_path'], 'test_casename': self.config.get('global').get('experiment'), 'test_path_history': regrid_path + os.sep, 'regrided_climo_path': regrid_path + os.sep, 'test_path_climo': temp_path, 'test_path_diag': output_path, 'start_year': start_year, 'end_year': end_year, 'year_set': 1, 'run_directory': output_path, 'template_path': template_path, 'diag_home': diag_home } amwg = AMWGDiagnostic(config=config, event_list=EventList()) self.assertEqual(amwg.status.name, 'VALID') amwg.execute(dryrun=True) self.assertEqual(amwg.status.name, 'COMPLETED')
def __init__(self, *args, **kwargs): super(TestAMWGDiagnostic, self).__init__(*args, **kwargs) self.config_path = os.path.join(os.getcwd(), 'tests', 'test_configs', 'test_amwg_complete.cfg') self.config = ConfigObj(self.config_path) self.config['global']['run_scripts_path'] = os.path.join( self.config['global']['project_path'], 'output', 'scripts') self.event_list = EventList()
def test_init_valid_config_simple(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') pargv = ['-c', 'tests/test_configs/valid_config_simple.cfg'] config, filemanager, runmanager = initialize( argv=pargv, version=__version__, branch=__branch__, event_list=EventList(), kill_event=threading.Event(), testing=True)
def test_event_list(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') elist = EventList() elist.push('beep boop', data=42) self.assertEquals(len(elist.list), 1) rmessage = 'boop beed' elist.replace(index=0, message=rmessage) self.assertEquals(elist.list[0].message, rmessage)
def test_filemanager_update_remote_no_sta(self): """ run filemanager setup and populate, then run update_remote_status with 10 years of atm output, and finally run all_data_remote to show that all the remote data has been recognized """ """ ############# SETUP ################## """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil' sta = False types = ['atm'] database = '{}.db'.format(inspect.stack()[0][3]) simstart = 51 simend = 60 experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil' mutex = threading.Lock() """ ################ TEST ################## """ filemanager = FileManager(event_list=EventList(), mutex=mutex, sta=False, types=types, database=database, remote_endpoint=self.remote_endpoint, remote_path=remote_path, local_endpoint=self.local_endpoint, local_path=self.local_path, experiment=experiment) filemanager.populate_file_list(simstart=simstart, simend=simend, experiment=experiment) client = get_client() filemanager.update_remote_status(client) filemanager.mutex.acquire() for datafile in DataFile.select(): if datafile.remote_status != 0: print datafile.name, datafile.remote_path, datafile.remote_status, datafile.datatype self.assertEqual(datafile.remote_status, 0) if filemanager.mutex.locked(): filemanager.mutex.release() self.assertTrue(filemanager.all_data_remote()) """ ############## CLEANUP ############### """ os.remove(database)
def test_filemanager_update_remote_yes_sta(self): """ run filemanager setup and populate, then run update_remote_status on a directory that has been short term archived """ """ ############### SETUP ################# """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') sta = True types = [ 'atm', 'ice', 'ocn', 'rest', 'streams.ocean', 'streams.cice', 'mpas-o_in', 'mpas-cice_in', 'meridionalHeatTransport' ] database = '{}.db'.format(inspect.stack()[0][3]) simstart = 51 source_path = '/global/cscratch1/sd/golaz/ACME_simulations/20180215.DECKv1b_1pctCO2.ne30_oEC.edison' simend = 60 experiment = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison' mutex = threading.Lock() """ ############### TEST ################# """ filemanager = FileManager(event_list=EventList(), mutex=mutex, sta=sta, types=types, database=database, remote_endpoint=self.remote_endpoint, remote_path=source_path, local_endpoint=self.local_endpoint, local_path=self.local_path, experiment=self.experiment) filemanager.populate_file_list(simstart=simstart, simend=simend, experiment=experiment) client = get_client() filemanager.update_remote_status(client) filemanager.mutex.acquire() for datafile in DataFile.select(): if datafile.remote_status != 0: print datafile.name, datafile.remote_path self.assertEqual(datafile.remote_status, 0) if filemanager.mutex.locked(): filemanager.mutex.release() self.assertTrue(filemanager.all_data_remote()) """ ############## CLEANUP ############### """ os.remove(database)
def test_ncclimo_execute_not_completed(self): """ Test that ncclimo will do all proper setup in an incomplete run """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') start_year = 51 end_year = 55 self.config['global'][ 'project_path'] = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_not_complete' self.config['global'][ 'exeriment'] = '20171011.beta2_FCT2-icedeep_branch.A_WCYCL1850S.ne30_oECv3_ICG.edison' year_set_string = '{start:04d}-{end:04d}'.format(start=start_year, end=end_year) config = { 'account': '', 'year_set': 1, 'start_year': start_year, 'end_year': end_year, 'caseId': self.config['global']['experiment'], 'annual_mode': 'sdd', 'input_directory': os.path.join(self.config['global']['project_path'], 'input', 'atm'), 'climo_output_directory': os.path.join(self.config['global']['project_path'], 'output', 'pp', 'ne30', 'climo', '5yr'), 'regrid_output_directory': os.path.join(self.config['global']['project_path'], 'output', 'pp', 'fv129x256', 'climo', '5yr'), 'regrid_map_path': self.config['ncclimo']['regrid_map_path'], 'year_set': 1, 'run_scripts_path': os.path.join(self.config['global']['project_path'], 'output', 'run_scripts') } ncclimo = Ncclimo(config=config, event_list=EventList()) self.assertEqual(ncclimo.status, JobStatus.VALID) self.assertFalse(ncclimo.execute(dryrun=True)) self.assertEqual(ncclimo.status.name, 'COMPLETED')
def test_init_missing_lnd(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') pargv = ['-c', 'tests/test_configs/invalid_config_missing_lnd.cfg'] config, filemanager, runmanager = initialize( argv=pargv, version=__version__, branch=__branch__, event_list=EventList(), kill_event=threading.Event(), testing=True) self.assertEqual(config, False) self.assertEqual(filemanager, False) self.assertEqual(runmanager, False)
def test_ncclimo_execute_bad_year(self): """ test that if given the wrong input year ncclimo will exit correctly """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') start_year = 55 end_year = 60 self.config['global'][ 'project_path'] = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_complete' self.config['global'][ 'exeriment'] = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison' year_set_string = '{start:04d}-{end:04d}'.format(start=start_year, end=end_year) config = { 'account': '', 'year_set': 1, 'start_year': start_year, 'end_year': end_year, 'caseId': self.config['global']['experiment'], 'annual_mode': 'sdd', 'input_directory': os.path.join(self.config['global']['project_path'], 'input', 'atm'), 'climo_output_directory': os.path.join(self.config['global']['project_path'], 'output', 'climo', '5yr'), 'regrid_output_directory': os.path.join(self.config['global']['project_path'], 'output', 'climo_regrid'), 'regrid_map_path': self.config['ncclimo']['regrid_map_path'], 'year_set': 1, 'run_scripts_path': os.path.join(self.config['global']['project_path'], 'output', 'run_scripts') } ncclimo = Ncclimo(config=config, event_list=EventList()) self.assertEqual(ncclimo.status, JobStatus.VALID) self.assertFalse(ncclimo.postvalidate())
def test_ncclimo_execute_bad_climo_dir(self): """ test that ncclimo will correctly exit if given a non-existant climo dir """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') start_year = 56 end_year = 60 self.config['global'][ 'project_path'] = '/p/cscratch/acme/baldwin32/20171016/' self.config['global'][ 'exeriment'] = '20171011.beta2_FCT2-icedeep_branch.A_WCYCL1850S.ne30_oECv3_ICG.edison' year_set_string = '{start:04d}-{end:04d}'.format(start=start_year, end=end_year) config = { 'account': '', 'year_set': 1, 'start_year': start_year, 'end_year': end_year, 'caseId': self.config['global']['experiment'], 'annual_mode': 'sdd', 'input_directory': os.path.join(self.config['global']['project_path'], 'input', 'atm'), 'climo_output_directory': os.path.join(self.config['global']['project_path'], 'output', 'climo', 'XXYYZZ'), 'regrid_output_directory': os.path.join(self.config['global']['project_path'], 'output', 'climo_regrid'), 'regrid_map_path': self.config['ncclimo']['regrid_map_path'], 'year_set': 1, 'run_scripts_path': os.path.join(self.config['global']['project_path'], 'output', 'run_scripts') } ncclimo = Ncclimo(config=config, event_list=EventList()) self.assertEqual(ncclimo.status, JobStatus.VALID) self.assertFalse(ncclimo.postvalidate())
def test_ncclimo_execute_completed(self): """ test that if ncclimo is told to run on a project thats already completed ncclimo for the given yearset it will varify that the output is present and not run again """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') start_year = 1 end_year = 10 # REAL DATA project_path = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_complete' year_set_string = '{start:04d}-{end:04d}'.format(start=start_year, end=end_year) config = { 'account': '', 'year_set': 1, 'start_year': start_year, 'end_year': end_year, 'caseId': '20180129.DECKv1b_piControl.ne30_oEC.edison', 'annual_mode': 'sdd', 'input_directory': os.path.join(project_path, 'input', 'atm'), 'climo_output_directory': os.path.join(project_path, 'output', 'pp', 'ne30', 'climo', '10yr'), 'regrid_output_directory': os.path.join(project_path, 'output', 'pp', 'fv129x256', 'climo', '10yr'), 'regrid_map_path': self.config['ncclimo']['regrid_map_path'], 'year_set': 1, 'run_scripts_path': os.path.join(project_path, 'output', 'run_scripts') } ncclimo = Ncclimo(config=config, event_list=EventList()) self.assertEqual(ncclimo.status, JobStatus.VALID) ncclimo.execute(dryrun=True) self.assertTrue(ncclimo.postvalidate())
def test_finilize_complete(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') pargv = ['-c', 'tests/test_configs/test_amwg_complete.cfg'] config, filemanager, runmanager = initialize( argv=pargv, version="2.0.0", branch="master", event_list=EventList(), kill_event=threading.Event(), testing=True) finalize(config=config, event_list=self.event_list, status=1, runmanager=runmanager)
def test_filemanager_setup_valid_from_scratch(self): """ run filemansger setup with no sta """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') sta = False db = '{}.db'.format(inspect.stack()[0][3]) config_path = 'tests/test_configs/valid_config_from_scratch.cfg' config = ConfigObj(config_path) experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil' filemanager = FileManager(database=db, event_list=EventList(), config=config) self.assertTrue(isinstance(filemanager, FileManager)) self.assertTrue(os.path.exists(db)) os.remove(db)
def test_filemanager_setup_valid_with_inplace_data(self): """ run the filemanager setup with sta turned on """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') config_path = 'tests/test_configs/e3sm_diags_complete.cfg' config = ConfigObj(config_path) db = '{}.db'.format(inspect.stack()[0][3]) filemanager = FileManager(database=db, event_list=EventList(), config=config) filemanager.populate_file_list() filemanager.update_local_status() self.assertTrue(isinstance(filemanager, FileManager)) self.assertTrue(os.path.exists(db)) self.assertTrue(filemanager.all_data_local()) os.remove(db)
def test_finilize_complete_marked_failed(self): pargv = ['-c', 'tests/test_configs/valid_config_simple.cfg'] event_list = EventList() kill_event = threading.Event() config, filemanager, runmanager = initialize(argv=pargv, version="0.0.0", branch="000", event_list=event_list, kill_event=kill_event, mutex=threading.Lock(), testing=True) self.assertNotEqual(config, False) self.assertNotEqual(filemanager, False) self.assertNotEqual(runmanager, False) finalize(config=config, event_list=event_list, kill_event=kill_event, status=-1, runmanager=runmanager) self.assertEqual(runmanager.is_all_done(), -1)
def test_ncclimo_valid_prevalidate(self): """ Test that valid input config will be marked as valid by the job """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') config = { 'account': '', 'year_set': 1, 'start_year': 50, 'end_year': 55, 'caseId': self.config['global']['experiment'], 'annual_mode': 'sdd', 'input_directory': os.path.join(self.project_path, 'input'), 'climo_output_directory': os.path.join(self.config['global']['project_path'], 'output', 'pp', 'ne30', 'climo', '5yr'), 'regrid_output_directory': os.path.join(self.config['global']['project_path'], 'output', 'pp', 'fv129x256', 'climo', '5yr'), 'regrid_map_path': self.config['ncclimo']['regrid_map_path'], 'year_set': 1, 'run_scripts_path': os.path.join(self.project_path, 'output', 'run_scripts') } ncclimo = Ncclimo(config=config, event_list=EventList()) ncclimo.status = JobStatus.VALID self.assertFalse(ncclimo.prevalidate(config))
def test_init_from_scratch_config_bad_project_dir(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') pargv = [ '-c', 'tests/test_configs/valid_config_from_scratch_bad_project_path.cfg' ] project_path = '/usr/testing/' with self.assertRaises(SystemExit) as exitexception: config, filemanager, runmanager = initialize( argv=pargv, version=__version__, branch=__branch__, event_list=EventList(), kill_event=threading.Event(), testing=True) self.assertEqual(config, False) self.assertEqual(filemanager, False) self.assertEqual(runmanager, False) self.assertEqual(os.path.exists(project_path), False) self.assertEqual(exitexception.exception.code, 1)
def test_ncclimo_setup(self): """ Run ncclimo setup on valid config """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') config = { 'account': '', 'year_set': 1, 'start_year': 50, 'end_year': 55, 'caseId': self.config['global']['experiment'], 'annual_mode': 'sdd', 'input_directory': os.path.join(self.project_path, 'input'), 'climo_output_directory': os.path.join(self.config['global']['project_path'], 'output', 'pp', 'ne30', 'climo', '5yr'), 'regrid_output_directory': os.path.join(self.config['global']['project_path'], 'output', 'pp', 'fv129x256', 'climo', '5yr'), 'regrid_map_path': self.config['ncclimo']['regrid_map_path'], 'year_set': 1, 'run_scripts_path': os.path.join(self.project_path, 'output', 'run_scripts') } ncclimo = Ncclimo(config=config, event_list=EventList()) self.assertEqual(ncclimo.status.name, 'VALID')
def test_filemanager_setup_with_sta(self): """ run the filemanager setup with sta turned on """ """ ############## SETUP ############### """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') sta = True types = [ 'atm', 'ice', 'ocn', 'rest', 'streams.ocean', 'streams.cice', 'mpas-o_in', 'mpas-cice_in', 'meridionalHeatTransport', 'lnd' ] database = '{}.db'.format(inspect.stack()[0][3]) mutex = threading.Lock() """ ############## TEST ############### """ filemanager = FileManager(mutex=mutex, event_list=EventList(), sta=sta, types=self.file_types, database=database, remote_endpoint=self.remote_endpoint, remote_path=self.remote_path, local_endpoint=self.local_endpoint, local_path=self.local_path, experiment=self.experiment) self.assertTrue(isinstance(filemanager, FileManager)) self.assertTrue(os.path.exists(database)) """ ############## CLEANUP ############### """ os.remove(database)
def test_AMWG_execute_missing_output_path(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') start_year = 10 end_year = 100 self.config['global'][ 'project_path'] = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_not_complete' self.config['global'][ 'exeriment'] = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison' set_string = '{start:04d}-{end:04d}'.format(start=start_year, end=end_year) web_directory = os.path.join( self.config.get('global').get('host_directory'), os.environ['USER'], self.config.get('global').get('experiment'), self.config.get('amwg').get('host_directory'), set_string) host_url = '/'.join([ self.config.get('global').get('img_host_server'), os.environ['USER'], self.config.get('global').get('experiment'), self.config.get('amwg').get('host_directory'), set_string ]) self.config['global']['output_path'] = os.path.join( self.config['global']['project_path'], 'output') self.config['global']['run_scripts_path'] = os.path.join( self.config['global']['project_path'], 'output', 'run_scripts') self.config['global']['resource_path'] = os.path.join( os.getcwd(), 'resources') regrid_path = os.path.join(self.config['global']['output_path'], 'pp', 'fv129x256', 'climo', '10yr') output_path = os.path.join(self.config['global']['output_path'], 'diags', self.config['global']['remap_grid_name'], self.config['amwg']['host_directory'], set_string) img_output_path = os.path.join( '/p/cscratch/acme/baldwin32/20171016/output/amwg_diag/', '{start:04d}-{end:04d}{experiment}-obs'.format( start=start_year, end=end_year, experiment=self.config['global']['experiment'])) if os.path.exists(output_path): shutil.rmtree(output_path) if os.path.exists(img_output_path): shutil.rmtree(img_output_path) template_path = os.path.join(self.config['global']['resource_path'], 'amwg_template.csh') diag_home = self.config['amwg']['diag_home'] temp_path = os.path.join(self.config['global']['output_path'], 'tmp', 'amwg', set_string) config = { 'web_dir': web_directory, 'host_url': host_url, 'experiment': self.config.get('global').get('experiment'), 'run_scripts_path': self.config['global']['run_scripts_path'], 'output_path': self.config['global']['output_path'], 'test_casename': self.config.get('global').get('experiment'), 'test_path_history': regrid_path + os.sep, 'regrided_climo_path': regrid_path + os.sep, 'test_path_climo': temp_path, 'test_path_diag': output_path, 'start_year': start_year, 'end_year': end_year, 'year_set': 1, 'run_directory': output_path, 'template_path': template_path, 'diag_home': diag_home } amwg = AMWGDiagnostic(config=config, event_list=EventList()) self.assertEqual(amwg.status.name, 'INVALID') self.assertFalse(amwg.postvalidate())