def prevalidate(self, config=None): """ Validates transfer inputs """ if self.status == JobStatus.VALID: return 0 if not check_logged_in(): self.status = JobStatus.INVALID return 1 for i in config: if i in self.inputs: if i == 'recursive': if config.get(i) == 'True': self.config[i] = True else: self.config[i] = False else: self.config[i] = config.get(i) for i in self.inputs: if i not in self.config or self.config[i] is None: if i == 'recursive': self.config[i] = False else: print_message('Missing transfer argument {}'.format(i)) self.status = JobStatus.INVALID return -1 self.status = JobStatus.VALID return 0
def test_send_mail_invalid(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') m = Mailer(src='xxyyzz', dst='xxyyzz') ret = m.send(status='THIS IS A TEST', msg='THIS IS ONLY A TEST') self.assertFalse(ret)
def test_send_mail_valid(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') m = Mailer(src='*****@*****.**', dst='*****@*****.**') ret = m.send(status='THIS IS A TEST', msg='THIS IS ONLY A TEST') self.assertTrue(ret)
def test_e3sm_diags_prevalidate(self): """ test that the e3sm_diags prevalidate and prerun setup works correctly """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') _args = ['-c', self.config_path, '-r', 'resources/'] config, filemanager, runmanager = initialize( argv=_args, version="2.0.0", branch="testing", event_list=self.event_list, kill_event=Event(), testing=True) self.assertFalse(config is None) self.assertFalse(filemanager is None) self.assertFalse(runmanager is None) config['global']['dryrun'] = True runmanager.check_data_ready() runmanager.start_ready_jobs() for case in runmanager.cases: for job in case['jobs']: if job.job_type == 'e3sm_diags': job.setup_data( config=config, filemanager=filemanager, case='20180129.DECKv1b_piControl.ne30_oEC.edison') job.execute(config=config, dryrun=True) self.assertEquals(job.status, JobStatus.COMPLETED)
def test_init_from_scratch_config_globus_bad_uuid(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') pargv = [ '-c', 'tests/test_configs/valid_config_from_scratch_globus_bad_uuid.cfg' ] project_path = '/p/user_pub/e3sm/baldwin32/testing/empty/' if os.path.exists(project_path): shutil.rmtree(project_path) config, filemanager, runmanager = initialize( argv=pargv, version=__version__, branch=__branch__, event_list=EventList(), kill_event=threading.Event(), testing=True) self.assertEqual(config, False) self.assertEqual(filemanager, False) self.assertEqual(runmanager, False) self.assertEqual(os.path.exists(project_path), True) if os.path.exists(project_path): shutil.rmtree(project_path)
def test_invalid_bad_data_type(self): print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') config_path = 'tests/test_configs/invalid_config_bad_img_hosting.cfg' config = ConfigObj(config_path) messages = verify_config(config) self.assertTrue('image hosting is turned on, but no img_host_server specified' in messages) self.assertTrue('image hosting is turned on, but no host_directory specified' in messages)
def split(self): """ Perform the requested variable extraction """ msg = f'starting extraction with nproc = {self._nproc}' logging.info(msg) print_message(msg, status='ok') pool_res = list() self._pool = Pool(self._nproc) for var in var_list: outfile = os.path.join( self._output_path, f'{var}_{self._start:04d}01_{self._end:04d}12.nc') pool_res.append( self._pool.apply_async( _split_one, [var, self._file_list, outfile])) out = None err = None for idx, res in enumerate(pool_res): out, err = res.get(9999999) if err: print out, err else: out += f', {idx + 1}/{len(pool_res)} jobs complete' print_message(out, 'ok') self._pool.close() self._pool.join() os.chdir(self._prev_dir)
def test_transfer_file(self): """ test moving a single file """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') source_file = { 'remote_path': '/global/homes/s/sbaldwin/test_directory/test_file.txt', 'local_path': os.path.join(project_path, 'test_file.txt') } source_path = '/global/homes/s/sbaldwin/test_directory' transfer = Transfer( { 'file_list': [source_file], 'recursive': False, 'source_endpoint': self.source_endpoint, 'destination_endpoint': self.destination_endpoint, 'source_path': source_path, 'destination_path': project_path, 'source_email': '*****@*****.**', 'display_event': threading.Event(), 'ui': False }, event_list=EventList()) transfer.execute(event=threading.Event()) self.assertTrue(transfer.postvalidate()) self.assertEqual(transfer.status.name, 'COMPLETED') self.assertTrue(os.path.exists(source_file['local_path']))
def test_timeseries_setup(self): """ test a valid timeseries setup """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') timeseries = Timeseries( { 'regrid_output_directory': os.getcwd(), 'filemanager': None, 'year_set': 1, 'annual_mode': 'sdd', 'start_year': 50, 'end_year': 55, 'output_directory': os.path.join(self.project_path, 'output', 'monthly'), 'var_list': self.config['ncclimo']['var_list'], 'caseId': self.config['global']['experiment'], 'run_scripts_path': os.path.join(self.project_path, 'output', 'run_scripts'), 'regrid_map_path': self.config['ncclimo']['regrid_map_path'], 'file_list': [], }, EventList()) self.assertEqual(timeseries.status.name, 'VALID')
def test_filemanager_update_local(self): """ run filemanager set and populate, then create a dummy file in the input directory and run update_local which should mark it as present """ """ ############# SETUP ################ """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') sta = False types = ['atm', 'ice', 'ocn', 'rest', 'streams.cice', 'streams.ocean'] database = '{}.db'.format(inspect.stack()[0][3]) simstart = 51 simend = 60 remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil' experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil' mutex = threading.Lock() """ ############### TEST ################# """ filemanager = FileManager(event_list=EventList(), mutex=mutex, sta=sta, types=types, database=database, remote_endpoint=self.remote_endpoint, remote_path=remote_path, local_endpoint=self.local_endpoint, local_path=self.local_path, experiment=experiment) filemanager.populate_file_list(simstart=simstart, simend=simend, experiment=experiment) filemanager.mutex.acquire() df = DataFile.select().limit(1) filemanager.mutex.release() name = df[0].name head, tail = os.path.split(df[0].local_path) if not os.path.exists(head): os.makedirs(head) dummy_file_path = df[0].local_path print '----- writing out dummy file at {} -----'.format( dummy_file_path) with open(dummy_file_path, 'w') as fp: fp.write('this is a test file') filemanager.update_local_status() filemanager.mutex.acquire() df = DataFile.select().where(DataFile.name == name)[0] filemanager.mutex.release() self.assertEqual(df.local_status, 0) self.assertTrue(df.local_size > 0) """ ############### CLEANUP ################# """ os.remove(database)
def test_filemanager_setup_no_sta(self): """ run filemansger setup with no sta """ """ ############## SETUP ################ """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') sta = False database = '{}.db'.format(inspect.stack()[0][3]) remote_path = '/global/homes/r/renata/ACME_simulations/20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil' mutex = threading.Lock() experiment = '20170926.FCT2.A_WCYCL1850S.ne30_oECv3.anvil' """ ############### TEST ################ """ filemanager = FileManager(mutex=mutex, event_list=EventList(), sta=sta, types=self.file_types, database=database, remote_endpoint=self.remote_endpoint, remote_path=remote_path, local_endpoint=self.local_endpoint, local_path=self.local_path, experiment=experiment) self.assertTrue(isinstance(filemanager, FileManager)) self.assertTrue(os.path.exists(database)) """ ############## CLEANUP ############### """ os.remove(database)
def test_invalid_missing_project(self): print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') config_path = 'tests/test_configs/invalid_config_no_project_path.cfg' config = ConfigObj(config_path) messages = verify_config(config) self.assertTrue('no project_path in global options' in messages) self.assertTrue('No data_types section found in config' in messages)
def test_filemanager_populate_no_sta(self): """ run filemanager set and populate with sta turned off """ """ ############### SETUP ################ """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') sta = False database = '{}.db'.format(inspect.stack()[0][3]) simstart = 1 simend = 10 experiment = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison' mutex = threading.Lock() """ ############## TEST ############### """ filemanager = FileManager(event_list=EventList(), mutex=mutex, sta=sta, types=self.file_types, database=database, remote_endpoint=self.remote_endpoint, remote_path=self.remote_path, local_endpoint=self.local_endpoint, local_path=self.local_path, experiment=self.experiment) filemanager.populate_file_list(simstart=simstart, simend=simend, experiment=experiment) filemanager.mutex.acquire() simlength = simend - simstart + 1 for _type in ['atm', 'lnd', 'ocn', 'ice']: file_names = [ x.name for x in DataFile.select().where(DataFile.datatype == _type) ] if not len(file_names) == (simlength * 12): print _type + ' does not have ' + str( simlength * 12) + ' files' self.assertEqual(len(file_names), (simlength * 12)) for year in range(simstart, simend + 1): for month in range(1, 13): name = (file_type_map[_type].replace( 'EXPERIMENT', experiment).replace('YEAR', '{:04d}'.format(year)).replace( 'MONTH', '{:02}'.format(month))) self.assertTrue(name in file_names) filemanager.mutex.release() """ ############## CLEANUP ############### """ os.remove(database)
def test_amwg_execution_completed_job(self): """ test that when run on a completed set of jobs, amwg recognizes that the run has already taken place and doesnt start again """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') _args = ['-c', self.config_path, '-r', 'resources/'] config, filemanager, runmanager = initialize( argv=_args, version="2.0.0", branch="testing", event_list=self.event_list, kill_event=Event(), testing=True) config['global']['dryrun'] = True runmanager.check_data_ready() runmanager.start_ready_jobs() for case in runmanager.cases: for job in case['jobs']: if job.job_type == 'amwg': job.setup_data( config=config, filemanager=filemanager, case='20180129.DECKv1b_piControl.ne30_oEC.edison') job.execute(config=config, dryrun=True) self.assertEquals(job.status, JobStatus.COMPLETED)
def test_render_bad_input_file(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') render_target = os.path.join(os.getcwd(), 'tests', 'DOES_NOT_EXIST') render_output = os.path.join(os.getcwd(), 'tests', 'render_output.txt') self.assertFalse(render({}, render_target, render_output))
def test_processflow_from_scratch_yes_sta(self): """ End to end test from scrath with no sta """ print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') config_path = os.path.join(os.getcwd(), 'tests', 'test_configs', 'test_processflow_with_inplace_data.cfg') output_path = os.path.join('/export/baldwin32/jenkins/workspace/', inspect.stack()[0][3], 'output') input_path = os.path.join('/export/baldwin32/jenkins/workspace/', inspect.stack()[0][3], 'input') if os.path.exists(output_path): print "removing previous output directory" shutil.rmtree(output_path) if os.path.exists(input_path): print "removing previous input directory" shutil.rmtree(input_path) print "---- project cleanup complete ----" testargs = [ '-c', config_path, '-n', '-f', '-r', './resources', '-o', output_path, '-i', input_path, '-d' ] ret = main(test=True, testargs=testargs) self.assertEqual(ret, 0)
def test_invalid_missing_global(self): print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') config_path = 'tests/test_configs/invalid_config_no_global.cfg' config = ConfigObj(config_path) messages = verify_config(config) self.assertTrue('No global section found in config' in messages) self.assertTrue('No simulations section found in config' in messages)
def verify_remote_files(self, client, case): """ Check that the user supplied file paths are valid for remote files Parameters: client: either an ssh_client or a globus_client case: the case to check remote paths for """ if not self._config['global']['verify']: return True msg = 'verifying remote file paths' print_line(msg, self._event_list) data_types_to_verify = [] q = (DataFile .select() .where( (DataFile.case == case) & (DataFile.local_status != FileStatus.PRESENT.value))) for datafile in q.execute(): if datafile.datatype not in data_types_to_verify: data_types_to_verify.append(datafile.datatype) found_all = True for datatype in data_types_to_verify: q = (DataFile .select() .where( (DataFile.case == case) & (DataFile.datatype == datatype))) files = q.execute() remote_path, _ = os.path.split(files[0].remote_path) msg = 'Checking {} files in {}'.format(datatype, remote_path) print_line(msg, self._event_list) if files[0].transfer_type == 'globus': from lib.globus_interface import get_ls as globus_ls remote_contents = globus_ls( client=client, path=remote_path, endpoint=self._config['simulations'][case]['remote_uuid']) elif files[0].transfer_type == 'sftp': from lib.ssh_interface import get_ls as ssh_ls remote_contents = ssh_ls( client=client, remote_path=remote_path) remote_names = [x['name'] for x in remote_contents] for df in files: if df.name not in remote_names: msg = 'Unable to find file {name} at {remote_path}'.format( name=df.name, remote_path=remote_path) print_message(msg, 'error') found_all = False if not found_all: return False else: msg = 'found all remote files for {}'.format(case) print_message(msg, 'ok') return True
def test_AMWG_execute_completed(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') start_year = 1 end_year = 10 self.config['global'][ 'project_path'] = '/p/user_pub/e3sm/baldwin32/E3SM_test_data/DECKv1b_1pctCO2_complete' self.config['global'][ 'exeriment'] = '20180215.DECKv1b_1pctCO2.ne30_oEC.edison' set_string = '{start:04d}-{end:04d}'.format(start=start_year, end=end_year) web_directory = os.path.join( self.config.get('global').get('host_directory'), os.environ['USER'], self.config.get('global').get('experiment'), self.config.get('amwg').get('host_directory'), set_string) host_url = '/'.join([ self.config.get('global').get('img_host_server'), os.environ['USER'], self.config.get('global').get('experiment'), self.config.get('amwg').get('host_directory'), set_string ]) self.config['global']['output_path'] = os.path.join( self.config['global']['project_path'], 'output') self.config['global']['run_scripts_path'] = os.path.join( self.config['global']['project_path'], 'output', 'run_scripts') self.config['global']['resource_path'] = os.path.join( os.getcwd(), 'resources') regrid_path = os.path.join(self.config['global']['output_path'], 'pp', 'fv129x256', 'climo', '10yr') output_path = os.path.join(self.config['global']['output_path'], 'amwg_diag', set_string) template_path = os.path.join(self.config['global']['resource_path'], 'amwg_template.csh') diag_home = self.config['amwg']['diag_home'] temp_path = os.path.join(output_path, 'tmp', 'amwg', set_string) config = { 'web_dir': web_directory, 'host_url': host_url, 'experiment': self.config.get('global').get('experiment'), 'run_scripts_path': self.config['global']['run_scripts_path'], 'output_path': self.config['global']['output_path'], 'test_casename': self.config.get('global').get('experiment'), 'test_path_history': regrid_path + os.sep, 'regrided_climo_path': regrid_path + os.sep, 'test_path_climo': temp_path, 'test_path_diag': output_path, 'start_year': start_year, 'end_year': end_year, 'year_set': 1, 'run_directory': output_path, 'template_path': template_path, 'diag_home': diag_home } amwg = AMWGDiagnostic(config=config, event_list=EventList()) self.assertEqual(amwg.status.name, 'VALID') amwg.execute(dryrun=True) self.assertEqual(amwg.status.name, 'COMPLETED')
def test_shownode(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') slurm = Slurm() node = os.environ['HOSTNAME'].lower().split('.')[0] node_info = slurm.shownode(node) self.assertTrue(node_info['Arch'] == 'x86_64')
def test_render_bad_outout_file(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') render_target = os.path.join(os.getcwd(), 'tests', 'test_render_target.txt') render_output = '/usr/local/NO_PERMISSIONS' self.assertFalse(render({}, render_target, render_output))
def test_invalid_bad_data_type(self): print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') config_path = 'tests/test_configs/invalid_config_data_job_type.cfg' config = ConfigObj(config_path) messages = verify_config(config) self.assertTrue('atm has no file_format' in messages) self.assertTrue('atm has no remote_path' in messages) self.assertTrue('atm has no local_path' in messages)
def test_init_print_version(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') argv = ['-v'] with self.assertRaises(SystemExit) as exitexception: a, b, c = initialize(argv=argv, version=__version__) self.assertEqual(exitexception.exception.code, 0)
def test_init_no_config(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') argv = [] a, b, c = initialize(argv=argv) self.assertEqual(a, False) self.assertEqual(b, False) self.assertEqual(c, False)
def test_invalid_bad_regrid(self): print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') config_path = 'tests/test_configs/invalid_config_bad_regrid.cfg' config = ConfigObj(config_path) messages = verify_config(config) self.assertTrue('no source_grid_path given for lnd regrid' in messages) self.assertTrue('no destination_grid_path given for lnd regrid' in messages) self.assertTrue('no destination_grid_name given for lnd regrid' in messages) self.assertTrue('regrid is set to run on data_type ocn, but this type is not set in simulation 20180129.DECKv1b_piControl.ne30_oEC.edison' in messages)
def test_shownode(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') slurm = Slurm() node = 'acme1' node_info = slurm.shownode(node) self.assertTrue(node_info['Arch'] == 'x86_64') self.assertTrue(node_info['CoresPerSocket'] == '24')
def test_runmanager_setup(self): """ Run the runmanager setup """ print '\n'; print_message('---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') db_path = os.path.join( self.project_path, '{}.db'.format(inspect.stack()[0][3])) if os.path.exists(self.project_path): shutil.rmtree(self.project_path) os.makedirs(self.project_path) filemanager = FileManager( ui=False, event_list=EventList(), database=db_path, types=['atm'], sta=False, mutex=self.mutex, remote_endpoint=self.remote_endpoint, remote_path=self.remote_path, local_endpoint=self.local_endpoint, local_path=self.project_path, experiment=self.experiment) runmanager = RunManager( short_name='testname', account='', resource_path='./resources', ui=False, event_list=EventList(), output_path=self.output_path, caseID=self.experiment, scripts_path=self.run_scripts_path, thread_list=[], event=threading.Event(), no_host=True, url_prefix='', always_copy=False) runmanager.setup_job_sets( set_frequency=[5, 10], sim_start_year=int(self.config['global']['simulation_start_year']), sim_end_year=int(self.config['global']['simulation_end_year']), config=self.config, filemanager=filemanager) self.assertEqual(len(runmanager.job_sets), 1) for job_set in runmanager.job_sets: if job_set.set_number == 1: self.assertEqual(job_set.length, 5) self.assertEqual(job_set.set_start_year, 1) self.assertEqual(job_set.set_end_year, 5) job_names = job_set.get_job_names() self.assertTrue('ncclimo' in job_names) self.assertTrue('timeseries' in job_names) self.assertTrue('aprime_diags' in job_names) self.assertTrue('e3sm_diags' in job_names)
def get_ls(client, path, endpoint): for fail_count in xrange(10): try: res = globus_ls(client, path, endpoint, False, 0, False) except Exception as e: sleep(fail_count) print_message("Globus server error, retrying") print_debug(e) else: return res
def test_init_valid_config_simple(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') pargv = ['-c', 'tests/test_configs/valid_config_simple.cfg'] config, filemanager, runmanager = initialize( argv=pargv, version=__version__, branch=__branch__, event_list=EventList(), kill_event=threading.Event(), testing=True)
def test_event_list(self): print '\n' print_message( '---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok') elist = EventList() elist.push('beep boop', data=42) self.assertEquals(len(elist.list), 1) rmessage = 'boop beed' elist.replace(index=0, message=rmessage) self.assertEquals(elist.list[0].message, rmessage)