def test_finilize_complete(self):
pargv = ['-c', 'tests/test_configs/valid_config_simple.cfg']
event_list = EventList()
kill_event = threading.Event()
config, filemanager, runmanager = initialize(argv=pargv,
version="0.0.0",
branch="000",
event_list=event_list,
kill_event=kill_event,
mutex=threading.Lock(),
testing=True)
self.assertNotEqual(config, False)
self.assertNotEqual(filemanager, False)
self.assertNotEqual(runmanager, False)
# all jobs should be complete already
# this will mark them as such
runmanager.check_data_ready()
runmanager.start_ready_jobs()
runmanager.monitor_running_jobs()
finalize(config=config,
event_list=event_list,
kill_event=kill_event,
status=runmanager.is_all_done(),
runmanager=runmanager)
self.assertTrue(runmanager.is_all_done())
def test_amwg_execution_completed_job(self):
"""
test that when run on a completed set of jobs, amwg recognizes that the run has already
taken place and doesnt start again
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
_args = ['-c', self.config_path, '-r', 'resources/']
config, filemanager, runmanager = initialize(
argv=_args,
version="2.0.0",
branch="testing",
event_list=self.event_list,
kill_event=Event(),
testing=True)
config['global']['dryrun'] = True
runmanager.check_data_ready()
runmanager.start_ready_jobs()
for case in runmanager.cases:
for job in case['jobs']:
if job.job_type == 'amwg':
job.setup_data(
config=config,
filemanager=filemanager,
case='20180129.DECKv1b_piControl.ne30_oEC.edison')
job.execute(config=config, dryrun=True)
self.assertEquals(job.status, JobStatus.COMPLETED)
def test_init_from_scratch_config_globus_bad_uuid(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = [
'-c',
'tests/test_configs/valid_config_from_scratch_globus_bad_uuid.cfg'
]
project_path = '/p/user_pub/e3sm/baldwin32/testing/empty/'
if os.path.exists(project_path):
shutil.rmtree(project_path)
config, filemanager, runmanager = initialize(
argv=pargv,
version=__version__,
branch=__branch__,
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
self.assertEqual(config, False)
self.assertEqual(filemanager, False)
self.assertEqual(runmanager, False)
self.assertEqual(os.path.exists(project_path), True)
if os.path.exists(project_path):
shutil.rmtree(project_path)
def test_e3sm_diags_prevalidate(self):
"""
test that the e3sm_diags prevalidate and prerun setup works correctly
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
_args = ['-c', self.config_path, '-r', 'resources/']
config, filemanager, runmanager = initialize(
argv=_args,
version="2.0.0",
branch="testing",
event_list=self.event_list,
kill_event=Event(),
testing=True)
self.assertFalse(config is None)
self.assertFalse(filemanager is None)
self.assertFalse(runmanager is None)
config['global']['dryrun'] = True
runmanager.check_data_ready()
runmanager.start_ready_jobs()
for case in runmanager.cases:
for job in case['jobs']:
if job.job_type == 'e3sm_diags':
job.setup_data(
config=config,
filemanager=filemanager,
case='20180129.DECKv1b_piControl.ne30_oEC.edison')
job.execute(config=config, dryrun=True)
self.assertEquals(job.status, JobStatus.COMPLETED)
def test_init_print_version(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
argv = ['-v']
with self.assertRaises(SystemExit) as exitexception:
a, b, c = initialize(argv=argv, version=__version__)
self.assertEqual(exitexception.exception.code, 0)
def test_init_no_config(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
argv = []
a, b, c = initialize(argv=argv)
self.assertEqual(a, False)
self.assertEqual(b, False)
self.assertEqual(c, False)
def test_init_valid_config_simple(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = ['-c', 'tests/test_configs/valid_config_simple.cfg']
config, filemanager, runmanager = initialize(
argv=pargv,
version=__version__,
branch=__branch__,
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
def test_init_missing_lnd(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = ['-c', 'tests/test_configs/invalid_config_missing_lnd.cfg']
config, filemanager, runmanager = initialize(
argv=pargv,
version=__version__,
branch=__branch__,
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
self.assertEqual(config, False)
self.assertEqual(filemanager, False)
self.assertEqual(runmanager, False)
def test_finilize_complete(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = ['-c', 'tests/test_configs/test_amwg_complete.cfg']
config, filemanager, runmanager = initialize(
argv=pargv,
version="2.0.0",
branch="master",
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
finalize(config=config,
event_list=self.event_list,
status=1,
runmanager=runmanager)
def test_amwg_prevalidate(self):
"""
test that the amwg execution (in dry run mode) words correctly
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
_args = ['-c', self.config_path]
config, filemanager, runmanager = initialize(
argv=_args,
version="2.0.0",
branch="testing",
event_list=self.event_list,
kill_event=Event(),
testing=True)
self.assertFalse(config is None)
self.assertFalse(filemanager is None)
self.assertFalse(runmanager is None)
def test_finilize_complete_marked_failed(self):
pargv = ['-c', 'tests/test_configs/valid_config_simple.cfg']
event_list = EventList()
kill_event = threading.Event()
config, filemanager, runmanager = initialize(argv=pargv,
version="0.0.0",
branch="000",
event_list=event_list,
kill_event=kill_event,
mutex=threading.Lock(),
testing=True)
self.assertNotEqual(config, False)
self.assertNotEqual(filemanager, False)
self.assertNotEqual(runmanager, False)
finalize(config=config,
event_list=event_list,
kill_event=kill_event,
status=-1,
runmanager=runmanager)
self.assertEqual(runmanager.is_all_done(), -1)
def test_init_from_scratch_config_bad_project_dir(self):
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
pargv = [
'-c',
'tests/test_configs/valid_config_from_scratch_bad_project_path.cfg'
]
project_path = '/usr/testing/'
with self.assertRaises(SystemExit) as exitexception:
config, filemanager, runmanager = initialize(
argv=pargv,
version=__version__,
branch=__branch__,
event_list=EventList(),
kill_event=threading.Event(),
testing=True)
self.assertEqual(config, False)
self.assertEqual(filemanager, False)
self.assertEqual(runmanager, False)
self.assertEqual(os.path.exists(project_path), False)
self.assertEqual(exitexception.exception.code, 1)
def test_amwg_setup(self):
"""
Test the amwg initialization and data setup works correctly
"""
print '\n'
print_message(
'---- Starting Test: {} ----'.format(inspect.stack()[0][3]), 'ok')
_args = ['-c', self.config_path]
config, filemanager, runmanager = initialize(
argv=_args,
version="2.0.0",
branch="testing",
event_list=self.event_list,
kill_event=Event(),
testing=True)
amwg = AMWG(short_name='piControl_testing',
case='20180129.DECKv1b_piControl.ne30_oEC.edison',
start=1,
end=2,
comparison='obs',
config=self.config)
self.assertTrue(isinstance(amwg, Diag))
def main(test=False, **kwargs):
"""
Processflow main
"""
# The master configuration object
config = {}
# An event to kill the threads on terminal exception
thread_kill_event = threading.Event()
mutex = threading.Lock()
# A flag to tell if we have all the data locally
all_data = False
all_data_remote = False
# get a globus client
client = get_client()
# Read in parameters from config
if test:
print '=========================================='
print '---- Processflow running in test mode ----'
print '=========================================='
_args = kwargs['testargs']
config, filemanager, runmanager = initialize(
argv=_args,
version=__version__,
branch=__branch__,
event_list=event_list,
kill_event=thread_kill_event,
mutex=mutex,
testing=True)
else:
config, filemanager, runmanager = initialize(
argv=sys.argv[1:],
version=__version__,
branch=__branch__,
event_list=event_list,
kill_event=thread_kill_event,
mutex=mutex)
# setup returned an error code
if isinstance(config, int):
print "Error in setup, exiting"
return -1
logging.info('Config setup complete')
debug = True if config['global'].get('debug') else False
msg = "Updating local file status"
print_line(line=msg, event_list=event_list)
filemanager.update_local_status()
all_data_local = filemanager.all_data_local()
if not all_data_local:
filemanager.transfer_needed(event_list=event_list,
event=thread_kill_event)
# msg = "Writing human readable state to file"
# print_line(msg, event_list)
# check if the case_scripts directory is present
# if its not, transfer it over
if config['global'].get('get_scripts'):
pass
# msg = 'transfering case_scripts from remote machine'
# print_line(
# line=msg,
# event_list=event_list)
# case_scripts_dir = os.path.join(
# config['global']['input_path'],
# 'case_scripts')
# if not os.path.exists(case_scripts_dir):
# logging.info(msg)
# src_path = os.path.join(
# config['global']['source_path'], 'case_scripts')
# while True:
# try:
# args = {
# 'source_endpoint': config['transfer']['source_endpoint'],
# 'destination_endpoint': config['transfer']['destination_endpoint'],
# 'src_path': src_path,
# 'dst_path': case_scripts_dir,
# 'event_list': event_list,
# 'event': thread_kill_event
# }
# thread = threading.Thread(
# target=transfer_directory,
# name='transfer_directory',
# kwargs=args)
# except:
# sleep(1)
# else:
# thread_list.append(thread)
# thread.start()
# break
# Main loop
printed = False
loop_delay = 10
state_path = os.path.join(config['global']['project_path'], 'output',
'state.txt')
try:
print "--------------------------"
print " Entering Main Loop "
print " Status file: {}".format(state_path)
print "--------------------------"
while True:
if not all_data_local:
if debug:
print_line(' -- Updating local status --', event_list)
if filemanager.update_local_status():
msg = filemanager.report_files_local()
print_line(msg, event_list)
filemanager.write_database()
all_data_local = filemanager.all_data_local()
if not all_data_local:
if debug:
print_line(' -- Additional data needed --', event_list)
filemanager.transfer_needed(event_list, thread_kill_event)
if debug: print_line(' -- checking data -- ', event_list)
runmanager.check_data_ready()
if debug: print_line(' -- starting ready jobs --', event_list)
runmanager.start_ready_jobs()
if debug: print_line(' -- monitoring running jobs --', event_list)
runmanager.monitor_running_jobs()
if debug: print_line(' -- writing out state -- ', event_list)
runmanager.write_job_sets(state_path)
status = runmanager.is_all_done()
# return -1 if still running
# return 0 if a jobset failed
# return 1 if all complete
if status >= 0:
msg = "Finishing up run"
print_line(msg, event_list)
printed = False
while not filemanager.all_data_local():
if not printed:
printed = True
msg = 'Jobs are complete, but additional data is being transfered'
print_line(msg, event_list)
filemanager.update_local_status()
if not filemanager.all_data_local():
filemanager.transfer_needed(event_list=event_list,
event=thread_kill_event)
sleep(10)
filemanager.write_database()
finalize(config=config,
event_list=event_list,
status=status,
kill_event=thread_kill_event,
runmanager=runmanager)
# SUCCESS EXIT
return 0
if debug: print_line(' -- sleeping', event_list)
sleep(loop_delay)
except KeyboardInterrupt as e:
print_message('\n----- KEYBOARD INTERRUPT -----')
runmanager.write_job_sets(state_path)
filemanager.terminate_transfers()
print_message('----- cleanup complete -----', 'ok')
except Exception as e:
print_message('----- AN UNEXPECTED EXCEPTION OCCURED -----')
print_debug(e)
runmanager.write_job_sets(state_path)
filemanager.terminate_transfers()
def main(test=False, **kwargs):
"""
Processflow main
Parameters:
test (bool): turns on test mode. Simply stops the logger from reloading itself, which
stops a crash when running from inside the test runner
kwargs (dict): when running in test mode, arguments are passed directly through the kwargs
which bypasses the argument parsing.
"""
# The master configuration object
config = {}
# An event to kill the threads on terminal exception
thread_kill_event = threading.Event()
# A flag to tell if we have all the data locally
all_data = False
all_data_remote = False
# get a globus client
client = get_client()
# Read in parameters from config
if test:
print '=========================================='
print '---- Processflow running in test mode ----'
print '=========================================='
_args = kwargs['testargs']
config, filemanager, runmanager = initialize(
argv=_args,
version=__version__,
branch=__branch__,
event_list=event_list,
kill_event=thread_kill_event,
testing=True)
else:
config, filemanager, runmanager = initialize(
argv=sys.argv[1:],
version=__version__,
branch=__branch__,
event_list=event_list,
kill_event=thread_kill_event)
# setup returned an error code
if isinstance(config, int):
print "Error in setup, exiting"
return -1
logging.info('Config setup complete')
debug = True if config['global'].get('debug') else False
msg = "Updating local file status"
print_line(line=msg, event_list=event_list)
filemanager.update_local_status()
all_data_local = filemanager.all_data_local()
if not all_data_local:
filemanager.transfer_needed(event_list=event_list,
event=thread_kill_event)
# Main loop
printed = False
loop_delay = 10
state_path = os.path.join(config['global']['project_path'], 'output',
'state.txt')
try:
print "--------------------------"
print " Entering Main Loop "
print " Status file: {}".format(state_path)
print "--------------------------"
while True:
if not all_data_local:
if debug:
print_line(' -- Updating local status --', event_list)
if filemanager.update_local_status():
msg = filemanager.report_files_local()
print_line(msg, event_list)
filemanager.write_database()
all_data_local = filemanager.all_data_local()
if not all_data_local:
if debug:
print_line(' -- Additional data needed --', event_list)
filemanager.transfer_needed(event_list, thread_kill_event)
if debug: print_line(' -- checking data -- ', event_list)
runmanager.check_data_ready()
if debug: print_line(' -- starting ready jobs --', event_list)
runmanager.start_ready_jobs()
if debug: print_line(' -- monitoring running jobs --', event_list)
runmanager.monitor_running_jobs()
if debug: print_line(' -- writing out state -- ', event_list)
runmanager.write_job_sets(state_path)
status = runmanager.is_all_done()
# return -1 if still running
# return 0 if a jobset failed
# return 1 if all complete
if status >= 0:
msg = "Finishing up run"
print_line(msg, event_list)
printed = False
while not filemanager.all_data_local():
if not printed:
printed = True
msg = 'Jobs are complete, but additional data is being transfered'
print_line(msg, event_list)
filemanager.update_local_status()
if not filemanager.all_data_local():
filemanager.transfer_needed(event_list=event_list,
event=thread_kill_event)
sleep(10)
filemanager.write_database()
finalize(config=config,
event_list=event_list,
status=status,
runmanager=runmanager)
# SUCCESS EXIT
return 0
if debug: print_line(' -- sleeping', event_list)
sleep(loop_delay)
except KeyboardInterrupt as e:
print_message('\n----- KEYBOARD INTERRUPT -----')
runmanager.write_job_sets(state_path)
filemanager.terminate_transfers()
print_message('----- cleanup complete -----', 'ok')
except Exception as e:
print_message('----- AN UNEXPECTED EXCEPTION OCCURED -----')
print_debug(e)
runmanager.write_job_sets(state_path)
filemanager.terminate_transfers()
