def CDF_postprocessing(project):
task_exec(trigger_CDF)
transfer_events(project, 'CDF')
create_pp_pipelines(project, 'CDF')
start_worker()
time.sleep(time_to_wait_to_complete_postprocessing_jobs)
exec_wrapper('check_CDF_postprocessing_result_%s' % project)
def CDF_postprocessing(project):
task_exec(trigger_CDF)
transfer_events(project,'CDF')
create_pp_pipelines(project,'CDF')
start_worker()
time.sleep(time_to_wait_to_complete_postprocessing_jobs)
exec_wrapper('check_CDF_postprocessing_result_%s'%project)
def run():
# check env.
task_exec(tc.execute_basic_sdt_command)
task_exec(tc.execute_basic_sdp_command)
task_exec(tc.check_sdt_version)
task_exec(tc.check_sdp_version)
# download & IPSL pipeline (CMIP5)
prepare()
discovery('CMIP5')
download('CMIP5')
IPSL_postprocessing('CMIP5')
# download & IPSL pipeline (CORDEX)
prepare()
discovery('CORDEX')
download('CORDEX')
IPSL_postprocessing('CORDEX')
# download & IPSL pipeline & CDF pipeline (CMIP5)
prepare()
discovery('CMIP5')
download('CMIP5')
IPSL_postprocessing('CMIP5')
stop_worker()
CDF_postprocessing('CMIP5')
print 'Test complete successfully !'
def transfer_events(project,pipeline):
"""Transfer events from SDT to SDP."""
task_exec(tc.start_sdp)
task_exec(tc.enable_postprocessing)
task_exec(tc.restart_sdt)
time.sleep(time_to_wait_for_transferring_event) # give some time for pp events to be transfered from SDT to SDP
exec_wrapper("check_transfer_events_result_%s_%s"%(project,pipeline))
def transfer_events(project, pipeline):
"""Transfer events from SDT to SDP."""
task_exec(tc.start_sdp)
task_exec(tc.enable_postprocessing)
task_exec(tc.restart_sdt)
time.sleep(
time_to_wait_for_transferring_event
) # give some time for pp events to be transfered from SDT to SDP
exec_wrapper("check_transfer_events_result_%s_%s" % (project, pipeline))
def run():
# check env.
task_exec(tc.execute_basic_sdt_command)
task_exec(tc.execute_basic_sdp_command)
task_exec(tc.check_sdt_version)
task_exec(tc.check_sdp_version)
# download & IPSL pipeline (CMIP5)
prepare()
discovery('CMIP5')
download('CMIP5')
IPSL_postprocessing('CMIP5')
# download & IPSL pipeline (CORDEX)
prepare()
discovery('CORDEX')
download('CORDEX')
IPSL_postprocessing('CORDEX')
# download & IPSL pipeline & CDF pipeline (CMIP5)
prepare()
discovery('CMIP5')
download('CMIP5')
IPSL_postprocessing('CMIP5')
stop_worker()
CDF_postprocessing('CMIP5')
print 'Test complete successfully !'
def prepare():
# stop all daemons
task_exec(tc.stop_all)
time.sleep(time_to_wait_for_daemon_to_stop
) # give some time for daemons stop to be effective
task_exec(check_sa_result)
# configure
task_exec(tc.disable_eventthread)
task_exec(tc.set_pipeline_folder_path)
#task_exec(tc.set_ipsl_indexes)
# reset
task_exec(tc.reset_all)
# start sdp daemon
task_exec(tc.start_sdp)
time.sleep(10) # give some time for daemon start to be effective
# test sdt / sdp communication
task_exec(tc.test_sdt_sdp_communication)
# test sdw / sdp communication
task_exec(tc.test_sdw_sdp_communication)
# stop all daemons
task_exec(tc.stop_all)
time.sleep(time_to_wait_for_daemon_to_stop
) # give some time for daemons stop to be effective
task_exec(check_sa_result)
def start_worker():
task_exec(tc.start_sdw)
def stop_worker():
task_exec(tc.stop_sdw)
# check number of file retrieved from ESGF index during second discovery
fabric_run(
"test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | tail -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19566"
)
# init.
testset = Testset()
testset.selection_file = './resource/template/JT_T1_CMIP5.txt'
first_discovery_timestamp_right_boundary = '2012-03-19T01:00:00Z'
second_discovery_timestamp_right_boundary = '2012-03-25T01:00:00Z'
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('action', nargs='?')
args = parser.parse_args()
if args.action is None:
run()
elif args.action == 'reset':
task_exec(tc.stop)
task_exec(tc.reset_sdt)
task_exec(tc.disable_download)
task_exec(tc.do_not_print_domain_inconsistency)
task_exec(tc.set_dkrz_indexes)
sys.stderr.write('Ok\n')
def create_pp_pipelines(project,pipeline):
task_exec(tc.enable_eventthread)
task_exec(tc.restart_sdp)
time.sleep(time_to_wait_for_ppprun_creation) # give some time for ppprun to be created
exec_wrapper("check_ppprun_creation_result_%s_%s"%(project,pipeline))
def download(project):
task_exec(tc.enable_download)
task_exec(tc.start_sdt)
time.sleep(time_to_wait_for_download) # give some time for the file to be downloaded
exec_wrapper('check_download_result_%s'%project)
def run():
task_exec(tc.stop)
task_exec(tc.disable_download)
task_exec(tc.do_not_print_domain_inconsistency)
task_exec(tc.configure)
task_exec(tc.execute_basic_command)
task_exec(tc.check_version)
task_exec(tc.reset_sdt)
task_exec(tc.retrieve_parameters)
task_exec(full_discovery_step_1)
task_exec(check_fds1_result)
start_time = SDTimer.get_time()
task_exec(full_discovery_step_2)
elapsed_time = SDTimer.get_elapsed_time(start_time)
task_exec(check_fds2_result)
task_exec(check_that_full_discovery_fetched_all_metadata)
print 'Full discovery took %d seconds to complete' % elapsed_time
print 'Test complete successfully !'
def run():
task_exec(tc.stop)
task_exec(tc.disable_download)
task_exec(tc.do_not_print_domain_inconsistency)
task_exec(tc.configure)
task_exec(tc.execute_basic_command)
task_exec(tc.check_version)
task_exec(tc.reset_sdt)
task_exec(tc.retrieve_parameters)
task_exec(full_discovery_step_1)
task_exec(check_fds1_result)
start_time=SDTimer.get_time()
task_exec(full_discovery_step_2)
elapsed_time=SDTimer.get_elapsed_time(start_time)
task_exec(check_fds2_result)
task_exec(check_that_full_discovery_fetched_all_metadata)
print 'Full discovery took %d seconds to complete'%elapsed_time
print 'Test complete successfully !'
fabric_run('test ! -f /srv/synda/sdt/data/cmip5/output1/MPI-M/MPI-ESM-LR/decadal1995/mon/land/Lmon/r2i1p1/v20120529/baresoilFrac/baresoilFrac_Lmon_MPI-ESM-LR_decadal1995_r2i1p1_199601-200512.nc')
# init.
time_to_wait_for_download=300
#time_to_wait_for_download=35 # fake download mode
time_to_wait_for_transferring_event=20
time_to_wait_for_ppprun_creation=10
time_to_wait_to_complete_postprocessing_jobs=20
time_to_wait_for_daemon_to_stop=20
scripts_pp='./resource/scripts_pp'
if __name__=='__main__':
parser = argparse.ArgumentParser()
parser.add_argument('action',nargs='?')
args = parser.parse_args()
if args.action is None:
run()
elif args.action=='reset':
prepare()
task_exec(tc.enable_download)
task_exec(tc.enable_postprocessing)
task_exec(tc.enable_eventthread)
sys.stderr.write('Ok\n')
def prepare():
# stop all daemons
task_exec(tc.stop_all)
time.sleep(time_to_wait_for_daemon_to_stop) # give some time for daemons stop to be effective
task_exec(check_sa_result)
# configure
task_exec(tc.disable_eventthread)
task_exec(tc.set_pipeline_folder_path)
#task_exec(tc.set_ipsl_indexes)
# reset
task_exec(tc.reset_all)
# start sdp daemon
task_exec(tc.start_sdp)
time.sleep(10) # give some time for daemon start to be effective
# test sdt / sdp communication
task_exec(tc.test_sdt_sdp_communication)
# test sdw / sdp communication
task_exec(tc.test_sdw_sdp_communication)
# stop all daemons
task_exec(tc.stop_all)
time.sleep(time_to_wait_for_daemon_to_stop) # give some time for daemons stop to be effective
task_exec(check_sa_result)
# check number of file retrieved from ESGF index during first discovery
fabric_run("test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | head -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19525")
# check number of file retrieved from ESGF index during second discovery
fabric_run("test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | tail -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 98")
# init.
testset=Testset()
testset.selection_file='./resource/template/JT_T1_CMIP5.txt'
first_discovery_timestamp_right_boundary='2012-03-19T01:00:00Z'
second_discovery_timestamp_right_boundary='2012-03-25T01:00:00Z'
if __name__=='__main__':
parser = argparse.ArgumentParser()
parser.add_argument('action',nargs='?')
args = parser.parse_args()
if args.action is None:
run()
elif args.action=='reset':
task_exec(tc.stop)
task_exec(tc.reset_sdt)
task_exec(tc.disable_download)
task_exec(tc.do_not_print_domain_inconsistency)
task_exec(tc.set_dkrz_indexes)
sys.stderr.write('Ok\n')
def stop_worker():
task_exec(tc.stop_sdw)
def start_worker():
task_exec(tc.start_sdw)
def create_pp_pipelines(project, pipeline):
task_exec(tc.enable_eventthread)
task_exec(tc.restart_sdp)
time.sleep(time_to_wait_for_ppprun_creation
) # give some time for ppprun to be created
exec_wrapper("check_ppprun_creation_result_%s_%s" % (project, pipeline))
# init.
time_to_wait_for_download = 300
#time_to_wait_for_download=35 # fake download mode
time_to_wait_for_transferring_event = 20
time_to_wait_for_ppprun_creation = 10
time_to_wait_to_complete_postprocessing_jobs = 20
time_to_wait_for_daemon_to_stop = 20
scripts_pp = './resource/scripts_pp'
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('action', nargs='?')
args = parser.parse_args()
if args.action is None:
run()
elif args.action == 'reset':
prepare()
task_exec(tc.enable_download)
task_exec(tc.enable_postprocessing)
task_exec(tc.enable_eventthread)
sys.stderr.write('Ok\n')
def download(project):
task_exec(tc.enable_download)
task_exec(tc.start_sdt)
time.sleep(time_to_wait_for_download
) # give some time for the file to be downloaded
exec_wrapper('check_download_result_%s' % project)
def exec_wrapper(name):
fu=globals()[name]
task_exec(fu)
def exec_wrapper(name):
fu = globals()[name]
task_exec(fu)
def run():
task_exec(tc.stop)
task_exec(tc.disable_download)
task_exec(tc.do_not_print_domain_inconsistency)
task_exec(tc.configure)
task_exec(tc.execute_basic_command)
task_exec(tc.check_version)
task_exec(tc.reset_sdt)
task_exec(tc.retrieve_parameters)
task_exec(normal_discovery)
task_exec(check_normal_discovery_result)
start_time=SDTimer.get_time()
task_exec(incremental_discovery)
elapsed_time=SDTimer.get_elapsed_time(start_time)
task_exec(check_incremental_discovery_result)
task_exec(check_that_incremental_discovery_fetched_only_the_delta)
print 'Incremental discovery took %d seconds to complete'%elapsed_time
print 'Test complete successfully !'
def run():
task_exec(tc.stop)
task_exec(tc.disable_download)
task_exec(tc.do_not_print_domain_inconsistency)
task_exec(tc.configure)
task_exec(tc.execute_basic_command)
task_exec(tc.check_version)
task_exec(tc.reset_sdt)
task_exec(tc.retrieve_parameters)
task_exec(normal_discovery)
task_exec(check_normal_discovery_result)
start_time = SDTimer.get_time()
task_exec(incremental_discovery)
elapsed_time = SDTimer.get_elapsed_time(start_time)
task_exec(check_incremental_discovery_result)
task_exec(check_that_incremental_discovery_fetched_only_the_delta)
print 'Incremental discovery took %d seconds to complete' % elapsed_time
print 'Test complete successfully !'