def check_that_full_discovery_fetched_all_metadata(): # check number of file retrieved from ESGF index during first discovery fabric_run("test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | head -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19525") # check number of file retrieved from ESGF index during second discovery fabric_run("test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | tail -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19566")
def check_download_result_CORDEX(): # check that all files are done fabric_run('test $(synda list limit=0 -f | grep "^done" | wc -l) -eq 10') # check that corresponding events have been created fabric_run("""test $(sqlite3 /var/lib/synda/sdt/sdt.db "select * from event where status='new'" | wc -l) -eq 1""")
def check_download_result_CORDEX(): # check that all files are done fabric_run('test $(synda list limit=0 -f | grep "^done" | wc -l) -eq 10') # check that corresponding events have been created fabric_run( """test $(sqlite3 /var/lib/synda/sdt/sdt.db "select * from event where status='new'" | wc -l) -eq 1""" )
def check_that_full_discovery_fetched_all_metadata(): # check number of file retrieved from ESGF index during first discovery fabric_run( "test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | head -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19525" ) # check number of file retrieved from ESGF index during second discovery fabric_run( "test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | tail -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19566" )
def check_one(jeutest_path, num): success = 69 pfx = 'test %s' % num err_raw( "-------------------------------------------------------------------------------\n" ) err("%s: start" % pfx) in_fn = '%s/check_dataset_version-%s-in.json' % (jeutest_path, num) out_fn = '%s/check_dataset_version-%s-out.txt' % (jeutest_path, num) tmp_path = '/tmp' if 'TMPDIR' in os.environ and len(os.environ['TMPDIR']) != 0: tmp_path = os.environ['TMPDIR'] tmp_path += '/svcheckdatasetversion' tmp_fn = tmp_path + '/%s-out.txt' % num cmd_synda = 'synda check --playback "%s" dataset_version >"%s"' % (in_fn, tmp_fn) cmd = '''mkdir -p -- "%s" || exit 1 %s; true ''' % (tmp_path, cmd_synda) fabric_run(cmd) if 1: (differ, s) = check_dataset_version_outputs_compare(out_fn, tmp_fn) if differ: err('%s: the output of synda is NOT what it should be:' % pfx) err('%s: %s' % (pfx, s)) success = 0 else: err('%s: the output of synda is what it should be:' % pfx) err('%s: %s' % (pfx, s)) success = 1 else: err('%s: an error occurred while running:' % pfx) err(cmd.rstrip('\n')) success = 0 if success: err('%s: pass' % pfx) return 1 else: err('%s: FAIL' % pfx) raise Exception('Test %s failed' % num) return 0
def check_that_incremental_discovery_fetched_only_the_delta(): # be sure that a previous record was found in history table fabric_run('! grep -q "SYNUTILS-008 No previous run found" /var/log/synda/sdt/discovery.log') # check number of file retrieved from ESGF index during first discovery fabric_run("test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | head -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19525") # check number of file retrieved from ESGF index during second discovery fabric_run("test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | tail -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 98")
def check_that_incremental_discovery_fetched_only_the_delta(): # be sure that a previous record was found in history table fabric_run( '! grep -q "SYNUTILS-008 No previous run found" /var/log/synda/sdt/discovery.log' ) # check number of file retrieved from ESGF index during first discovery fabric_run( "test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | head -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19525" ) # check number of file retrieved from ESGF index during second discovery fabric_run( "test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | tail -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 98" )
def check_normal_discovery_result(): fabric_run('test $(synda list limit=0 -f | wc -l) -eq 19525')
def check_incremental_discovery_result(): fabric_run('test $(synda list limit=0 -f | wc -l) -eq 19566')
def full_discovery_step_2(): fabric_run('synda install -y --timestamp_right_boundary %s -s %s'%(second_discovery_timestamp_right_boundary,testset.selection_file,))
def incremental_discovery(): fabric_run('synda install -i -y --timestamp_right_boundary %s -s %s' % ( second_discovery_timestamp_right_boundary, testset.selection_file, ))
def install_CORDEX(): fabric_run('sudo synda install -y -s ./resource/template/JT_T3_CORDEX.txt')
def check_sa_result(): # sa stands for "Stop All" fabric_run('! pgrep spdaemon') fabric_run('! pgrep synda')
def fail(): fabric_run('false')
def check_install_result_CMIP5(): fabric_run('test $(synda list limit=0 -f | wc -l) -eq 4')
def check_fds2_result(): fabric_run('test $(synda list limit=0 -f | wc -l) -eq 19566')
def full_discovery_step_2(): fabric_run('synda install -y --timestamp_right_boundary %s -s %s' % ( second_discovery_timestamp_right_boundary, testset.selection_file, ))
def fake(): fabric_run( 'test ! -f /srv/synda/sdt/data/cmip5/output1/MPI-M/MPI-ESM-LR/decadal1995/mon/land/Lmon/r2i1p1/v20120529/baresoilFrac/baresoilFrac_Lmon_MPI-ESM-LR_decadal1995_r2i1p1_199601-200512.nc' )
def check_install_result_CORDEX(): fabric_run('test $(synda list limit=0 -f | wc -l) -eq 10')
def trigger_CDF(): fabric_run('sudo synda pexec cdf -s ./resource/template/JT_T3_CMIP5.txt')
def check_dataset_version(): fabric_run('synda help check') fabric_run('synda check --help')
def check_ppprun_creation_result_CMIP5_CDF(): fabric_run("""test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status in ('waiting')" | wc -l) -eq 3""")
def fake(): fabric_run('test ! -f /srv/synda/sdt/data/cmip5/output1/MPI-M/MPI-ESM-LR/decadal1995/mon/land/Lmon/r2i1p1/v20120529/baresoilFrac/baresoilFrac_Lmon_MPI-ESM-LR_decadal1995_r2i1p1_199601-200512.nc')
def check_ppprun_creation_result_CORDEX_IPSL(): fabric_run("""test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status in ('waiting','pause')" | wc -l) -eq 1""")
def check_transfer_events_result_CMIP5_CDF(): fabric_run("""test $(sqlite3 /var/lib/synda/sdt/sdt.db "select * from event where status='old'" | wc -l) -eq 18""")
def check_ppprun_creation_result_CORDEX_IPSL(): fabric_run( """test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status in ('waiting','pause')" | wc -l) -eq 1""" )
def check_CDF_postprocessing_result_CMIP5(): fabric_run("""test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status='done'" | wc -l) -eq 18""")
def normal_discovery(): fabric_run('synda install -y --timestamp_right_boundary %s -s %s'%(first_discovery_timestamp_right_boundary,testset.selection_file,))
def check_transfer_events_result_CMIP5_CDF(): fabric_run( """test $(sqlite3 /var/lib/synda/sdt/sdt.db "select * from event where status='old'" | wc -l) -eq 18""" )
def incremental_discovery(): fabric_run('synda install -i -y --timestamp_right_boundary %s -s %s'%(second_discovery_timestamp_right_boundary,testset.selection_file,))
def check_ppprun_creation_result_CMIP5_CDF(): fabric_run( """test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status in ('waiting')" | wc -l) -eq 3""" )
def check_CDF_postprocessing_result_CMIP5(): fabric_run( """test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status='done'" | wc -l) -eq 18""" )
def normal_discovery(): fabric_run('synda install -y --timestamp_right_boundary %s -s %s' % ( first_discovery_timestamp_right_boundary, testset.selection_file, ))