def check_that_full_discovery_fetched_all_metadata():
# check number of file retrieved from ESGF index during first discovery
fabric_run("test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | head -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19525")
# check number of file retrieved from ESGF index during second discovery
fabric_run("test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | tail -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19566")
def check_download_result_CORDEX():
# check that all files are done
fabric_run('test $(synda list limit=0 -f | grep "^done" | wc -l) -eq 10')
# check that corresponding events have been created
fabric_run("""test $(sqlite3 /var/lib/synda/sdt/sdt.db "select * from event where status='new'" | wc -l) -eq 1""")
def check_download_result_CORDEX():
# check that all files are done
fabric_run('test $(synda list limit=0 -f | grep "^done" | wc -l) -eq 10')
# check that corresponding events have been created
fabric_run(
"""test $(sqlite3 /var/lib/synda/sdt/sdt.db "select * from event where status='new'" | wc -l) -eq 1"""
)
def check_that_full_discovery_fetched_all_metadata():
# check number of file retrieved from ESGF index during first discovery
fabric_run(
"test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | head -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19525"
)
# check number of file retrieved from ESGF index during second discovery
fabric_run(
"test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | tail -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19566"
)
def check_one(jeutest_path, num):
success = 69
pfx = 'test %s' % num
err_raw(
"-------------------------------------------------------------------------------\n"
)
err("%s: start" % pfx)
in_fn = '%s/check_dataset_version-%s-in.json' % (jeutest_path, num)
out_fn = '%s/check_dataset_version-%s-out.txt' % (jeutest_path, num)
tmp_path = '/tmp'
if 'TMPDIR' in os.environ and len(os.environ['TMPDIR']) != 0:
tmp_path = os.environ['TMPDIR']
tmp_path += '/svcheckdatasetversion'
tmp_fn = tmp_path + '/%s-out.txt' % num
cmd_synda = 'synda check --playback "%s" dataset_version >"%s"' % (in_fn,
tmp_fn)
cmd = '''mkdir -p -- "%s" || exit 1
%s;
true
''' % (tmp_path, cmd_synda)
fabric_run(cmd)
if 1:
(differ, s) = check_dataset_version_outputs_compare(out_fn, tmp_fn)
if differ:
err('%s: the output of synda is NOT what it should be:' % pfx)
err('%s: %s' % (pfx, s))
success = 0
else:
err('%s: the output of synda is what it should be:' % pfx)
err('%s: %s' % (pfx, s))
success = 1
else:
err('%s: an error occurred while running:' % pfx)
err(cmd.rstrip('\n'))
success = 0
if success:
err('%s: pass' % pfx)
return 1
else:
err('%s: FAIL' % pfx)
raise Exception('Test %s failed' % num)
return 0
def check_that_incremental_discovery_fetched_only_the_delta():
# be sure that a previous record was found in history table
fabric_run('! grep -q "SYNUTILS-008 No previous run found" /var/log/synda/sdt/discovery.log')
# check number of file retrieved from ESGF index during first discovery
fabric_run("test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | head -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19525")
# check number of file retrieved from ESGF index during second discovery
fabric_run("test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | tail -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 98")
def check_that_incremental_discovery_fetched_only_the_delta():
# be sure that a previous record was found in history table
fabric_run(
'! grep -q "SYNUTILS-008 No previous run found" /var/log/synda/sdt/discovery.log'
)
# check number of file retrieved from ESGF index during first discovery
fabric_run(
"test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | head -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 19525"
)
# check number of file retrieved from ESGF index during second discovery
fabric_run(
"test $(grep SDSEARCH-584 /var/log/synda/sdt/discovery.log | tail -1 | sed 's/^.*(\(.*\) files)/\\1/') -eq 98"
)
def check_normal_discovery_result():
fabric_run('test $(synda list limit=0 -f | wc -l) -eq 19525')
def check_incremental_discovery_result():
fabric_run('test $(synda list limit=0 -f | wc -l) -eq 19566')
def full_discovery_step_2():
fabric_run('synda install -y --timestamp_right_boundary %s -s %s'%(second_discovery_timestamp_right_boundary,testset.selection_file,))
def incremental_discovery():
fabric_run('synda install -i -y --timestamp_right_boundary %s -s %s' % (
second_discovery_timestamp_right_boundary,
testset.selection_file,
))
def install_CORDEX():
fabric_run('sudo synda install -y -s ./resource/template/JT_T3_CORDEX.txt')
def check_sa_result(): # sa stands for "Stop All"
fabric_run('! pgrep spdaemon')
fabric_run('! pgrep synda')
def fail():
fabric_run('false')
def install_CORDEX():
fabric_run('sudo synda install -y -s ./resource/template/JT_T3_CORDEX.txt')
def check_install_result_CMIP5():
fabric_run('test $(synda list limit=0 -f | wc -l) -eq 4')
def check_sa_result(): # sa stands for "Stop All"
fabric_run('! pgrep spdaemon')
fabric_run('! pgrep synda')
def check_fds2_result():
fabric_run('test $(synda list limit=0 -f | wc -l) -eq 19566')
def full_discovery_step_2():
fabric_run('synda install -y --timestamp_right_boundary %s -s %s' % (
second_discovery_timestamp_right_boundary,
testset.selection_file,
))
def fake():
fabric_run(
'test ! -f /srv/synda/sdt/data/cmip5/output1/MPI-M/MPI-ESM-LR/decadal1995/mon/land/Lmon/r2i1p1/v20120529/baresoilFrac/baresoilFrac_Lmon_MPI-ESM-LR_decadal1995_r2i1p1_199601-200512.nc'
)
def check_install_result_CORDEX():
fabric_run('test $(synda list limit=0 -f | wc -l) -eq 10')
def check_incremental_discovery_result():
fabric_run('test $(synda list limit=0 -f | wc -l) -eq 19566')
def trigger_CDF():
fabric_run('sudo synda pexec cdf -s ./resource/template/JT_T3_CMIP5.txt')
def check_dataset_version():
fabric_run('synda help check')
fabric_run('synda check --help')
def check_ppprun_creation_result_CMIP5_CDF():
fabric_run("""test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status in ('waiting')" | wc -l) -eq 3""")
def check_install_result_CMIP5():
fabric_run('test $(synda list limit=0 -f | wc -l) -eq 4')
def fake():
fabric_run('test ! -f /srv/synda/sdt/data/cmip5/output1/MPI-M/MPI-ESM-LR/decadal1995/mon/land/Lmon/r2i1p1/v20120529/baresoilFrac/baresoilFrac_Lmon_MPI-ESM-LR_decadal1995_r2i1p1_199601-200512.nc')
def check_install_result_CORDEX():
fabric_run('test $(synda list limit=0 -f | wc -l) -eq 10')
def check_normal_discovery_result():
fabric_run('test $(synda list limit=0 -f | wc -l) -eq 19525')
def check_ppprun_creation_result_CORDEX_IPSL():
fabric_run("""test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status in ('waiting','pause')" | wc -l) -eq 1""")
def check_fds2_result():
fabric_run('test $(synda list limit=0 -f | wc -l) -eq 19566')
def check_transfer_events_result_CMIP5_CDF():
fabric_run("""test $(sqlite3 /var/lib/synda/sdt/sdt.db "select * from event where status='old'" | wc -l) -eq 18""")
def check_ppprun_creation_result_CORDEX_IPSL():
fabric_run(
"""test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status in ('waiting','pause')" | wc -l) -eq 1"""
)
def check_CDF_postprocessing_result_CMIP5():
fabric_run("""test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status='done'" | wc -l) -eq 18""")
def trigger_CDF():
fabric_run('sudo synda pexec cdf -s ./resource/template/JT_T3_CMIP5.txt')
def normal_discovery():
fabric_run('synda install -y --timestamp_right_boundary %s -s %s'%(first_discovery_timestamp_right_boundary,testset.selection_file,))
def check_transfer_events_result_CMIP5_CDF():
fabric_run(
"""test $(sqlite3 /var/lib/synda/sdt/sdt.db "select * from event where status='old'" | wc -l) -eq 18"""
)
def incremental_discovery():
fabric_run('synda install -i -y --timestamp_right_boundary %s -s %s'%(second_discovery_timestamp_right_boundary,testset.selection_file,))
def check_ppprun_creation_result_CMIP5_CDF():
fabric_run(
"""test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status in ('waiting')" | wc -l) -eq 3"""
)
def check_CDF_postprocessing_result_CMIP5():
fabric_run(
"""test $(sqlite3 /var/lib/synda/sdp/sdp.db "select * from ppprun where status='done'" | wc -l) -eq 18"""
)
def normal_discovery():
fabric_run('synda install -y --timestamp_right_boundary %s -s %s' % (
first_discovery_timestamp_right_boundary,
testset.selection_file,
))
