def coretst(temp_dir, ipppssoot, input_uri, output_uri):
"""Run every `ipppssoot` through process.process() downloading input files from
astroquery and writing output files to local storage. Verify that call to process
outputs the expected files with reasonable sizes into it's CWD.
"""
print("Test case", ipppssoot, input_uri, output_uri)
if not input_uri or not output_uri:
print("Skipping case", ipppssoot, input_uri, output_uri)
return
working_dir = os.path.join(temp_dir, ipppssoot)
os.makedirs(working_dir, exist_ok=True)
os.chdir(working_dir)
try:
if input_uri.startswith("file"):
setup_io(ipppssoot, input_uri, output_uri)
process.process(ipppssoot, input_uri, output_uri)
messages.log_metrics(log_file="process.txt",
metrics="process_metrics.txt")
create_previews.main(ipppssoot, input_uri, output_uri)
messages.log_metrics(log_file="preview.txt",
metrics="preview_metrics.txt")
expected_inputs, expected_outputs = expected(RESULTS, ipppssoot)
actual_inputs = list_inputs(ipppssoot, input_uri)
actual_outputs = list_outputs(ipppssoot, output_uri)
check_inputs(input_uri, expected_inputs, actual_inputs)
check_outputs(output_uri, expected_outputs, actual_outputs)
messages.main(input_uri, output_uri, ipppssoot)
check_logs(input_uri, output_uri, ipppssoot)
check_messages(ipppssoot, output_uri)
finally:
os.chdir(temp_dir)
def coretst(temp_dir, ipppssoot, input_uri, output_uri):
"""Run every `ipppssoot` through process.process() downloading input files from
astroquery and writing output files to local storage. Verify that call to process
outputs the expected files with reasonable sizes into it's CWD.
"""
print("Test case", ipppssoot, input_uri, output_uri)
if not input_uri or not output_uri:
print("Skipping case", ipppssoot, input_uri, output_uri)
return
working_dir = os.path.join(temp_dir, ipppssoot)
os.makedirs(working_dir, exist_ok=True)
os.chdir(working_dir)
try:
if input_uri.startswith("file"):
setup_io(ipppssoot, input_uri, output_uri)
tarball = check_tarball_in(ipppssoot)
process.process(ipppssoot, input_uri, output_uri)
messages.log_metrics(log_file="process.txt",
metrics="process_metrics.txt")
check_messages(ipppssoot, output_uri, status="processing")
create_previews.main(ipppssoot, input_uri, output_uri)
messages.log_metrics(log_file="preview.txt",
metrics="preview_metrics.txt")
expected_inputs, expected_outputs = expected(RESULTS, ipppssoot)
actual_inputs = list_inputs(ipppssoot, input_uri)
actual_outputs = list_outputs(ipppssoot, output_uri)
check_inputs(input_uri, expected_inputs, actual_inputs)
check_outputs(output_uri, expected_outputs, actual_outputs)
messages.main(input_uri, output_uri, ipppssoot)
check_s3_outputs(S3_OUTPUTS, actual_outputs, ipppssoot, output_uri)
check_logs(input_uri, output_uri, ipppssoot)
check_messages(ipppssoot, output_uri, status="processed.trigger")
# tests whether file_ops gracefully handles an exception type
file_ops.clean_up([], ipppssoot, dirs=["dummy_dir"])
if input_uri.startswith(
"file"): # create tarfile if s3 access unavailable
actual_tarfiles = check_tarball_out(ipppssoot, input_uri,
output_uri)
check_tarfiles(TARFILES, actual_tarfiles, ipppssoot, output_uri)
check_pathfinder(ipppssoot)
message_status_check(input_uri, output_uri, ipppssoot)
os.remove(tarball)
# check output tarball for failed jobs
check_failed_job_tarball(ipppssoot, input_uri, output_uri)
check_messages_cleanup(ipppssoot)
if input_uri.startswith("astroquery"):
check_IO_clean_up(ipppssoot)
check_sysexit_retry()
finally:
os.chdir(temp_dir)