def get_local_outpath(output_uri, ipppssoot):
"""Returns full path to folder containing output files."""
if output_uri.startswith("s3"):
local_outpath = process.get_output_path("file:outputs", ipppssoot)
else:
local_outpath = process.get_output_path(output_uri, ipppssoot)
return local_outpath
def main(ipppssoot, input_uri_prefix, output_uri_prefix):
"""Generates previews based on input and output directories
according to specified args
"""
output_path = messages.get_local_outpath(output_uri_prefix, ipppssoot)
msg = messages.Messages(output_uri_prefix, output_path, ipppssoot)
msg.preview_message() # processing
logger = log.CaldpLogger(enable_console=False, log_file="preview.txt")
input_dir = file_ops.get_input_path(input_uri_prefix, ipppssoot)
# append process.txt to trailer file
# file_ops.append_trailer(input_dir, output_path, ipppssoot)
input_paths = get_inputs(ipppssoot, input_dir)
instr = process.get_instrument(ipppssoot)
preview_inputs = get_preview_inputs(instr, input_paths)
# create previews
previews = create_previews(input_dir, preview_inputs)
# upload/copy previews
log.info("Saving previews...")
if output_uri_prefix.startswith("s3"):
preview_output = process.get_output_path("file:outputs",
ipppssoot) + "/previews"
os.makedirs(preview_output, exist_ok=True)
copy_previews(previews, preview_output)
log.info("Preparing files for s3 upload...")
file_ops.tar_outputs(ipppssoot, input_uri_prefix, output_uri_prefix)
elif output_uri_prefix.startswith("file"):
preview_output = process.get_output_path(output_uri_prefix,
ipppssoot) + "/previews"
os.makedirs(preview_output, exist_ok=True)
copy_previews(previews, preview_output)
else:
return
del logger
def main(ipppssoot, input_uri_prefix, output_uri_prefix):
"""Generates previews based on input and output directories
according to specified args
"""
# set appropriate path variables
logger = log.CaldpLogger(enable_console=False, log_file="preview.txt")
cwd = os.getcwd()
if input_uri_prefix.startswith("file"):
in_path = input_uri_prefix.split(":")[-1] or "."
else:
in_path = ipppssoot
input_dir = os.path.join(cwd, in_path)
input_paths = get_inputs(ipppssoot, input_dir)
output_path = process.get_output_path(output_uri_prefix, ipppssoot) + "/previews"
# create previews
previews = create_previews(input_dir, input_paths)
# upload/copy previews
if len(previews) > 0:
if output_uri_prefix.startswith("s3"):
log.info("Uploading previews...")
upload_previews(previews, output_path)
elif output_uri_prefix.startswith("file"):
log.info("Saving previews...")
os.makedirs(output_path, exist_ok=True)
copy_previews(previews, output_path)
else:
return
else:
log.error("Error - Previews not generated.")
del logger
def list_outputs(ipppssoot, output_uri):
"""Routinely log input, output, and CWD files to aid setting up expected results."""
# List files from all modes, they'll be nothing for inapplicable modes.
# Choose files from print to define truth values for future tests.
output_path = process.get_output_path(output_uri, ipppssoot)
if output_uri.startswith("file"):
outputs = list_files(output_path, ipppssoot)
elif CALDP_S3_TEST_OUTPUTS and output_uri.lower().startswith("s3"):
outputs = list_objects(output_path)
return outputs
def tar_outputs(ipppssoot, output_uri):
working_dir = os.getcwd()
output_path = process.get_output_path(output_uri, ipppssoot)
output_dir = get_output_dir(output_uri)
os.chdir(output_dir) # create tarfile with ipst/*fits (ipst is parent dir)
file_list = find_files(ipppssoot)
tar = make_tar(file_list, ipppssoot)
upload_tar(tar, output_path)
os.chdir(working_dir)
clean_up(file_list, ipppssoot, dirs=["previews", "env", "logs"])
if output_uri.startswith("file"): # test cov only
return tar, file_list # , local_outpath
def path_finder(input_uri, output_uri_prefix, ipppssoot):
if output_uri_prefix.lower().startswith("none"):
if input_uri.startswith("file"):
output_uri = input_uri
output_dir = output_uri.split(":")[-1] or "."
output_path = os.path.abspath(output_dir)
else:
output_dir = os.path.join(os.getcwd(), ipppssoot)
output_uri = f"file:{output_dir}"
output_path = os.path.abspath(output_dir)
else:
output_uri = output_uri_prefix
output_path = process.get_output_path(output_uri, ipppssoot)
return output_uri, output_path
def tar_outputs(ipppssoot, input_uri, output_uri):
working_dir = os.getcwd()
output_path = process.get_output_path(output_uri, ipppssoot)
output_dir = get_output_dir(output_uri)
os.chdir(output_dir) # create tarfile with ipst/*fits (ipst is parent dir)
output_files = find_output_files(ipppssoot)
if len(output_files) == 0:
log.info("No output files found. Tarring inputs for debugging.")
os.chdir(working_dir)
input_dir = get_input_dir(input_uri)
os.chdir(input_dir)
file_list = find_input_files(ipppssoot)
else:
file_list = find_previews(ipppssoot, output_files)
tar = make_tar(file_list, ipppssoot)
upload_tar(tar, output_path)
clean_up(file_list, ipppssoot, dirs=["previews", "env"])
os.chdir(working_dir)
if output_uri.startswith("file"): # test cov only
return tar, file_list # , local_outpath
