def main(ipppssoot, input_uri_prefix, output_uri_prefix): """Generates previews based on input and output directories according to specified args """ # set appropriate path variables logger = log.CaldpLogger(enable_console=False, log_file="preview.txt") cwd = os.getcwd() if input_uri_prefix.startswith("file"): in_path = input_uri_prefix.split(":")[-1] or "." else: in_path = ipppssoot input_dir = os.path.join(cwd, in_path) input_paths = get_inputs(ipppssoot, input_dir) output_path = process.get_output_path(output_uri_prefix, ipppssoot) + "/previews" # create previews previews = create_previews(input_dir, input_paths) # upload/copy previews if len(previews) > 0: if output_uri_prefix.startswith("s3"): log.info("Uploading previews...") upload_previews(previews, output_path) elif output_uri_prefix.startswith("file"): log.info("Saving previews...") os.makedirs(output_path, exist_ok=True) copy_previews(previews, output_path) else: return else: log.error("Error - Previews not generated.") del logger
def process(ipppssoot, input_uri, output_uri): """Given an `ipppssoot`, `input_uri`, and `output_uri` where products should be stored, perform all required processing steps for the `ipppssoot` and store all products to `output_uri`. Parameters ---------- ipppssoot : str The HST dataset name to be processed. output_uri : str The base path to which outputs will be copied, nominally S3://bucket/subdir/.../subdir input_uri : str either a local directory (path in the container) or astroquery to download from MAST Returns ------- None """ process_log = log.CaldpLogger(enable_console=False, log_file="process.txt") if output_uri is None: output_uri, output_path = messages.path_finder(input_uri, output_uri, ipppssoot) output_path = get_output_path(output_uri, ipppssoot) msg = messages.Messages(output_uri, output_path, ipppssoot) msg.init() msg.process_message() # processing-ipst manager = get_instrument_manager(ipppssoot, input_uri, output_uri) manager.main() del process_log
def main(ipppssoot, input_uri_prefix, output_uri_prefix): """Generates previews based on input and output directories according to specified args """ output_path = messages.get_local_outpath(output_uri_prefix, ipppssoot) msg = messages.Messages(output_uri_prefix, output_path, ipppssoot) msg.preview_message() # processing logger = log.CaldpLogger(enable_console=False, log_file="preview.txt") input_dir = file_ops.get_input_path(input_uri_prefix, ipppssoot) # append process.txt to trailer file # file_ops.append_trailer(input_dir, output_path, ipppssoot) input_paths = get_inputs(ipppssoot, input_dir) instr = process.get_instrument(ipppssoot) preview_inputs = get_preview_inputs(instr, input_paths) # create previews previews = create_previews(input_dir, preview_inputs) # upload/copy previews log.info("Saving previews...") if output_uri_prefix.startswith("s3"): preview_output = process.get_output_path("file:outputs", ipppssoot) + "/previews" os.makedirs(preview_output, exist_ok=True) copy_previews(previews, preview_output) log.info("Preparing files for s3 upload...") file_ops.tar_outputs(ipppssoot, input_uri_prefix, output_uri_prefix) elif output_uri_prefix.startswith("file"): preview_output = process.get_output_path(output_uri_prefix, ipppssoot) + "/previews" os.makedirs(preview_output, exist_ok=True) copy_previews(previews, preview_output) else: return del logger