def upload_tar(tar, output_path):
with sysexit.exit_on_exception(exit_codes.S3_UPLOAD_ERROR,
"S3 tar upload of", tar, "to", output_path,
"FAILED."):
client = boto3.client("s3")
parts = output_path[5:].split("/")
bucket, prefix = parts[0], "/".join(parts[1:])
objectname = prefix + "/" + os.path.basename(tar)
log.info(f"Uploading: s3://{bucket}/{objectname}")
if output_path.startswith("s3"):
with open(tar, "rb") as f:
client.upload_fileobj(f,
bucket,
objectname,
Callback=ProgressPercentage(tar))
def download(self):
"""Called if input_uri starts is `astroquery`
Download any data files for the `ipppssoot`, issuing start and
stop divider messages.
Returns
-------
filepaths : sorted list
Local file system paths of files which were downloaded for `ipppssoot`,
some of which will be selected for calibration processing.
"""
with sysexit.exit_on_exception(
exit_codes.ASTROQUERY_ERROR, "Astroquery exception downloading suffixes:", self.download_suffixes
):
self.divider("Retrieving data files for:", self.download_suffixes)
files = retrieve_observation(self.ipppssoot, suffix=self.download_suffixes)
self.divider("Download data complete.")
return list(sorted([os.path.abspath(f) for f in files]))
def find_input_files(self):
"""Scrape the input_uri for the needed input_files.
Called if input_uri starts with `file:`
Returns
-------
filepaths : sorted list
Local file system paths of files which were found for `ipppssoot`,
some of which will be selected for calibration processing.
"""
test_path = self.input_uri.split(":")[-1]
if os.path.isdir(test_path):
base_path = os.path.abspath(test_path)
elif os.path.isdir(os.path.join(os.getcwd(), test_path)):
base_path = os.path.join(os.getcwd(), test_path)
else:
raise ValueError(f"input path {test_path} does not exist")
# check for tarred inputs
cwd = os.getcwd()
search_tar = f"{base_path}/{self.ipppssoot.lower()[0:5]}*.tar.gz"
tar_files = glob.glob(search_tar)
with sysexit.exit_on_exception(exit_codes.INPUT_TAR_FILE_ERROR,
"Failed extracting inputs from",
tar_files):
if len(tar_files) == 0:
raise RuntimeError(
f"No input tar files for: {repr(search_tar)}")
elif len(tar_files) == 1:
log.info("Extracting inputs from: ", tar_files)
os.chdir(base_path)
with tarfile.open(tar_files[0], "r:gz") as tar_ref:
tar_ref.extractall()
else:
raise RuntimeError(
f"Too many tar files for: {repr(search_tar)} = {tar_files}"
)
os.chdir(cwd)
# get input files
search_str = f"{base_path}/{self.ipppssoot.lower()[0:5]}*.fits"
self.divider("Finding input data using:", repr(search_str))
# find the base path to the files
files = glob.glob(search_str)
return list(sorted(files))