def test_init(self) -> None:
# test length
with self.assertRaises(Exception):
HstFilename("123456")
# test instrument name
with self.assertRaises(Exception):
HstFilename("e123456")
HstFilename("I123456") # case-less
def _copy_fits_files(self, bundle_segment: str, mast_downloads_dir: str,
primary_files_dir: str) -> None:
if not os.path.isdir(mast_downloads_dir):
raise ValueError(f"{mast_downloads_dir} doesn't exist.")
try:
PDS_LOGGER.open("Copy fits files to corresponding directories")
with make_osfs(
mast_downloads_dir) as mast_downloads_fs, make_sv_osfs(
primary_files_dir) as primary_files_fs:
# Walk the mast_downloads_dir for FITS file and file
# them into the COW filesystem.
for filepath in mast_downloads_fs.walk.files(
filter=["*.fits"]):
parts = fs.path.iteratepath(filepath)
depth = len(parts)
if depth != 3:
raise ValueError(f"{parts} length is not 3.")
# New way: product name comes from the filename
_, _, filename = parts
filename = filename.lower()
hst_filename = HstFilename(filename)
product = hst_filename.rootname()
instrument_name = hst_filename.instrument_name()
suffix = hst_filename.suffix()
collection_type = get_collection_type(
suffix=suffix, instrument_id=instrument_name)
coll = f"{collection_type}_{instrument_name.lower()}_{suffix}"
new_path = fs.path.join(
to_segment_dir(bundle_segment),
to_segment_dir(coll),
to_segment_dir(product),
filename,
)
dirs, filename = fs.path.split(new_path)
primary_files_fs.makedirs(dirs)
PDS_LOGGER.log("info", f"Copy {filename} to {new_path}")
fs.copy.copy_file(mast_downloads_fs, filepath,
primary_files_fs, new_path)
if not os.path.isdir(primary_files_dir + "-sv"):
raise ValueError(f"{primary_files_dir + '-sv'} doesn't exist.")
# # If I made it to here, it should be safe to delete the downloads
# shutil.rmtree(mast_downloads_dir)
# assert not os.path.isdir(mast_downloads_dir)
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()
def short_lidvid_to_dirpath(lidvid: LIDVID) -> str:
lid = lidvid.lid()
# parts are collection, product
parts = lid.parts()[1:]
if len(parts) >= 2 and parts[0] not in NO_VISIT_COLLECTIONS:
fake_filename = f"{parts[1]}_raw.fits"
visit = HstFilename(fake_filename).visit()
visit_part = f"visit_{visit}"
parts[1] = visit_part
return fs.path.join(*parts)
def _populate_hdus_associations_and_cards(
db: BundleDB, pyfits_obj: _PYFITS_OBJ, file_basename: str, fits_product_lidvid: str
) -> None:
def create_hdu_dict(index: int, hdu: _PYFITS_HDU) -> Dict[str, Any]:
fileinfo = hdu.fileinfo()
return {
"product_lidvid": fits_product_lidvid,
"hdu_index": index,
"hdr_loc": fileinfo["hdrLoc"],
"dat_loc": fileinfo["datLoc"],
"dat_span": fileinfo["datSpan"],
}
hdu_dicts = [create_hdu_dict(index, hdu) for index, hdu in enumerate(pyfits_obj)]
db.session.bulk_insert_mappings(Hdu, hdu_dicts)
def handle_undefined(val: bool) -> Optional[bool]:
"""Convert undefined values to None"""
if isinstance(val, astropy.io.fits.card.Undefined):
return None
else:
return val
def desired_keyword(kw: str) -> bool:
"""Return True if the keyword is wanted"""
# For now we want all of them.
return kw is not None
def create_card_dict(
hdu_index: int, card_index: int, card: _PYFITS_CARD
) -> Dict[str, Any]:
return {
"product_lidvid": fits_product_lidvid,
"hdu_index": hdu_index,
"card_index": card_index,
"keyword": card.keyword,
"value": handle_undefined(card.value),
}
card_dicts = [
create_card_dict(hdu_index, card_index, card)
for hdu_index, hdu in enumerate(pyfits_obj)
for card_index, card in enumerate(hdu.header.cards)
]
db.session.bulk_insert_mappings(Card, card_dicts)
if HstFilename(file_basename).suffix() == "asn":
_populate_associations(db, fits_product_lidvid, pyfits_obj)
db.session.commit()
def translate_filepath(archive_filepath: str) -> str:
"""
Translate an archive-style filepath (with "$" after bundle,
collection and product directory names) into a deliverable-style
filepath (with no "$"s and products organized into visit_NN
directories instead of their own).
"""
# The first part is always "/", so we drop it.
parts = fs.path.parts(archive_filepath)[1:]
# Special case for the root.
if len(parts) == 0:
return "/"
# The path should consist of a sequence of parts all ending in
# "$"--those show PDS4 structure--then the rest should *not* end
# in "$": those make up the full path within the PDS4 bundle,
# collection or product. (Currently, we don't use subdirectories
# within PDS4 objects, but it *is* allowed by the PDS4
# specification.)
indx = _first_index(lambda part: part[-1] != "$", parts)
# Take the parts that end in "$", then drop the "$"s.
pds4 = [part[:-1] for part in parts[:indx]]
# Take the remaining parts that don't end in "$".
non_pds4 = parts[indx:]
if len(pds4) in [1, 2]:
return fs.path.join("/", *pds4, *non_pds4)
if len(pds4) == 3:
collection_segment = pds4[1]
if collection_segment in NO_VISIT_COLLECTIONS:
return fs.path.join("/", *pds4, *non_pds4)
else:
product_segment = pds4[2]
fake_filename = f"{product_segment}_raw.fits"
visit = HstFilename(fake_filename).visit()
return fs.path.join("/", *pds4[0:2], f"visit_{visit}", *non_pds4)
raise ValueError(f"unexpected number of pds4 parts in {archive_filepath}")
def test_visit(self) -> None:
hst = HstFilename(_TEST_FILENAME)
self.assertEqual("01", hst.visit())
def test_hst_internal_proposal_id(self) -> None:
hst = HstFilename(_TEST_FILENAME)
self.assertEqual("6gp", hst.hst_internal_proposal_id())
def test_rootname(self) -> None:
hst = HstFilename(_TEST_FILENAME)
self.assertEqual("j6gp01mmq", hst.rootname())
def test_instrument_name(self) -> None:
hst = HstFilename(_TEST_FILENAME)
self.assertEqual("ACS", hst.instrument_name())
def test_repr(self) -> None:
hst = HstFilename(_TEST_FILENAME)
self.assertEqual("HstFilename('" + _TEST_FILENAME + "')", repr(hst))
def test_str(self) -> None:
hst = HstFilename(_TEST_FILENAME)
self.assertEqual(_TEST_FILENAME, hst.__str__())
def get_instr(dir: str) -> str:
for root, dirs, files in os.walk(dir):
for file in files:
if os.path.splitext(file)[1] == ".fits":
return HstFilename(file).instrument_name()
assert False, "didn't find any FITS files"