def test_init(self) -> None: # test length with self.assertRaises(Exception): HstFilename("123456") # test instrument name with self.assertRaises(Exception): HstFilename("e123456") HstFilename("I123456") # case-less
def _copy_fits_files(self, bundle_segment: str, mast_downloads_dir: str, primary_files_dir: str) -> None: if not os.path.isdir(mast_downloads_dir): raise ValueError(f"{mast_downloads_dir} doesn't exist.") try: PDS_LOGGER.open("Copy fits files to corresponding directories") with make_osfs( mast_downloads_dir) as mast_downloads_fs, make_sv_osfs( primary_files_dir) as primary_files_fs: # Walk the mast_downloads_dir for FITS file and file # them into the COW filesystem. for filepath in mast_downloads_fs.walk.files( filter=["*.fits"]): parts = fs.path.iteratepath(filepath) depth = len(parts) if depth != 3: raise ValueError(f"{parts} length is not 3.") # New way: product name comes from the filename _, _, filename = parts filename = filename.lower() hst_filename = HstFilename(filename) product = hst_filename.rootname() instrument_name = hst_filename.instrument_name() suffix = hst_filename.suffix() collection_type = get_collection_type( suffix=suffix, instrument_id=instrument_name) coll = f"{collection_type}_{instrument_name.lower()}_{suffix}" new_path = fs.path.join( to_segment_dir(bundle_segment), to_segment_dir(coll), to_segment_dir(product), filename, ) dirs, filename = fs.path.split(new_path) primary_files_fs.makedirs(dirs) PDS_LOGGER.log("info", f"Copy {filename} to {new_path}") fs.copy.copy_file(mast_downloads_fs, filepath, primary_files_fs, new_path) if not os.path.isdir(primary_files_dir + "-sv"): raise ValueError(f"{primary_files_dir + '-sv'} doesn't exist.") # # If I made it to here, it should be safe to delete the downloads # shutil.rmtree(mast_downloads_dir) # assert not os.path.isdir(mast_downloads_dir) except Exception as e: PDS_LOGGER.exception(e) finally: PDS_LOGGER.close()
def short_lidvid_to_dirpath(lidvid: LIDVID) -> str: lid = lidvid.lid() # parts are collection, product parts = lid.parts()[1:] if len(parts) >= 2 and parts[0] not in NO_VISIT_COLLECTIONS: fake_filename = f"{parts[1]}_raw.fits" visit = HstFilename(fake_filename).visit() visit_part = f"visit_{visit}" parts[1] = visit_part return fs.path.join(*parts)
def _populate_hdus_associations_and_cards( db: BundleDB, pyfits_obj: _PYFITS_OBJ, file_basename: str, fits_product_lidvid: str ) -> None: def create_hdu_dict(index: int, hdu: _PYFITS_HDU) -> Dict[str, Any]: fileinfo = hdu.fileinfo() return { "product_lidvid": fits_product_lidvid, "hdu_index": index, "hdr_loc": fileinfo["hdrLoc"], "dat_loc": fileinfo["datLoc"], "dat_span": fileinfo["datSpan"], } hdu_dicts = [create_hdu_dict(index, hdu) for index, hdu in enumerate(pyfits_obj)] db.session.bulk_insert_mappings(Hdu, hdu_dicts) def handle_undefined(val: bool) -> Optional[bool]: """Convert undefined values to None""" if isinstance(val, astropy.io.fits.card.Undefined): return None else: return val def desired_keyword(kw: str) -> bool: """Return True if the keyword is wanted""" # For now we want all of them. return kw is not None def create_card_dict( hdu_index: int, card_index: int, card: _PYFITS_CARD ) -> Dict[str, Any]: return { "product_lidvid": fits_product_lidvid, "hdu_index": hdu_index, "card_index": card_index, "keyword": card.keyword, "value": handle_undefined(card.value), } card_dicts = [ create_card_dict(hdu_index, card_index, card) for hdu_index, hdu in enumerate(pyfits_obj) for card_index, card in enumerate(hdu.header.cards) ] db.session.bulk_insert_mappings(Card, card_dicts) if HstFilename(file_basename).suffix() == "asn": _populate_associations(db, fits_product_lidvid, pyfits_obj) db.session.commit()
def translate_filepath(archive_filepath: str) -> str: """ Translate an archive-style filepath (with "$" after bundle, collection and product directory names) into a deliverable-style filepath (with no "$"s and products organized into visit_NN directories instead of their own). """ # The first part is always "/", so we drop it. parts = fs.path.parts(archive_filepath)[1:] # Special case for the root. if len(parts) == 0: return "/" # The path should consist of a sequence of parts all ending in # "$"--those show PDS4 structure--then the rest should *not* end # in "$": those make up the full path within the PDS4 bundle, # collection or product. (Currently, we don't use subdirectories # within PDS4 objects, but it *is* allowed by the PDS4 # specification.) indx = _first_index(lambda part: part[-1] != "$", parts) # Take the parts that end in "$", then drop the "$"s. pds4 = [part[:-1] for part in parts[:indx]] # Take the remaining parts that don't end in "$". non_pds4 = parts[indx:] if len(pds4) in [1, 2]: return fs.path.join("/", *pds4, *non_pds4) if len(pds4) == 3: collection_segment = pds4[1] if collection_segment in NO_VISIT_COLLECTIONS: return fs.path.join("/", *pds4, *non_pds4) else: product_segment = pds4[2] fake_filename = f"{product_segment}_raw.fits" visit = HstFilename(fake_filename).visit() return fs.path.join("/", *pds4[0:2], f"visit_{visit}", *non_pds4) raise ValueError(f"unexpected number of pds4 parts in {archive_filepath}")
def test_visit(self) -> None: hst = HstFilename(_TEST_FILENAME) self.assertEqual("01", hst.visit())
def test_hst_internal_proposal_id(self) -> None: hst = HstFilename(_TEST_FILENAME) self.assertEqual("6gp", hst.hst_internal_proposal_id())
def test_rootname(self) -> None: hst = HstFilename(_TEST_FILENAME) self.assertEqual("j6gp01mmq", hst.rootname())
def test_instrument_name(self) -> None: hst = HstFilename(_TEST_FILENAME) self.assertEqual("ACS", hst.instrument_name())
def test_repr(self) -> None: hst = HstFilename(_TEST_FILENAME) self.assertEqual("HstFilename('" + _TEST_FILENAME + "')", repr(hst))
def test_str(self) -> None: hst = HstFilename(_TEST_FILENAME) self.assertEqual(_TEST_FILENAME, hst.__str__())
def get_instr(dir: str) -> str: for root, dirs, files in os.walk(dir): for file in files: if os.path.splitext(file)[1] == ".fits": return HstFilename(file).instrument_name() assert False, "didn't find any FITS files"