def plain_lidvid_to_visits_dirpath(lidvid: LIDVID) -> str: # This is only run on directories, and always relative to /hst_NNNNN. lid = lidvid.lid() parts = lid.parts()[1:] if len(parts) == 2: visit = _visit_of(parts[1]) if visit is not None: parts[1] = visit return fs.path.join("/", *parts)
def _munge_lidvid(product_lidvid: str, suffix: str, new_basename: str) -> str: bundle_id, collection_id, product_id = LIDVID(product_lidvid).lid().parts() # TODO This is a hack collection_type = get_collection_type(suffix=suffix) first_underscore_idx = collection_id.index("_") new_collection_id = (collection_type + collection_id[first_underscore_idx:-3] + suffix.lower()) # TODO This is a hack new_product_id = new_basename[0:9] new_lid = LID.create_from_parts( [bundle_id, new_collection_id, new_product_id]) # TODO This is a hack. Fix it. vid = VID("1.0") new_lidvid = LIDVID.create_from_lid_and_vid(new_lid, vid) return str(new_lidvid)
def short_lidvid_to_dirpath(lidvid: LIDVID) -> str: lid = lidvid.lid() # parts are collection, product parts = lid.parts()[1:] if len(parts) >= 2 and parts[0] not in NO_VISIT_COLLECTIONS: fake_filename = f"{parts[1]}_raw.fits" visit = HstFilename(fake_filename).visit() visit_part = f"visit_{visit}" parts[1] = visit_part return fs.path.join(*parts)
def test_getitem(self) -> None: # A list of VersionViews and LIDVIDs that live inside them. vvs_lidvids = [ (self.vv, LIDVID("urn:nasa:pds:b::1.2")), (self.vv, LIDVID("urn:nasa:pds:b:c::1.1")), (self.vv2, LIDVID("urn:nasa:pds:b::2.0")), ] for vv, lidvid in vvs_lidvids: # Contents of a LIDVID can be found two different ways. # First, they can come directly from the Multiversioned. # Second, you can build a VersionView from that LIDVID or # from a parent of it, then get the contents from its LID. # Check that the two are equal, after reducing LIDVIDs to # LIDs. lid = lidvid.lid() vv_contents = vv[lid] mv_contents = self.mv[lidvid].to_lid_version_contents() self.assertEqual(mv_contents, vv_contents)
def visit_document_collection( self, bundle_lidvid: str, document_collection: DocumentCollection, post: bool, ) -> None: if post: if not changes_dict.changed( LIDVID(document_collection.lidvid).lid()): return self._post_visit_collection(document_collection)
def test_eq(self) -> None: self.assertTrue( LIDVID("urn:nasa:pds:b:c:p::1.0") == LIDVID( "urn:nasa:pds:b:c:p::1.0")) self.assertFalse( LIDVID("urn:nasa:pds:b:c:p::1.1") == LIDVID( "urn:nasa:pds:b:c:p::1.0")) self.assertTrue( LIDVID("urn:nasa:pds:b:c:p::1.1") != LIDVID( "urn:nasa:pds:b:c:p::1.0")) self.assertFalse( LIDVID("urn:nasa:pds:b:c:p::1.0") != LIDVID( "urn:nasa:pds:b:c:p::1.0"))
def create_deliverable_view( bundle_db: BundleDB, mv: Multiversioned, lid: LID, vid: Optional[VID] = None) -> "DeliverableView": if vid is None: vv = mv.create_version_view(lid) else: lidvid = LIDVID.create_from_lid_and_vid(lid, vid) vv = VersionView(mv, lidvid) return DeliverableView(vv)
def test_make_version_view(self) -> None: with make_mv_osfs(join(self.temp_dir, "foo")) as base_fs: mv = Multiversioned(base_fs) lidvid = LIDVID("urn:nasa:pds:b::1.0") no_lidvids: Set[LIDVID] = set() mv[lidvid] = VersionContents.create_from_lidvids( no_lidvids, TempFS(), set()) names = OSFS(self.temp_dir).walk.dirs() self.assertEqual({"/foo-mv", "/foo-mv/b", "/foo-mv/b/v$1.0"}, set(names)) with make_version_view(base_fs, "b") as vv: self.assertEqual(["/b$"], list(vv.walk.dirs()))
def std_is_new(lidvid: LIDVID, contents: VersionContents, mv: "Multiversioned") -> bool: """ The standard IS_NEW_TEST function. If it's a document collection, it checks the document files for changes; otherwise, it checks only FITS files. """ lid = lidvid.lid() if lid.is_collection_lid() and lid.collection_id == "document": filt = doc_filter else: filt = fits_filter return mv[lidvid].filter_filepaths(filt) != contents
def demo_version_views() -> None: """ Demonstration of the use of VersionViews to see individual versions within a Multiversioned object. """ global m # Set up the Multiversioned archive. demo_multi() # Get a view on the first version and check the contents. vv1 = VersionView(m, LIDVID("urn:nasa:pds:hst_00001::1.0")) show_fs(vv1, "this is a view on version 1.0") print( "in version 1.0, j12345_raw.fits contains: ", vv1.readtext("hst_00001$/data_acs_raw$/j12345s$/j12345s_raw.fits"), ) # Get a view on the second version and check *its* contents. vv2 = VersionView(m, LIDVID("urn:nasa:pds:hst_00001::2.0")) show_fs(vv2, "this is a view on version 2.0") print( "in version 2.0, j12345_raw.fits contains: ", vv2.readtext("hst_00001$/data_acs_raw$/j12345s$/j12345s_raw.fits"), )
def _run(self) -> None: working_dir: str = self.working_dir() archive_dir: str = self.archive_dir() deliverable_dir: str = self.deliverable_dir() manifest_dir: str = self.manifest_dir() try: PDS_LOGGER.open("Create deliverable directory") if os.path.isdir(deliverable_dir): raise ValueError( f"{deliverable_dir} cannot exist for MakeDeliverable.") changes_path = os.path.join(working_dir, CHANGES_DICT_NAME) changes_dict = read_changes_dict(changes_path) with make_osfs(archive_dir) as archive_osfs, make_multiversioned( archive_osfs) as mv: bundle_segment = self._bundle_segment bundle_lid = LID.create_from_parts([bundle_segment]) bundle_vid = changes_dict.vid(bundle_lid) bundle_lidvid = LIDVID.create_from_lid_and_vid( bundle_lid, bundle_vid) version_view = VersionView(mv, bundle_lidvid) synth_files: Dict[str, bytes] = dict() # open the database db_filepath = fs.path.join(working_dir, _BUNDLE_DB_NAME) bundle_db = create_bundle_db_from_os_filepath(db_filepath) bundle_lidvid_str = str(bundle_lidvid) synth_files = dict() cm = make_checksum_manifest(bundle_db, bundle_lidvid_str, short_lidvid_to_dirpath) synth_files["/checksum.manifest.txt"] = cm.encode("utf-8") tm = make_transfer_manifest(bundle_db, bundle_lidvid_str, short_lidvid_to_dirpath) synth_files["/transfer.manifest.txt"] = tm.encode("utf-8") deliverable_view = DeliverableView(version_view, synth_files) os.mkdir(deliverable_dir) deliverable_osfs = OSFS(deliverable_dir) copy_fs(deliverable_view, deliverable_osfs) PDS_LOGGER.log("info", f"Deliverable: {deliverable_dir}") except Exception as e: PDS_LOGGER.exception(e) finally: PDS_LOGGER.close()
def _populate_collections(changes_dict: ChangesDict, db: BundleDB) -> None: for lid, (vid, changed) in changes_dict.items(): if lid.is_collection_lid(): lidvid = LIDVID.create_from_lid_and_vid(lid, vid) bundle_lidvid = changes_dict.parent_lidvid(lidvid) if changed: if lid.collection_id == "document": db.create_document_collection(str(lidvid), str(bundle_lidvid)) elif lid.collection_id == "schema": # it's created separately _populate_schema_collection(db, str(bundle_lidvid)) else: db.create_other_collection(str(lidvid), str(bundle_lidvid)) else: if changes_dict.changed(bundle_lidvid.lid()): db.create_bundle_collection_link(str(bundle_lidvid), str(lidvid))
def _populate_target_identification(changes_dict: ChangesDict, db: BundleDB, sv_deltas: COWFS) -> None: for lid, (vid, changed) in changes_dict.items(): if changed and lid.is_product_lid(): lidvid = LIDVID.create_from_lid_and_vid(lid, vid) product_path = lid_to_dirpath(lidvid.lid()) # Get a list of SHM/SPT/SHP fits files fits_files = [ fits_file for fits_file in sv_deltas.listdir(product_path) if (fs.path.splitext(fits_file)[1].lower() == ".fits" and has_suffix_shm_spt_shf( fs.path.splitext(fits_file)[0].lower())) ] # Pass the path of SHM/SPT/SHP fits files to create a record in # target identification table for fits_file in fits_files: fits_file_path = fs.path.join(product_path, fits_file) fits_os_path = sv_deltas.getsyspath(fits_file_path) db.create_target_identification(fits_os_path)
def _post_visit_collection(self, collection: Collection) -> None: """Common implementation for all collections.""" if not changes_dict.changed(LIDVID(collection.lidvid).lid()): return collection_lidvid = collection.lidvid collection_dir_path = _lidvid_to_dir(collection_lidvid) inventory = make_collection_inventory(self.db, collection_lidvid) inventory_filename = get_collection_inventory_name( self.db, collection_lidvid) inventory_filepath = fs.path.join(collection_dir_path, inventory_filename) # TODO Remove this kludge to fix COWFS.setbytes() bug. if label_deltas.exists(inventory_filepath): label_deltas.remove(inventory_filepath) label_deltas.setbytes(inventory_filepath, inventory) bundle_db.create_collection_inventory( label_deltas.getsyspath(inventory_filepath), inventory_filename, collection_lidvid, ) log_label("collection", collection_lidvid) label = make_collection_label(self.db, info, collection_lidvid, str(bundle_lidvid), _VERIFY) label_filename = get_collection_label_name(self.db, collection_lidvid) label_filepath = fs.path.join(collection_dir_path, label_filename) # TODO Remove this kludge to fix COWFS.setbytes() bug. if label_deltas.exists(label_filepath): label_deltas.remove(label_filepath) label_deltas.setbytes(label_filepath, label) bundle_db.create_collection_label( label_deltas.getsyspath(label_filepath), label_filename, collection_lidvid, )
def _populate_products(changes_dict: ChangesDict, db: BundleDB, sv_deltas: COWFS) -> None: for lid, (vid, changed) in changes_dict.items(): if lid.is_product_lid(): lidvid = LIDVID.create_from_lid_and_vid(lid, vid) collection_lidvid = changes_dict.parent_lidvid(lidvid) if changed: product_path = lid_to_dirpath(lidvid.lid()) if collection_lidvid.lid().collection_id == "document": db.create_document_product(str(lidvid), str(collection_lidvid)) doc_files = [ doc_file for doc_file in sv_deltas.listdir(product_path) if (fs.path.splitext(doc_file)[1].lower() in DOCUMENT_SUFFIXES) ] for doc_file in doc_files: sys_filepath = sv_deltas.getsyspath( fs.path.join(product_path, doc_file)) db.create_document_file(sys_filepath, doc_file, str(lidvid)) else: db.create_fits_product(str(lidvid), str(collection_lidvid)) fits_files = [ fits_file for fits_file in sv_deltas.listdir(product_path) if fs.path.splitext(fits_file)[1].lower() == ".fits" ] for fits_file in fits_files: fits_file_path = fs.path.join(product_path, fits_file) fits_os_path = sv_deltas.getsyspath(fits_file_path) populate_database_from_fits_file( db, fits_os_path, str(lidvid)) else: if changes_dict.changed(collection_lidvid.lid()): db.create_collection_product_link(str(collection_lidvid), str(lidvid))
def visit_fits_file(self, collection_lidvid: str, fits_file: FitsFile) -> None: if not changes_dict.changed( LIDVID(fits_file.product_lidvid).lid()): return log_label("FITS product", fits_file.product_lidvid) label = make_fits_product_label( working_dir, self.db, collection_lidvid, str(fits_file.product_lidvid), str(bundle_lidvid), str(fits_file.basename), _VERIFY, ) label_base = fs.path.splitext(fits_file.basename)[0] label_filename = label_base + ".xml" product_lidvid = str(fits_file.product_lidvid) product_dir_path = _lidvid_to_dir(product_lidvid) label_filepath = fs.path.join(product_dir_path, label_filename) label_deltas.setbytes(label_filepath, label) bundle_db.create_product_label( label_deltas.getsyspath(label_filepath), label_filename, product_lidvid)
def label_to_filepath(collection_label: CollectionLabel) -> str: dir = lidvid_to_dirpath(LIDVID(collection_label.collection_lidvid)) return fs.path.relpath(fs.path.join(dir, collection_label.basename))
def test_is_product_lidvid_property(self, lidvid: LIDVID) -> None: self.assertEqual(lidvid.is_product_lidvid(), lidvid.lid().is_product_lid())
def test_is_product_lidvid(self) -> None: self.assertFalse(LIDVID("urn:nasa:pds:b::1.0").is_product_lidvid()) self.assertFalse(LIDVID("urn:nasa:pds:b:c::1.0").is_product_lidvid()) self.assertTrue(LIDVID("urn:nasa:pds:b:c:p::1.0").is_product_lidvid())
def test_is_collection_lidvid_property(self, lidvid: LIDVID) -> None: self.assertEqual(lidvid.is_collection_lidvid(), lidvid.lid().is_collection_lid())
def test_vid(self) -> None: self.assertEqual(VID("666.0"), LIDVID("urn:nasa:pds:b:c:p::666.0").vid()) self.assertEqual(VID("3.14159"), LIDVID("urn:nasa:pds:b:c:p::3.14159").vid())
def test_lid(self) -> None: self.assertEqual(LID("urn:nasa:pds:b:c:p"), LIDVID("urn:nasa:pds:b:c:p::666.666").lid())
def file_to_filepath(file: File) -> str: dir = lidvid_to_dirpath(LIDVID(file.product_lidvid)) return fs.path.relpath(fs.path.join(dir, file.basename))
def inventory_to_filepath(collection_inventory: CollectionInventory) -> str: dir = lidvid_to_dirpath(LIDVID(collection_inventory.collection_lidvid)) return fs.path.relpath(fs.path.join(dir, collection_inventory.basename))
def label_to_filepath(product_label: ProductLabel) -> str: dir = lidvid_to_dirpath(LIDVID(product_label.product_lidvid)) return fs.path.relpath(fs.path.join(dir, product_label.basename))
def test_create_from_lid_and_vid(self) -> None: lid = LID("urn:nasa:pds:ssc01.hirespc.cruise:browse") vid = VID("2.5") lidvid = LIDVID.create_from_lid_and_vid(lid, vid) self.assertEqual( LIDVID("urn:nasa:pds:ssc01.hirespc.cruise:browse::2.5"), lidvid)
def plain_lidvid_to_dirpath(lidvid: LIDVID) -> str: lid = lidvid.lid() parts = lid.parts()[1:] return fs.path.join("/", *parts)
def test_lidvid_lid_vid_properties(self, lid: LID, vid: VID) -> None: lidvid = LIDVID.create_from_lid_and_vid(lid, vid) self.assertEqual(lidvid.lid(), lid) self.assertEqual(lidvid.vid(), vid)
def label_to_filepath(bundle_label: BundleLabel) -> str: dir = lidvid_to_dirpath(LIDVID(bundle_label.bundle_lidvid)) return fs.path.relpath(fs.path.join(dir, bundle_label.basename))
def test_repr(self) -> None: self.assertEqual("LIDVID('urn:nasa:pds:b:c:p::1.0')", repr(LIDVID("urn:nasa:pds:b:c:p::1.0")))