def _fill_in_old_browse_collection( db: BundleDB, changes_dict: ChangesDict, bundle_lidvid: LIDVID, data_collection_lidvid: LIDVID, ) -> None: bundle_segment = bundle_lidvid.lid().parts()[0] collection_segment = data_collection_lidvid.lid().parts()[1] browse_collection_lid = data_collection_lidvid.lid().to_browse_lid() browse_collection_segment = browse_collection_lid.collection_id browse_collection_vid = data_collection_lidvid.vid() browse_collection_lidvid = LIDVID.create_from_lid_and_vid( browse_collection_lid, browse_collection_vid) changes_dict.set(browse_collection_lid, browse_collection_vid, False) db.create_bundle_collection_link(str(bundle_lidvid), str(browse_collection_lidvid)) try: PDS_LOGGER.open("Fill in old browse collection") PDS_LOGGER.log( "info", f"Created link and change for {browse_collection_lidvid}") for product in db.get_collection_products( str(browse_collection_lidvid)): product_lidvid = LIDVID(product.lidvid) changes_dict.set(product_lidvid.lid(), product_lidvid.vid(), False) PDS_LOGGER.log("info", f"Created link and change for {product_lidvid}") except Exception as e: PDS_LOGGER.exception(e) finally: PDS_LOGGER.close()
def _create_context_collection(self, bundle: Bundle) -> None: context_products = bundle_db.get_context_products() if not context_products: return bundle_lidvid = str(bundle.lidvid) collection_lidvid = _extend_lidvid(bundle_lidvid, "context") bundle_db.create_context_collection(collection_lidvid, bundle_lidvid) clv = LIDVID(collection_lidvid) changes_dict.set(clv.lid(), clv.vid(), True) bundle_dir_path = _lidvid_to_dir(bundle_lidvid) context_coll_dir_path = fs.path.join(bundle_dir_path, "context$") label_deltas.makedir(context_coll_dir_path) # Create target lable if it doesn't exist in PDS page. self._create_context_target_label(context_coll_dir_path, collection_lidvid) # Create investigation label self._create_context_investigation_label(bundle_lidvid, context_coll_dir_path, collection_lidvid) collection = bundle_db.get_collection(collection_lidvid) self._post_visit_collection(collection)
def __setitem__(self, lidvid: LIDVID, contents: VersionContents) -> None: if lidvid in self: # illegal to set twice raise IndexError(f"Repeated setting at {lidvid}") # check that contents aren't contradictory lidvid_parts = lidvid.lid().parts() d = {} for sub_lidvid in contents.lidvids(): sub_parts = sub_lidvid.lid().parts() d[sub_parts[-1]] = str(sub_lidvid.vid()) if sub_parts[:-1] != lidvid_parts: raise ValueError(f"LIDVID {sub_lidvid} in contents cannot be " f"a child of index LIDVID {lidvid}") lidvid_dir = self.make_lidvid_dir(lidvid) if d: write_subdir_versions_to_directory(self.fs, lidvid_dir, d) for src_filepath in contents.filepaths: dst_filepath = fs.path.join(lidvid_dir, fs.path.relpath(src_filepath)) self.fs.makedirs(fs.path.dirname(dst_filepath), None, True) fs.copy.copy_file(contents.fs, src_filepath, self.fs, dst_filepath) if not self.fs.isdir(lidvid_path(lidvid)): raise ValueError(f"Lidvid path for {lidvid} is not a directory.") if lidvid not in self: raise IndexError(f"{lidvid} is not set in the wrapped filesystem.")
def plain_lidvid_to_visits_dirpath(lidvid: LIDVID) -> str: # This is only run on directories, and always relative to /hst_NNNNN. lid = lidvid.lid() parts = lid.parts()[1:] if len(parts) == 2: visit = _visit_of(parts[1]) if visit is not None: parts[1] = visit return fs.path.join("/", *parts)
def parent_lidvid(self, lidvid: LIDVID) -> LIDVID: lid = lidvid.lid() if lid not in self.changes_dict: raise KeyError(f"lid={lid} not in changes_dict.") parent_lid = lid.parent_lid() if parent_lid not in self.changes_dict: raise KeyError(f"parent_lid={parent_lid} not in changes_dict.") parent_vid = self.vid(parent_lid) return LIDVID.create_from_lid_and_vid(parent_lid, parent_vid)
def short_lidvid_to_dirpath(lidvid: LIDVID) -> str: lid = lidvid.lid() # parts are collection, product parts = lid.parts()[1:] if len(parts) >= 2 and parts[0] not in NO_VISIT_COLLECTIONS: fake_filename = f"{parts[1]}_raw.fits" visit = HstFilename(fake_filename).visit() visit_part = f"visit_{visit}" parts[1] = visit_part return fs.path.join(*parts)
def label_to_filepath(context_product_label: ContextProductLabel) -> str: # This is a workaround because lidvid under context doesn't # match the directory path. For example: # target lidvid: urn:nasa:pds:context:target:asteroid.762_pulcova::1.0 # path: hst_09059/context/asteroid.762_pulcova_1.0.xml lidvid_mod = str(context_product_label.collection_lidvid) dir = lidvid_to_dirpath(LIDVID(lidvid_mod)) lidvid = LIDVID(lidvid_mod) lid = lidvid.lid() # parts are collection, product parts = lid.parts()[1:] return fs.path.relpath(fs.path.join(dir, context_product_label.basename))
def std_is_new(lidvid: LIDVID, contents: VersionContents, mv: "Multiversioned") -> bool: """ The standard IS_NEW_TEST function. If it's a document collection, it checks the document files for changes; otherwise, it checks only FITS files. """ lid = lidvid.lid() if lid.is_collection_lid() and lid.collection_id == "document": filt = doc_filter else: filt = fits_filter return mv[lidvid].filter_filepaths(filt) != contents
def lidvid_path(lidvid: LIDVID) -> str: """ Return the directory path corresponding to this LIDVID. """ def vid_to_dir_part(vid: VID) -> str: """ Convert a VID to a directory name. """ return f"v${vid}" lid = lidvid.lid() dir = lid_path(lid) vid_bit = vid_to_dir_part(lidvid.vid()) return fs.path.join(dir, vid_bit)
def plain_lidvid_to_dirpath(lidvid: LIDVID) -> str: lid = lidvid.lid() parts = lid.parts()[1:] return fs.path.join("/", *parts)
def test_is_product_lidvid_property(self, lidvid: LIDVID) -> None: self.assertEqual(lidvid.is_product_lidvid(), lidvid.lid().is_product_lid())
def test_is_collection_lidvid_property(self, lidvid: LIDVID) -> None: self.assertEqual(lidvid.is_collection_lidvid(), lidvid.lid().is_collection_lid())
def test_is_next_minor_lidvid_property(self, lidvid: LIDVID) -> None: self.assertEqual( LIDVID.create_from_lid_and_vid(lidvid.lid(), lidvid.vid().next_minor_vid()), lidvid.next_minor_lidvid(), )
def _extend_lidvid(lidvid_str: str, segment: str) -> str: lidvid = LIDVID(lidvid_str) lid = lidvid.lid().extend_lid(segment) new_lidvid = LIDVID.create_from_lid_and_vid(lid, lidvid.vid()) return str(new_lidvid)
def _build_browse_collection( db: BundleDB, changes_dict: ChangesDict, browse_deltas: COWFS, bundle_lidvid: LIDVID, data_collection_lidvid: LIDVID, bundle_path: str, ) -> None: bundle_segment = bundle_lidvid.lid().parts()[0] collection_segment = data_collection_lidvid.lid().parts()[1] browse_collection_lid = data_collection_lidvid.lid().to_browse_lid() collection_path = f"{bundle_path}{collection_segment}$/" browse_collection_segment = browse_collection_lid.collection_id browse_collection_path = f"{bundle_path}{browse_collection_segment}$/" browse_collection_vid = data_collection_lidvid.vid() browse_collection_lidvid = LIDVID.create_from_lid_and_vid( browse_collection_lid, browse_collection_vid) changes_dict.set(browse_collection_lid, browse_collection_vid, True) browse_deltas.makedirs(browse_collection_path, recreate=True) db.create_other_collection(str(browse_collection_lidvid), str(bundle_lidvid)) db.create_bundle_collection_link(str(bundle_lidvid), str(browse_collection_lidvid)) product_segments = [ str(prod[:-1]) for prod in browse_deltas.listdir(collection_path) if "$" in prod ] for product_segment in product_segments: # These product_segments are from the data_collection product_lid = LID.create_from_parts( [bundle_segment, collection_segment, product_segment]) product_vid = changes_dict.vid(product_lid) product_path = f"{collection_path}{product_segment}$/" browse_product_path = f"{browse_collection_path}{product_segment}$/" browse_product_lidvid = _extend_lidvid(browse_collection_lid, product_vid, product_segment) if changes_dict.changed(product_lid): fits_product_lidvid = _extend_lidvid( data_collection_lidvid.lid(), data_collection_lidvid.vid(), product_segment, ) bpl = LIDVID(browse_product_lidvid) changes_dict.set(bpl.lid(), bpl.vid(), True) browse_deltas.makedirs(browse_product_path, recreate=True) db.create_browse_product( browse_product_lidvid, fits_product_lidvid, str(browse_collection_lidvid), ) db.create_collection_product_link(str(browse_collection_lidvid), browse_product_lidvid) for fits_file in browse_deltas.listdir(product_path): fits_filepath = fs.path.join(product_path, fits_file) fits_os_filepath = browse_deltas.getsyspath(fits_filepath) browse_file = fs.path.splitext(fits_file)[0] + ".jpg" browse_filepath = fs.path.join(browse_product_path, browse_file) # In a COWFS, a directory does not have a # syspath, only files. So we write a stub # file into the directory, find its syspath # and its directory's syspath. Then we remove # the stub file. browse_deltas.touch(browse_filepath) browse_product_os_filepath = browse_deltas.getsyspath( browse_filepath) browse_deltas.remove(browse_filepath) browse_product_os_dirpath = fs.path.dirname( browse_product_os_filepath) # Picmaker expects a list of strings. If you give it # str, it'll index into it and complain about '/' # not being a file. So don't do that! try: picmaker.ImagesToPics( [str(fits_os_filepath)], browse_product_os_dirpath, filter="None", percentiles=(1, 99), ) except IndexError as e: tb = traceback.format_exc() message = f"File {fits_file}: {e}\n{tb}" raise Exception(message) browse_os_filepath = fs.path.join(browse_product_os_dirpath, browse_file) size = os.stat(browse_os_filepath).st_size db.create_browse_file(browse_os_filepath, browse_file, browse_product_lidvid, size) else: bpl = LIDVID(browse_product_lidvid) changes_dict.set(bpl.lid(), bpl.vid(), False) db.create_collection_product_link(str(browse_collection_lidvid), browse_product_lidvid)
def _lidvid_to_dirpath(lidvid: LIDVID) -> str: lid = lidvid.lid() return fs.path.relpath(lid_to_dir(lid))