def plain_lidvid_to_visits_dirpath(lidvid: LIDVID) -> str:
# This is only run on directories, and always relative to /hst_NNNNN.
lid = lidvid.lid()
parts = lid.parts()[1:]
if len(parts) == 2:
visit = _visit_of(parts[1])
if visit is not None:
parts[1] = visit
return fs.path.join("/", *parts)
def _munge_lidvid(product_lidvid: str, suffix: str, new_basename: str) -> str:
bundle_id, collection_id, product_id = LIDVID(product_lidvid).lid().parts()
# TODO This is a hack
collection_type = get_collection_type(suffix=suffix)
first_underscore_idx = collection_id.index("_")
new_collection_id = (collection_type +
collection_id[first_underscore_idx:-3] +
suffix.lower())
# TODO This is a hack
new_product_id = new_basename[0:9]
new_lid = LID.create_from_parts(
[bundle_id, new_collection_id, new_product_id])
# TODO This is a hack. Fix it.
vid = VID("1.0")
new_lidvid = LIDVID.create_from_lid_and_vid(new_lid, vid)
return str(new_lidvid)
def short_lidvid_to_dirpath(lidvid: LIDVID) -> str:
lid = lidvid.lid()
# parts are collection, product
parts = lid.parts()[1:]
if len(parts) >= 2 and parts[0] not in NO_VISIT_COLLECTIONS:
fake_filename = f"{parts[1]}_raw.fits"
visit = HstFilename(fake_filename).visit()
visit_part = f"visit_{visit}"
parts[1] = visit_part
return fs.path.join(*parts)
def test_getitem(self) -> None:
# A list of VersionViews and LIDVIDs that live inside them.
vvs_lidvids = [
(self.vv, LIDVID("urn:nasa:pds:b::1.2")),
(self.vv, LIDVID("urn:nasa:pds:b:c::1.1")),
(self.vv2, LIDVID("urn:nasa:pds:b::2.0")),
]
for vv, lidvid in vvs_lidvids:
# Contents of a LIDVID can be found two different ways.
# First, they can come directly from the Multiversioned.
# Second, you can build a VersionView from that LIDVID or
# from a parent of it, then get the contents from its LID.
# Check that the two are equal, after reducing LIDVIDs to
# LIDs.
lid = lidvid.lid()
vv_contents = vv[lid]
mv_contents = self.mv[lidvid].to_lid_version_contents()
self.assertEqual(mv_contents, vv_contents)
def visit_document_collection(
self,
bundle_lidvid: str,
document_collection: DocumentCollection,
post: bool,
) -> None:
if post:
if not changes_dict.changed(
LIDVID(document_collection.lidvid).lid()):
return
self._post_visit_collection(document_collection)
def test_eq(self) -> None:
self.assertTrue(
LIDVID("urn:nasa:pds:b:c:p::1.0") == LIDVID(
"urn:nasa:pds:b:c:p::1.0"))
self.assertFalse(
LIDVID("urn:nasa:pds:b:c:p::1.1") == LIDVID(
"urn:nasa:pds:b:c:p::1.0"))
self.assertTrue(
LIDVID("urn:nasa:pds:b:c:p::1.1") != LIDVID(
"urn:nasa:pds:b:c:p::1.0"))
self.assertFalse(
LIDVID("urn:nasa:pds:b:c:p::1.0") != LIDVID(
"urn:nasa:pds:b:c:p::1.0"))
def create_deliverable_view(
bundle_db: BundleDB,
mv: Multiversioned,
lid: LID,
vid: Optional[VID] = None) -> "DeliverableView":
if vid is None:
vv = mv.create_version_view(lid)
else:
lidvid = LIDVID.create_from_lid_and_vid(lid, vid)
vv = VersionView(mv, lidvid)
return DeliverableView(vv)
def test_make_version_view(self) -> None:
with make_mv_osfs(join(self.temp_dir, "foo")) as base_fs:
mv = Multiversioned(base_fs)
lidvid = LIDVID("urn:nasa:pds:b::1.0")
no_lidvids: Set[LIDVID] = set()
mv[lidvid] = VersionContents.create_from_lidvids(
no_lidvids, TempFS(), set())
names = OSFS(self.temp_dir).walk.dirs()
self.assertEqual({"/foo-mv", "/foo-mv/b", "/foo-mv/b/v$1.0"},
set(names))
with make_version_view(base_fs, "b") as vv:
self.assertEqual(["/b$"], list(vv.walk.dirs()))
def std_is_new(lidvid: LIDVID, contents: VersionContents,
mv: "Multiversioned") -> bool:
"""
The standard IS_NEW_TEST function. If it's a document collection,
it checks the document files for changes; otherwise, it checks
only FITS files.
"""
lid = lidvid.lid()
if lid.is_collection_lid() and lid.collection_id == "document":
filt = doc_filter
else:
filt = fits_filter
return mv[lidvid].filter_filepaths(filt) != contents
def demo_version_views() -> None:
"""
Demonstration of the use of VersionViews to see individual
versions within a Multiversioned object.
"""
global m
# Set up the Multiversioned archive.
demo_multi()
# Get a view on the first version and check the contents.
vv1 = VersionView(m, LIDVID("urn:nasa:pds:hst_00001::1.0"))
show_fs(vv1, "this is a view on version 1.0")
print(
"in version 1.0, j12345_raw.fits contains: ",
vv1.readtext("hst_00001$/data_acs_raw$/j12345s$/j12345s_raw.fits"),
)
# Get a view on the second version and check *its* contents.
vv2 = VersionView(m, LIDVID("urn:nasa:pds:hst_00001::2.0"))
show_fs(vv2, "this is a view on version 2.0")
print(
"in version 2.0, j12345_raw.fits contains: ",
vv2.readtext("hst_00001$/data_acs_raw$/j12345s$/j12345s_raw.fits"),
)
def _run(self) -> None:
working_dir: str = self.working_dir()
archive_dir: str = self.archive_dir()
deliverable_dir: str = self.deliverable_dir()
manifest_dir: str = self.manifest_dir()
try:
PDS_LOGGER.open("Create deliverable directory")
if os.path.isdir(deliverable_dir):
raise ValueError(
f"{deliverable_dir} cannot exist for MakeDeliverable.")
changes_path = os.path.join(working_dir, CHANGES_DICT_NAME)
changes_dict = read_changes_dict(changes_path)
with make_osfs(archive_dir) as archive_osfs, make_multiversioned(
archive_osfs) as mv:
bundle_segment = self._bundle_segment
bundle_lid = LID.create_from_parts([bundle_segment])
bundle_vid = changes_dict.vid(bundle_lid)
bundle_lidvid = LIDVID.create_from_lid_and_vid(
bundle_lid, bundle_vid)
version_view = VersionView(mv, bundle_lidvid)
synth_files: Dict[str, bytes] = dict()
# open the database
db_filepath = fs.path.join(working_dir, _BUNDLE_DB_NAME)
bundle_db = create_bundle_db_from_os_filepath(db_filepath)
bundle_lidvid_str = str(bundle_lidvid)
synth_files = dict()
cm = make_checksum_manifest(bundle_db, bundle_lidvid_str,
short_lidvid_to_dirpath)
synth_files["/checksum.manifest.txt"] = cm.encode("utf-8")
tm = make_transfer_manifest(bundle_db, bundle_lidvid_str,
short_lidvid_to_dirpath)
synth_files["/transfer.manifest.txt"] = tm.encode("utf-8")
deliverable_view = DeliverableView(version_view, synth_files)
os.mkdir(deliverable_dir)
deliverable_osfs = OSFS(deliverable_dir)
copy_fs(deliverable_view, deliverable_osfs)
PDS_LOGGER.log("info", f"Deliverable: {deliverable_dir}")
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()
def _populate_collections(changes_dict: ChangesDict, db: BundleDB) -> None:
for lid, (vid, changed) in changes_dict.items():
if lid.is_collection_lid():
lidvid = LIDVID.create_from_lid_and_vid(lid, vid)
bundle_lidvid = changes_dict.parent_lidvid(lidvid)
if changed:
if lid.collection_id == "document":
db.create_document_collection(str(lidvid),
str(bundle_lidvid))
elif lid.collection_id == "schema":
# it's created separately
_populate_schema_collection(db, str(bundle_lidvid))
else:
db.create_other_collection(str(lidvid), str(bundle_lidvid))
else:
if changes_dict.changed(bundle_lidvid.lid()):
db.create_bundle_collection_link(str(bundle_lidvid),
str(lidvid))
def _populate_target_identification(changes_dict: ChangesDict, db: BundleDB,
sv_deltas: COWFS) -> None:
for lid, (vid, changed) in changes_dict.items():
if changed and lid.is_product_lid():
lidvid = LIDVID.create_from_lid_and_vid(lid, vid)
product_path = lid_to_dirpath(lidvid.lid())
# Get a list of SHM/SPT/SHP fits files
fits_files = [
fits_file for fits_file in sv_deltas.listdir(product_path)
if (fs.path.splitext(fits_file)[1].lower() == ".fits"
and has_suffix_shm_spt_shf(
fs.path.splitext(fits_file)[0].lower()))
]
# Pass the path of SHM/SPT/SHP fits files to create a record in
# target identification table
for fits_file in fits_files:
fits_file_path = fs.path.join(product_path, fits_file)
fits_os_path = sv_deltas.getsyspath(fits_file_path)
db.create_target_identification(fits_os_path)
def _post_visit_collection(self, collection: Collection) -> None:
"""Common implementation for all collections."""
if not changes_dict.changed(LIDVID(collection.lidvid).lid()):
return
collection_lidvid = collection.lidvid
collection_dir_path = _lidvid_to_dir(collection_lidvid)
inventory = make_collection_inventory(self.db, collection_lidvid)
inventory_filename = get_collection_inventory_name(
self.db, collection_lidvid)
inventory_filepath = fs.path.join(collection_dir_path,
inventory_filename)
# TODO Remove this kludge to fix COWFS.setbytes() bug.
if label_deltas.exists(inventory_filepath):
label_deltas.remove(inventory_filepath)
label_deltas.setbytes(inventory_filepath, inventory)
bundle_db.create_collection_inventory(
label_deltas.getsyspath(inventory_filepath),
inventory_filename,
collection_lidvid,
)
log_label("collection", collection_lidvid)
label = make_collection_label(self.db, info, collection_lidvid,
str(bundle_lidvid), _VERIFY)
label_filename = get_collection_label_name(self.db,
collection_lidvid)
label_filepath = fs.path.join(collection_dir_path, label_filename)
# TODO Remove this kludge to fix COWFS.setbytes() bug.
if label_deltas.exists(label_filepath):
label_deltas.remove(label_filepath)
label_deltas.setbytes(label_filepath, label)
bundle_db.create_collection_label(
label_deltas.getsyspath(label_filepath),
label_filename,
collection_lidvid,
)
def _populate_products(changes_dict: ChangesDict, db: BundleDB,
sv_deltas: COWFS) -> None:
for lid, (vid, changed) in changes_dict.items():
if lid.is_product_lid():
lidvid = LIDVID.create_from_lid_and_vid(lid, vid)
collection_lidvid = changes_dict.parent_lidvid(lidvid)
if changed:
product_path = lid_to_dirpath(lidvid.lid())
if collection_lidvid.lid().collection_id == "document":
db.create_document_product(str(lidvid),
str(collection_lidvid))
doc_files = [
doc_file
for doc_file in sv_deltas.listdir(product_path)
if (fs.path.splitext(doc_file)[1].lower() in
DOCUMENT_SUFFIXES)
]
for doc_file in doc_files:
sys_filepath = sv_deltas.getsyspath(
fs.path.join(product_path, doc_file))
db.create_document_file(sys_filepath, doc_file,
str(lidvid))
else:
db.create_fits_product(str(lidvid), str(collection_lidvid))
fits_files = [
fits_file
for fits_file in sv_deltas.listdir(product_path)
if fs.path.splitext(fits_file)[1].lower() == ".fits"
]
for fits_file in fits_files:
fits_file_path = fs.path.join(product_path, fits_file)
fits_os_path = sv_deltas.getsyspath(fits_file_path)
populate_database_from_fits_file(
db, fits_os_path, str(lidvid))
else:
if changes_dict.changed(collection_lidvid.lid()):
db.create_collection_product_link(str(collection_lidvid),
str(lidvid))
def visit_fits_file(self, collection_lidvid: str,
fits_file: FitsFile) -> None:
if not changes_dict.changed(
LIDVID(fits_file.product_lidvid).lid()):
return
log_label("FITS product", fits_file.product_lidvid)
label = make_fits_product_label(
working_dir,
self.db,
collection_lidvid,
str(fits_file.product_lidvid),
str(bundle_lidvid),
str(fits_file.basename),
_VERIFY,
)
label_base = fs.path.splitext(fits_file.basename)[0]
label_filename = label_base + ".xml"
product_lidvid = str(fits_file.product_lidvid)
product_dir_path = _lidvid_to_dir(product_lidvid)
label_filepath = fs.path.join(product_dir_path, label_filename)
label_deltas.setbytes(label_filepath, label)
bundle_db.create_product_label(
label_deltas.getsyspath(label_filepath), label_filename,
product_lidvid)
def label_to_filepath(collection_label: CollectionLabel) -> str:
dir = lidvid_to_dirpath(LIDVID(collection_label.collection_lidvid))
return fs.path.relpath(fs.path.join(dir, collection_label.basename))
def test_is_product_lidvid_property(self, lidvid: LIDVID) -> None:
self.assertEqual(lidvid.is_product_lidvid(),
lidvid.lid().is_product_lid())
def test_is_product_lidvid(self) -> None:
self.assertFalse(LIDVID("urn:nasa:pds:b::1.0").is_product_lidvid())
self.assertFalse(LIDVID("urn:nasa:pds:b:c::1.0").is_product_lidvid())
self.assertTrue(LIDVID("urn:nasa:pds:b:c:p::1.0").is_product_lidvid())
def test_is_collection_lidvid_property(self, lidvid: LIDVID) -> None:
self.assertEqual(lidvid.is_collection_lidvid(),
lidvid.lid().is_collection_lid())
def test_vid(self) -> None:
self.assertEqual(VID("666.0"),
LIDVID("urn:nasa:pds:b:c:p::666.0").vid())
self.assertEqual(VID("3.14159"),
LIDVID("urn:nasa:pds:b:c:p::3.14159").vid())
def test_lid(self) -> None:
self.assertEqual(LID("urn:nasa:pds:b:c:p"),
LIDVID("urn:nasa:pds:b:c:p::666.666").lid())
def file_to_filepath(file: File) -> str:
dir = lidvid_to_dirpath(LIDVID(file.product_lidvid))
return fs.path.relpath(fs.path.join(dir, file.basename))
def inventory_to_filepath(collection_inventory: CollectionInventory) -> str:
dir = lidvid_to_dirpath(LIDVID(collection_inventory.collection_lidvid))
return fs.path.relpath(fs.path.join(dir, collection_inventory.basename))
def label_to_filepath(product_label: ProductLabel) -> str:
dir = lidvid_to_dirpath(LIDVID(product_label.product_lidvid))
return fs.path.relpath(fs.path.join(dir, product_label.basename))
def test_create_from_lid_and_vid(self) -> None:
lid = LID("urn:nasa:pds:ssc01.hirespc.cruise:browse")
vid = VID("2.5")
lidvid = LIDVID.create_from_lid_and_vid(lid, vid)
self.assertEqual(
LIDVID("urn:nasa:pds:ssc01.hirespc.cruise:browse::2.5"), lidvid)
def plain_lidvid_to_dirpath(lidvid: LIDVID) -> str:
lid = lidvid.lid()
parts = lid.parts()[1:]
return fs.path.join("/", *parts)
def test_lidvid_lid_vid_properties(self, lid: LID, vid: VID) -> None:
lidvid = LIDVID.create_from_lid_and_vid(lid, vid)
self.assertEqual(lidvid.lid(), lid)
self.assertEqual(lidvid.vid(), vid)
def label_to_filepath(bundle_label: BundleLabel) -> str:
dir = lidvid_to_dirpath(LIDVID(bundle_label.bundle_lidvid))
return fs.path.relpath(fs.path.join(dir, bundle_label.basename))
def test_repr(self) -> None:
self.assertEqual("LIDVID('urn:nasa:pds:b:c:p::1.0')",
repr(LIDVID("urn:nasa:pds:b:c:p::1.0")))
