def _fill_in_old_browse_collection(
db: BundleDB,
changes_dict: ChangesDict,
bundle_lidvid: LIDVID,
data_collection_lidvid: LIDVID,
) -> None:
bundle_segment = bundle_lidvid.lid().parts()[0]
collection_segment = data_collection_lidvid.lid().parts()[1]
browse_collection_lid = data_collection_lidvid.lid().to_browse_lid()
browse_collection_segment = browse_collection_lid.collection_id
browse_collection_vid = data_collection_lidvid.vid()
browse_collection_lidvid = LIDVID.create_from_lid_and_vid(
browse_collection_lid, browse_collection_vid)
changes_dict.set(browse_collection_lid, browse_collection_vid, False)
db.create_bundle_collection_link(str(bundle_lidvid),
str(browse_collection_lidvid))
try:
PDS_LOGGER.open("Fill in old browse collection")
PDS_LOGGER.log(
"info", f"Created link and change for {browse_collection_lidvid}")
for product in db.get_collection_products(
str(browse_collection_lidvid)):
product_lidvid = LIDVID(product.lidvid)
changes_dict.set(product_lidvid.lid(), product_lidvid.vid(), False)
PDS_LOGGER.log("info",
f"Created link and change for {product_lidvid}")
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()
def _create_context_collection(self, bundle: Bundle) -> None:
context_products = bundle_db.get_context_products()
if not context_products:
return
bundle_lidvid = str(bundle.lidvid)
collection_lidvid = _extend_lidvid(bundle_lidvid, "context")
bundle_db.create_context_collection(collection_lidvid,
bundle_lidvid)
clv = LIDVID(collection_lidvid)
changes_dict.set(clv.lid(), clv.vid(), True)
bundle_dir_path = _lidvid_to_dir(bundle_lidvid)
context_coll_dir_path = fs.path.join(bundle_dir_path, "context$")
label_deltas.makedir(context_coll_dir_path)
# Create target lable if it doesn't exist in PDS page.
self._create_context_target_label(context_coll_dir_path,
collection_lidvid)
# Create investigation label
self._create_context_investigation_label(bundle_lidvid,
context_coll_dir_path,
collection_lidvid)
collection = bundle_db.get_collection(collection_lidvid)
self._post_visit_collection(collection)
def __setitem__(self, lidvid: LIDVID, contents: VersionContents) -> None:
if lidvid in self:
# illegal to set twice
raise IndexError(f"Repeated setting at {lidvid}")
# check that contents aren't contradictory
lidvid_parts = lidvid.lid().parts()
d = {}
for sub_lidvid in contents.lidvids():
sub_parts = sub_lidvid.lid().parts()
d[sub_parts[-1]] = str(sub_lidvid.vid())
if sub_parts[:-1] != lidvid_parts:
raise ValueError(f"LIDVID {sub_lidvid} in contents cannot be "
f"a child of index LIDVID {lidvid}")
lidvid_dir = self.make_lidvid_dir(lidvid)
if d:
write_subdir_versions_to_directory(self.fs, lidvid_dir, d)
for src_filepath in contents.filepaths:
dst_filepath = fs.path.join(lidvid_dir,
fs.path.relpath(src_filepath))
self.fs.makedirs(fs.path.dirname(dst_filepath), None, True)
fs.copy.copy_file(contents.fs, src_filepath, self.fs, dst_filepath)
if not self.fs.isdir(lidvid_path(lidvid)):
raise ValueError(f"Lidvid path for {lidvid} is not a directory.")
if lidvid not in self:
raise IndexError(f"{lidvid} is not set in the wrapped filesystem.")
def plain_lidvid_to_visits_dirpath(lidvid: LIDVID) -> str:
# This is only run on directories, and always relative to /hst_NNNNN.
lid = lidvid.lid()
parts = lid.parts()[1:]
if len(parts) == 2:
visit = _visit_of(parts[1])
if visit is not None:
parts[1] = visit
return fs.path.join("/", *parts)
def parent_lidvid(self, lidvid: LIDVID) -> LIDVID:
lid = lidvid.lid()
if lid not in self.changes_dict:
raise KeyError(f"lid={lid} not in changes_dict.")
parent_lid = lid.parent_lid()
if parent_lid not in self.changes_dict:
raise KeyError(f"parent_lid={parent_lid} not in changes_dict.")
parent_vid = self.vid(parent_lid)
return LIDVID.create_from_lid_and_vid(parent_lid, parent_vid)
def short_lidvid_to_dirpath(lidvid: LIDVID) -> str:
lid = lidvid.lid()
# parts are collection, product
parts = lid.parts()[1:]
if len(parts) >= 2 and parts[0] not in NO_VISIT_COLLECTIONS:
fake_filename = f"{parts[1]}_raw.fits"
visit = HstFilename(fake_filename).visit()
visit_part = f"visit_{visit}"
parts[1] = visit_part
return fs.path.join(*parts)
def label_to_filepath(context_product_label: ContextProductLabel) -> str:
# This is a workaround because lidvid under context doesn't
# match the directory path. For example:
# target lidvid: urn:nasa:pds:context:target:asteroid.762_pulcova::1.0
# path: hst_09059/context/asteroid.762_pulcova_1.0.xml
lidvid_mod = str(context_product_label.collection_lidvid)
dir = lidvid_to_dirpath(LIDVID(lidvid_mod))
lidvid = LIDVID(lidvid_mod)
lid = lidvid.lid()
# parts are collection, product
parts = lid.parts()[1:]
return fs.path.relpath(fs.path.join(dir, context_product_label.basename))
def std_is_new(lidvid: LIDVID, contents: VersionContents,
mv: "Multiversioned") -> bool:
"""
The standard IS_NEW_TEST function. If it's a document collection,
it checks the document files for changes; otherwise, it checks
only FITS files.
"""
lid = lidvid.lid()
if lid.is_collection_lid() and lid.collection_id == "document":
filt = doc_filter
else:
filt = fits_filter
return mv[lidvid].filter_filepaths(filt) != contents
def lidvid_path(lidvid: LIDVID) -> str:
"""
Return the directory path corresponding to this LIDVID.
"""
def vid_to_dir_part(vid: VID) -> str:
"""
Convert a VID to a directory name.
"""
return f"v${vid}"
lid = lidvid.lid()
dir = lid_path(lid)
vid_bit = vid_to_dir_part(lidvid.vid())
return fs.path.join(dir, vid_bit)
def plain_lidvid_to_dirpath(lidvid: LIDVID) -> str:
lid = lidvid.lid()
parts = lid.parts()[1:]
return fs.path.join("/", *parts)
def test_is_product_lidvid_property(self, lidvid: LIDVID) -> None:
self.assertEqual(lidvid.is_product_lidvid(),
lidvid.lid().is_product_lid())
def test_is_collection_lidvid_property(self, lidvid: LIDVID) -> None:
self.assertEqual(lidvid.is_collection_lidvid(),
lidvid.lid().is_collection_lid())
def test_is_next_minor_lidvid_property(self, lidvid: LIDVID) -> None:
self.assertEqual(
LIDVID.create_from_lid_and_vid(lidvid.lid(),
lidvid.vid().next_minor_vid()),
lidvid.next_minor_lidvid(),
)
def _extend_lidvid(lidvid_str: str, segment: str) -> str:
lidvid = LIDVID(lidvid_str)
lid = lidvid.lid().extend_lid(segment)
new_lidvid = LIDVID.create_from_lid_and_vid(lid, lidvid.vid())
return str(new_lidvid)
def _build_browse_collection(
db: BundleDB,
changes_dict: ChangesDict,
browse_deltas: COWFS,
bundle_lidvid: LIDVID,
data_collection_lidvid: LIDVID,
bundle_path: str,
) -> None:
bundle_segment = bundle_lidvid.lid().parts()[0]
collection_segment = data_collection_lidvid.lid().parts()[1]
browse_collection_lid = data_collection_lidvid.lid().to_browse_lid()
collection_path = f"{bundle_path}{collection_segment}$/"
browse_collection_segment = browse_collection_lid.collection_id
browse_collection_path = f"{bundle_path}{browse_collection_segment}$/"
browse_collection_vid = data_collection_lidvid.vid()
browse_collection_lidvid = LIDVID.create_from_lid_and_vid(
browse_collection_lid, browse_collection_vid)
changes_dict.set(browse_collection_lid, browse_collection_vid, True)
browse_deltas.makedirs(browse_collection_path, recreate=True)
db.create_other_collection(str(browse_collection_lidvid),
str(bundle_lidvid))
db.create_bundle_collection_link(str(bundle_lidvid),
str(browse_collection_lidvid))
product_segments = [
str(prod[:-1]) for prod in browse_deltas.listdir(collection_path)
if "$" in prod
]
for product_segment in product_segments:
# These product_segments are from the data_collection
product_lid = LID.create_from_parts(
[bundle_segment, collection_segment, product_segment])
product_vid = changes_dict.vid(product_lid)
product_path = f"{collection_path}{product_segment}$/"
browse_product_path = f"{browse_collection_path}{product_segment}$/"
browse_product_lidvid = _extend_lidvid(browse_collection_lid,
product_vid, product_segment)
if changes_dict.changed(product_lid):
fits_product_lidvid = _extend_lidvid(
data_collection_lidvid.lid(),
data_collection_lidvid.vid(),
product_segment,
)
bpl = LIDVID(browse_product_lidvid)
changes_dict.set(bpl.lid(), bpl.vid(), True)
browse_deltas.makedirs(browse_product_path, recreate=True)
db.create_browse_product(
browse_product_lidvid,
fits_product_lidvid,
str(browse_collection_lidvid),
)
db.create_collection_product_link(str(browse_collection_lidvid),
browse_product_lidvid)
for fits_file in browse_deltas.listdir(product_path):
fits_filepath = fs.path.join(product_path, fits_file)
fits_os_filepath = browse_deltas.getsyspath(fits_filepath)
browse_file = fs.path.splitext(fits_file)[0] + ".jpg"
browse_filepath = fs.path.join(browse_product_path,
browse_file)
# In a COWFS, a directory does not have a
# syspath, only files. So we write a stub
# file into the directory, find its syspath
# and its directory's syspath. Then we remove
# the stub file.
browse_deltas.touch(browse_filepath)
browse_product_os_filepath = browse_deltas.getsyspath(
browse_filepath)
browse_deltas.remove(browse_filepath)
browse_product_os_dirpath = fs.path.dirname(
browse_product_os_filepath)
# Picmaker expects a list of strings. If you give it
# str, it'll index into it and complain about '/'
# not being a file. So don't do that!
try:
picmaker.ImagesToPics(
[str(fits_os_filepath)],
browse_product_os_dirpath,
filter="None",
percentiles=(1, 99),
)
except IndexError as e:
tb = traceback.format_exc()
message = f"File {fits_file}: {e}\n{tb}"
raise Exception(message)
browse_os_filepath = fs.path.join(browse_product_os_dirpath,
browse_file)
size = os.stat(browse_os_filepath).st_size
db.create_browse_file(browse_os_filepath, browse_file,
browse_product_lidvid, size)
else:
bpl = LIDVID(browse_product_lidvid)
changes_dict.set(bpl.lid(), bpl.vid(), False)
db.create_collection_product_link(str(browse_collection_lidvid),
browse_product_lidvid)
def _lidvid_to_dirpath(lidvid: LIDVID) -> str:
lid = lidvid.lid()
return fs.path.relpath(lid_to_dir(lid))