def _run(self) -> None:
working_dir: str = self.working_dir()
archive_dir: str = self.archive_dir()
archive_primary_deltas_dir: str = self.archive_primary_deltas_dir()
archive_browse_deltas_dir: str = self.archive_browse_deltas_dir()
archive_label_deltas_dir: str = self.archive_label_deltas_dir()
if os.path.isdir(self.deliverable_dir()):
raise ValueError(
f"{self.deliverable_dir()} cannot exist for UpdateArchive."
)
with make_osfs(archive_dir) as archive_osfs, make_version_view(
archive_osfs, self._bundle_segment
) as version_view, make_sv_deltas(
version_view, archive_primary_deltas_dir
) as sv_deltas, make_sv_deltas(
sv_deltas, archive_browse_deltas_dir
) as browse_deltas, make_sv_deltas(
browse_deltas, archive_label_deltas_dir
) as label_deltas:
# TODO I *think* this is a hack and will only work for the
# initial import...but maybe I accidentally wrote better code
# than I think and it'll work for all cases. Investigate.
mv = Multiversioned(archive_osfs)
mv.update_from_single_version(std_is_new, label_deltas)
shutil.rmtree(archive_primary_deltas_dir + "-deltas-sv")
shutil.rmtree(archive_browse_deltas_dir + "-deltas-sv")
shutil.rmtree(archive_label_deltas_dir + "-deltas-sv")
if os.path.isdir(archive_primary_deltas_dir + "-deltas-sv"):
raise ValueError(f"{archive_primary_deltas_dir}-deltas-sv shouldn't exist.")
if os.path.isdir(archive_browse_deltas_dir + "-deltas-sv"):
raise ValueError(f"{archive_browse_deltas_dir}-deltas-sv shouldn't exist.")
if os.path.isdir(archive_label_deltas_dir + "-deltas-sv"):
raise ValueError(f"{archive_label_deltas_dir}-deltas-sv shouldn't exist.")
if not os.path.isdir(archive_dir):
raise ValueError(f"{archive_dir} doesn't exist.")
def _run(self) -> None:
working_dir: str = self.working_dir()
archive_dir: str = self.archive_dir()
archive_primary_deltas_dir: str = self.archive_primary_deltas_dir()
if os.path.isdir(self.deliverable_dir()):
raise ValueError(
f"{self.deliverable_dir()} cannot exist for PopulateDatabase.")
changes_path = os.path.join(working_dir, CHANGES_DICT_NAME)
changes_dict = read_changes_dict(changes_path)
bundle_lid = LID.create_from_parts([self._bundle_segment])
first_round = changes_dict.vid(bundle_lid) == VID("1.0")
schema_collection_lid = LID.create_from_parts(
[self._bundle_segment, "schema"])
changes_dict.set(schema_collection_lid, VID("1.0"), first_round)
write_changes_dict(changes_dict, changes_path)
db_filepath = os.path.join(working_dir, _BUNDLE_DB_NAME)
db_exists = os.path.isfile(db_filepath)
db = create_bundle_db_from_os_filepath(db_filepath)
with make_osfs(archive_dir) as archive_osfs, make_version_view(
archive_osfs,
self._bundle_segment) as version_view, make_sv_deltas(
version_view, archive_primary_deltas_dir) as sv_deltas:
if not db_exists:
db.create_tables()
documents_dir = f"/{self._bundle_segment}$/document$/phase2$"
docs = set(sv_deltas.listdir(documents_dir))
# Pass this to create citation info db in _populate_citation_info
info_param: Tuple = (sv_deltas, documents_dir, docs)
bundle_lidvid = _populate_bundle(changes_dict, db)
_populate_collections(changes_dict, db)
_populate_products(changes_dict, db, sv_deltas)
_populate_target_identification(changes_dict, db, sv_deltas)
_populate_citation_info(changes_dict, db, info_param)
if not db:
raise ValueError("db doesn't exist.")
if not os.path.isfile(db_filepath):
raise ValueError(f"{db_filepath} is not a file.")
def _run(self) -> None:
working_dir: str = self.working_dir()
primary_files_dir: str = self.primary_files_dir()
archive_dir: str = self.archive_dir()
archive_primary_deltas_dir: str = self.archive_primary_deltas_dir()
try:
PDS_LOGGER.open(
"Create a directory for a new version of the bundle")
if os.path.isdir(self.deliverable_dir()):
raise ValueError(
f"{self.deliverable_dir()} cannot exist for InsertChanges."
)
changes_path = os.path.join(working_dir, CHANGES_DICT_NAME)
with make_osfs(archive_dir) as archive_osfs, make_version_view(
archive_osfs, self._bundle_segment
) as version_view, make_sv_osfs(
primary_files_dir) as primary_files_osfs, make_sv_deltas(
version_view, archive_primary_deltas_dir) as sv_deltas:
archive_dirs = list(archive_osfs.walk.dirs())
changes_dict = read_changes_dict(changes_path)
_merge_primaries(changes_dict, primary_files_osfs, sv_deltas)
shutil.rmtree(primary_files_dir + "-sv")
if not os.path.isdir(archive_dir):
raise ValueError(f"{archive_dir} doesn't exist.")
dirpath = archive_primary_deltas_dir + "-deltas-sv"
PDS_LOGGER.log("info", f"Directory for the new version: {dirpath}")
if not os.path.isdir(dirpath):
raise ValueError(f"{dirpath} doesn't exist.")
if not os.path.isfile(changes_path):
raise ValueError(f"{changes_path} is not a file.")
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()
def _run(self) -> None:
working_dir: str = self.working_dir()
archive_dir: str = self.archive_dir()
archive_primary_deltas_dir: str = self.archive_primary_deltas_dir()
archive_browse_deltas_dir: str = self.archive_browse_deltas_dir()
archive_label_deltas_dir: str = self.archive_label_deltas_dir()
if os.path.isdir(self.deliverable_dir()):
raise ValueError(
f"{self.deliverable_dir()} cannot exist for BuildLabels.")
changes_path = fs.path.join(working_dir, CHANGES_DICT_NAME)
changes_dict = read_changes_dict(changes_path)
with make_osfs(archive_dir) as archive_osfs, make_version_view(
archive_osfs,
self._bundle_segment) as version_view, make_sv_deltas(
version_view,
archive_primary_deltas_dir) as sv_deltas, make_sv_deltas(
sv_deltas, archive_browse_deltas_dir
) as browse_deltas, make_sv_deltas(
browse_deltas,
archive_label_deltas_dir) as label_deltas:
changes_path = fs.path.join(working_dir, CHANGES_DICT_NAME)
changes_dict = read_changes_dict(changes_path)
# open the database
db_filepath = fs.path.join(working_dir, _BUNDLE_DB_NAME)
db = create_bundle_db_from_os_filepath(db_filepath)
# create labels
bundle_lid = LID.create_from_parts([self._bundle_segment])
bundle_vid = changes_dict.vid(bundle_lid)
bundle_lidvid = LIDVID.create_from_lid_and_vid(
bundle_lid, bundle_vid)
documents_dir = f"/{self._bundle_segment}$/document$/phase2$"
docs = set(sv_deltas.listdir(documents_dir))
# fetch citation info from database
citation_info_from_db = db.get_citation(str(bundle_lidvid))
info = Citation_Information(
citation_info_from_db.filename,
citation_info_from_db.propno,
citation_info_from_db.category,
citation_info_from_db.cycle,
citation_info_from_db.authors.split(","),
citation_info_from_db.title,
citation_info_from_db.submission_year,
citation_info_from_db.timing_year,
citation_info_from_db.abstract.split("\n"),
)
info.set_publication_year(PUBLICATION_YEAR)
try:
PDS_LOGGER.open("BuildLabels")
# create_pds4_labels() may change changes_dict, because we
# create the context collection if it doesn't exist.
create_pds4_labels(working_dir, db, bundle_lidvid,
changes_dict, label_deltas, info)
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()
write_changes_dict(changes_dict, changes_path)
def _run(self) -> None:
try:
PDS_LOGGER.open("BuildBrowse")
PDS_LOGGER.log("info", "Entering BuildBrowse.")
working_dir: str = self.working_dir()
archive_dir: str = self.archive_dir()
archive_primary_deltas_dir: str = self.archive_primary_deltas_dir()
archive_browse_deltas_dir: str = self.archive_browse_deltas_dir()
if os.path.isdir(self.deliverable_dir()):
raise ValueError(f"{self.deliverable_dir()} cannot exist " +
"for BuildBrowse.")
changes_path = os.path.join(working_dir, CHANGES_DICT_NAME)
changes_dict = read_changes_dict(changes_path)
db_filepath = os.path.join(working_dir, _BUNDLE_DB_NAME)
db = create_bundle_db_from_os_filepath(db_filepath)
bundle_lid = LID.create_from_parts([self._bundle_segment])
bundle_vid = changes_dict.vid(bundle_lid)
bundle_lidvid = LIDVID.create_from_lid_and_vid(
bundle_lid, bundle_vid)
with make_osfs(archive_dir) as archive_osfs, make_version_view(
archive_osfs,
self._bundle_segment) as version_view, make_sv_deltas(
version_view, archive_primary_deltas_dir
) as sv_deltas, make_sv_deltas(
sv_deltas, archive_browse_deltas_dir) as browse_deltas:
bundle_path = f"/{self._bundle_segment}$/"
collection_segments = [
str(coll[:-1])
for coll in browse_deltas.listdir(bundle_path)
if "$" in coll
]
for collection_segment in collection_segments:
collection_lid = LID.create_from_parts(
[self._bundle_segment, collection_segment])
if _requires_browse_collection(collection_segment):
collection_vid = changes_dict.vid(collection_lid)
collection_lidvid = LIDVID.create_from_lid_and_vid(
collection_lid, collection_vid)
if changes_dict.changed(collection_lid):
PDS_LOGGER.log(
"info",
f"Making browse for {collection_lidvid}")
_build_browse_collection(
db,
changes_dict,
browse_deltas,
bundle_lidvid,
collection_lidvid,
bundle_path,
)
else:
_fill_in_old_browse_collection(
db, changes_dict, bundle_lidvid,
collection_lidvid)
write_changes_dict(changes_dict, changes_path)
PDS_LOGGER.log("info", "Leaving BuildBrowse.")
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()