def _run(self) -> None:
working_dir: str = self.working_dir()
archive_dir: str = self.archive_dir()
deliverable_dir: str = self.deliverable_dir()
manifest_dir: str = self.manifest_dir()
try:
PDS_LOGGER.open("Create deliverable directory")
if os.path.isdir(deliverable_dir):
raise ValueError(
f"{deliverable_dir} cannot exist for MakeDeliverable.")
changes_path = os.path.join(working_dir, CHANGES_DICT_NAME)
changes_dict = read_changes_dict(changes_path)
with make_osfs(archive_dir) as archive_osfs, make_multiversioned(
archive_osfs) as mv:
bundle_segment = self._bundle_segment
bundle_lid = LID.create_from_parts([bundle_segment])
bundle_vid = changes_dict.vid(bundle_lid)
bundle_lidvid = LIDVID.create_from_lid_and_vid(
bundle_lid, bundle_vid)
version_view = VersionView(mv, bundle_lidvid)
synth_files: Dict[str, bytes] = dict()
# open the database
db_filepath = fs.path.join(working_dir, _BUNDLE_DB_NAME)
bundle_db = create_bundle_db_from_os_filepath(db_filepath)
bundle_lidvid_str = str(bundle_lidvid)
synth_files = dict()
cm = make_checksum_manifest(bundle_db, bundle_lidvid_str,
short_lidvid_to_dirpath)
synth_files["/checksum.manifest.txt"] = cm.encode("utf-8")
tm = make_transfer_manifest(bundle_db, bundle_lidvid_str,
short_lidvid_to_dirpath)
synth_files["/transfer.manifest.txt"] = tm.encode("utf-8")
deliverable_view = DeliverableView(version_view, synth_files)
os.mkdir(deliverable_dir)
deliverable_osfs = OSFS(deliverable_dir)
copy_fs(deliverable_view, deliverable_osfs)
PDS_LOGGER.log("info", f"Deliverable: {deliverable_dir}")
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()
def _run(self) -> None:
working_dir: str = self.working_dir()
archive_dir: str = self.archive_dir()
archive_primary_deltas_dir: str = self.archive_primary_deltas_dir()
if os.path.isdir(self.deliverable_dir()):
raise ValueError(
f"{self.deliverable_dir()} cannot exist for PopulateDatabase.")
changes_path = os.path.join(working_dir, CHANGES_DICT_NAME)
changes_dict = read_changes_dict(changes_path)
bundle_lid = LID.create_from_parts([self._bundle_segment])
first_round = changes_dict.vid(bundle_lid) == VID("1.0")
schema_collection_lid = LID.create_from_parts(
[self._bundle_segment, "schema"])
changes_dict.set(schema_collection_lid, VID("1.0"), first_round)
write_changes_dict(changes_dict, changes_path)
db_filepath = os.path.join(working_dir, _BUNDLE_DB_NAME)
db_exists = os.path.isfile(db_filepath)
db = create_bundle_db_from_os_filepath(db_filepath)
with make_osfs(archive_dir) as archive_osfs, make_version_view(
archive_osfs,
self._bundle_segment) as version_view, make_sv_deltas(
version_view, archive_primary_deltas_dir) as sv_deltas:
if not db_exists:
db.create_tables()
documents_dir = f"/{self._bundle_segment}$/document$/phase2$"
docs = set(sv_deltas.listdir(documents_dir))
# Pass this to create citation info db in _populate_citation_info
info_param: Tuple = (sv_deltas, documents_dir, docs)
bundle_lidvid = _populate_bundle(changes_dict, db)
_populate_collections(changes_dict, db)
_populate_products(changes_dict, db, sv_deltas)
_populate_target_identification(changes_dict, db, sv_deltas)
_populate_citation_info(changes_dict, db, info_param)
if not db:
raise ValueError("db doesn't exist.")
if not os.path.isfile(db_filepath):
raise ValueError(f"{db_filepath} is not a file.")
def _run(self) -> None:
def change_fits_file(rel_path: str) -> None:
abs_path = fs.path.join(self.mast_downloads_dir(),
fs.path.relpath(rel_path))
from touch_fits import touch_fits
try:
PDS_LOGGER.open("Change fits file")
PDS_LOGGER.log("info", f"Touching {abs_path}")
touch_fits(abs_path)
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()
with make_osfs(self.mast_downloads_dir()) as mast_fs:
def _change_fits_file() -> None:
which_file = 0
PDS_LOGGER.open("Change fits file")
for path in mast_fs.walk.files(filter=["*.fits"]):
# change only the n-th FITS file then return
if which_file == 0:
change_fits_file(path)
PDS_LOGGER.log("info", f"CHANGED {path}")
PDS_LOGGER.close()
return
which_file = which_file - 1
raise RuntimeError(
"Fell off the end of change_fits_file in ChangeFiles.")
def _delete_directory() -> None:
PDS_LOGGER.open("Delete directory")
for path in mast_fs.walk.dirs():
if len(fs.path.parts(path)) == 3:
PDS_LOGGER.log("info", f"REMOVED {path}")
mast_fs.removetree(path)
PDS_LOGGER.close()
return
raise RuntimeError(
"Fell off the end of delete_directory in ChangeFiles.")
# _change_fits_file()
_delete_directory()
def _run(self) -> None:
working_dir: str = self.working_dir()
archive_dir: str = self.archive_dir()
archive_primary_deltas_dir: str = self.archive_primary_deltas_dir()
archive_browse_deltas_dir: str = self.archive_browse_deltas_dir()
archive_label_deltas_dir: str = self.archive_label_deltas_dir()
if os.path.isdir(self.deliverable_dir()):
raise ValueError(
f"{self.deliverable_dir()} cannot exist for UpdateArchive."
)
with make_osfs(archive_dir) as archive_osfs, make_version_view(
archive_osfs, self._bundle_segment
) as version_view, make_sv_deltas(
version_view, archive_primary_deltas_dir
) as sv_deltas, make_sv_deltas(
sv_deltas, archive_browse_deltas_dir
) as browse_deltas, make_sv_deltas(
browse_deltas, archive_label_deltas_dir
) as label_deltas:
# TODO I *think* this is a hack and will only work for the
# initial import...but maybe I accidentally wrote better code
# than I think and it'll work for all cases. Investigate.
mv = Multiversioned(archive_osfs)
mv.update_from_single_version(std_is_new, label_deltas)
shutil.rmtree(archive_primary_deltas_dir + "-deltas-sv")
shutil.rmtree(archive_browse_deltas_dir + "-deltas-sv")
shutil.rmtree(archive_label_deltas_dir + "-deltas-sv")
if os.path.isdir(archive_primary_deltas_dir + "-deltas-sv"):
raise ValueError(f"{archive_primary_deltas_dir}-deltas-sv shouldn't exist.")
if os.path.isdir(archive_browse_deltas_dir + "-deltas-sv"):
raise ValueError(f"{archive_browse_deltas_dir}-deltas-sv shouldn't exist.")
if os.path.isdir(archive_label_deltas_dir + "-deltas-sv"):
raise ValueError(f"{archive_label_deltas_dir}-deltas-sv shouldn't exist.")
if not os.path.isdir(archive_dir):
raise ValueError(f"{archive_dir} doesn't exist.")
def _run(self) -> None:
working_dir: str = self.working_dir()
primary_files_dir: str = self.primary_files_dir()
archive_dir: str = self.archive_dir()
if os.path.isdir(self.deliverable_dir()):
raise ValueError(
f"{self.deliverable_dir()} cannot exist " + "for RecordChanges"
)
if not os.path.isdir(working_dir):
raise ValueError(f"{working_dir} doesn't exist.")
if not os.path.isdir(primary_files_dir + "-sv"):
raise ValueError(f"{primary_files_dir}-sv doesn't exist.")
changes: Dict[LIDVID, bool] = dict()
changes_path = os.path.join(working_dir, CHANGES_DICT_NAME)
with make_osfs(archive_dir) as archive_osfs, make_version_view(
archive_osfs, self._bundle_segment
) as latest_version:
with make_sv_osfs(primary_files_dir) as primary_fs:
mv = Multiversioned(archive_osfs)
d = _get_primary_changes(mv, primary_fs, latest_version)
write_changes_dict(d, changes_path)
if not d.has_changes():
print("#### PRIMARY_FS ################")
primary_fs.tree()
print("#### LATEST_VERSION ################")
latest_version.tree()
if not os.path.isdir(primary_files_dir + "-sv"):
raise ValueError(f"{primary_files_dir}-sv doesn't exist.")
if not os.path.isfile(os.path.join(working_dir, CHANGES_DICT_NAME)):
raise ValueError(
f"{os.path.join(working_dir, CHANGES_DICT_NAME)} " + "doesn't exist."
)
def _run(self) -> None:
working_dir: str = self.working_dir()
primary_files_dir: str = self.primary_files_dir()
archive_dir: str = self.archive_dir()
archive_primary_deltas_dir: str = self.archive_primary_deltas_dir()
try:
PDS_LOGGER.open(
"Create a directory for a new version of the bundle")
if os.path.isdir(self.deliverable_dir()):
raise ValueError(
f"{self.deliverable_dir()} cannot exist for InsertChanges."
)
changes_path = os.path.join(working_dir, CHANGES_DICT_NAME)
with make_osfs(archive_dir) as archive_osfs, make_version_view(
archive_osfs, self._bundle_segment
) as version_view, make_sv_osfs(
primary_files_dir) as primary_files_osfs, make_sv_deltas(
version_view, archive_primary_deltas_dir) as sv_deltas:
archive_dirs = list(archive_osfs.walk.dirs())
changes_dict = read_changes_dict(changes_path)
_merge_primaries(changes_dict, primary_files_osfs, sv_deltas)
shutil.rmtree(primary_files_dir + "-sv")
if not os.path.isdir(archive_dir):
raise ValueError(f"{archive_dir} doesn't exist.")
dirpath = archive_primary_deltas_dir + "-deltas-sv"
PDS_LOGGER.log("info", f"Directory for the new version: {dirpath}")
if not os.path.isdir(dirpath):
raise ValueError(f"{dirpath} doesn't exist.")
if not os.path.isfile(changes_path):
raise ValueError(f"{changes_path} is not a file.")
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()
def _run(self) -> None:
working_dir: str = self.working_dir()
archive_dir: str = self.archive_dir()
archive_primary_deltas_dir: str = self.archive_primary_deltas_dir()
archive_browse_deltas_dir: str = self.archive_browse_deltas_dir()
archive_label_deltas_dir: str = self.archive_label_deltas_dir()
if os.path.isdir(self.deliverable_dir()):
raise ValueError(
f"{self.deliverable_dir()} cannot exist for BuildLabels.")
changes_path = fs.path.join(working_dir, CHANGES_DICT_NAME)
changes_dict = read_changes_dict(changes_path)
with make_osfs(archive_dir) as archive_osfs, make_version_view(
archive_osfs,
self._bundle_segment) as version_view, make_sv_deltas(
version_view,
archive_primary_deltas_dir) as sv_deltas, make_sv_deltas(
sv_deltas, archive_browse_deltas_dir
) as browse_deltas, make_sv_deltas(
browse_deltas,
archive_label_deltas_dir) as label_deltas:
changes_path = fs.path.join(working_dir, CHANGES_DICT_NAME)
changes_dict = read_changes_dict(changes_path)
# open the database
db_filepath = fs.path.join(working_dir, _BUNDLE_DB_NAME)
db = create_bundle_db_from_os_filepath(db_filepath)
# create labels
bundle_lid = LID.create_from_parts([self._bundle_segment])
bundle_vid = changes_dict.vid(bundle_lid)
bundle_lidvid = LIDVID.create_from_lid_and_vid(
bundle_lid, bundle_vid)
documents_dir = f"/{self._bundle_segment}$/document$/phase2$"
docs = set(sv_deltas.listdir(documents_dir))
# fetch citation info from database
citation_info_from_db = db.get_citation(str(bundle_lidvid))
info = Citation_Information(
citation_info_from_db.filename,
citation_info_from_db.propno,
citation_info_from_db.category,
citation_info_from_db.cycle,
citation_info_from_db.authors.split(","),
citation_info_from_db.title,
citation_info_from_db.submission_year,
citation_info_from_db.timing_year,
citation_info_from_db.abstract.split("\n"),
)
info.set_publication_year(PUBLICATION_YEAR)
try:
PDS_LOGGER.open("BuildLabels")
# create_pds4_labels() may change changes_dict, because we
# create the context collection if it doesn't exist.
create_pds4_labels(working_dir, db, bundle_lidvid,
changes_dict, label_deltas, info)
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()
write_changes_dict(changes_dict, changes_path)
def _run(self) -> None:
try:
PDS_LOGGER.open("BuildBrowse")
PDS_LOGGER.log("info", "Entering BuildBrowse.")
working_dir: str = self.working_dir()
archive_dir: str = self.archive_dir()
archive_primary_deltas_dir: str = self.archive_primary_deltas_dir()
archive_browse_deltas_dir: str = self.archive_browse_deltas_dir()
if os.path.isdir(self.deliverable_dir()):
raise ValueError(f"{self.deliverable_dir()} cannot exist " +
"for BuildBrowse.")
changes_path = os.path.join(working_dir, CHANGES_DICT_NAME)
changes_dict = read_changes_dict(changes_path)
db_filepath = os.path.join(working_dir, _BUNDLE_DB_NAME)
db = create_bundle_db_from_os_filepath(db_filepath)
bundle_lid = LID.create_from_parts([self._bundle_segment])
bundle_vid = changes_dict.vid(bundle_lid)
bundle_lidvid = LIDVID.create_from_lid_and_vid(
bundle_lid, bundle_vid)
with make_osfs(archive_dir) as archive_osfs, make_version_view(
archive_osfs,
self._bundle_segment) as version_view, make_sv_deltas(
version_view, archive_primary_deltas_dir
) as sv_deltas, make_sv_deltas(
sv_deltas, archive_browse_deltas_dir) as browse_deltas:
bundle_path = f"/{self._bundle_segment}$/"
collection_segments = [
str(coll[:-1])
for coll in browse_deltas.listdir(bundle_path)
if "$" in coll
]
for collection_segment in collection_segments:
collection_lid = LID.create_from_parts(
[self._bundle_segment, collection_segment])
if _requires_browse_collection(collection_segment):
collection_vid = changes_dict.vid(collection_lid)
collection_lidvid = LIDVID.create_from_lid_and_vid(
collection_lid, collection_vid)
if changes_dict.changed(collection_lid):
PDS_LOGGER.log(
"info",
f"Making browse for {collection_lidvid}")
_build_browse_collection(
db,
changes_dict,
browse_deltas,
bundle_lidvid,
collection_lidvid,
bundle_path,
)
else:
_fill_in_old_browse_collection(
db, changes_dict, bundle_lidvid,
collection_lidvid)
write_changes_dict(changes_dict, changes_path)
PDS_LOGGER.log("info", "Leaving BuildBrowse.")
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()