def copy_fs(version_view: FS, deliverable: FS) -> None:
# TODO The note below is probably obsolete. It was written when
# we were using the DeliverableFS to make the deliverable. We've
# now switched to using the DeliverableView. (The obsolete
# DeliverableFS has been deleted from the repo.) In any case,
# this doesn't seem to hurt anything, but it can probably all be
# replaced by fs.copy.copy_file()--see the last line. Try it and
# see.
# TODO I could (and used to) just do a fs.copy.copy_fs() from the
# version_view to a DeliverableFS. I removed it to debug issues
# with the validation tool. Now I find this hack is just as easy
# (though I wonder about efficiency). It bothers me that this
# visits hack parallels a visits hack in
# plain_lidvid_to_visits_dirpath(). I should figure this out and
# make it clean. For now, though, this works.
# Uses dollar-terminated paths
for path, dirs, files in version_view.walk():
parts = fs.path.parts(path)
if len(parts) == 4:
if len(parts[3]) == 10:
visit = "visit_" + parts[3][4:6].lower() + "$"
parts[3] = visit
new_path = fs.path.join(*parts)
if not deliverable.isdir(new_path):
deliverable.makedir(new_path)
for file in files:
old_filepath = fs.path.join(path, file.name)
new_filepath = fs.path.join(new_path, file.name)
fs.copy.copy_file(version_view, old_filepath, deliverable,
new_filepath)
def _merge_primaries(changes_dict: ChangesDict, src_fs: FS,
dst_fs: FS) -> None:
# TODO Not sure that this hits all cases, including removal of
# files and directories. Think about it.
for dirpath in src_fs.walk.dirs(search="depth"):
if _is_component_path(dirpath):
lid = dirpath_to_lid(dirpath)
changed = changes_dict.changed(lid)
if changed:
if not dst_fs.isdir(dirpath):
dst_fs.makedirs(dirpath)
src_sub_fs = SubFS(src_fs, dirpath)
dst_sub_fs = SubFS(dst_fs, dirpath)
# delete directories in dst that don't exist in src
for subdirpath in dst_sub_fs.walk.dirs(search="depth"):
if not src_sub_fs.isdir(subdirpath):
dst_sub_fs.removetree(subdirpath)
# delete the files in the destination (if any)
for filepath in component_files(dst_fs, dirpath):
dst_sub_fs.remove(filepath)
# copy the new files across
src_sub_fs = SubFS(src_fs, dirpath)
for filepath in component_files(src_fs, dirpath):
fs.copy.copy_file(src_sub_fs, filepath, dst_sub_fs,
filepath)
def _get_primary_changes(
mv: Multiversioned, primary_fs: FS, latest_version_fs: FS
) -> ChangesDict:
"""
Walk through the two filesystems, when you find differences
between the two, if it involves primary files, note it in a
ChangesDict and return all the changes.
TODO This function was hacked together. It works but it's
possible that the check is quadratic in the size of the tree, not
linear. That would be bad. Review this function and fix it if
it's wrong. When done, remove this comment.
"""
result = ChangesDict()
def filter_to_primary_files(dir: str, filenames: Iterator[str]) -> Set[str]:
return {
filename for filename in filenames if _is_primary_file(join(dir, filename))
}
def filter_to_primary_dirs(dir: str, dirnames: Iterator[str]) -> Set[str]:
return {dirname for dirname in dirnames if _is_primary_dir(join(dir, dirname))}
def dirs_match(dirpath: str) -> bool:
primary_dirs = filter_to_primary_dirs(
dirpath,
(
relpath(dir)
for dir in SubFS(primary_fs, dirpath).walk.dirs()
if "$" in dir
),
)
latest_dirs = filter_to_primary_dirs(
dirpath,
(
relpath(dir)
for dir in SubFS(latest_version_fs, dirpath).walk.dirs()
if "$" in dir
),
)
PDS_LOGGER.open("Directory changes detected")
if primary_dirs == latest_dirs:
for dir in primary_dirs:
full_dirpath = join(dirpath, relpath(dir))
lid = dirpath_to_lid(full_dirpath)
if lid not in result.changes_dict:
raise KeyError(f"{lid} not in changes_dict.")
if result.changed(lid):
PDS_LOGGER.log(
"info", f"CHANGE DETECTED in {dirpath}: {lid} changed"
)
PDS_LOGGER.close()
return False
PDS_LOGGER.close()
return True
else:
# list of dirs does not match
added = primary_dirs - latest_dirs
removed = latest_dirs - primary_dirs
if added and removed:
PDS_LOGGER.log(
"info",
f"CHANGE DETECTED IN {dirpath}: added {added}; removed {removed}",
)
elif added:
PDS_LOGGER.log("info", f"CHANGE DETECTED IN {dirpath}: added {added}")
else: # removed
PDS_LOGGER.log(
"info", f"CHANGE DETECTED IN {dirpath}: removed {removed}"
)
PDS_LOGGER.close()
return False
def files_match(dirpath: str) -> bool:
# All files in subcomponents will have a "$" in their path (it
# comes after the name of the subcomponent), so by filtering
# them out, we get only the files for this component. PDS4
# *does* allow directories in a component (that aren't part of
# a subcomponent), so we use walk instead of listdir() to get
# *all* the files, not just the top-level ones.
primary_files = filter_to_primary_files(
dirpath,
(
relpath(filepath)
for filepath in SubFS(primary_fs, dirpath).walk.files()
if "$" not in filepath
),
)
latest_files = filter_to_primary_files(
dirpath,
(
relpath(filepath)
for filepath in SubFS(latest_version_fs, dirpath).walk.files()
if "$" not in filepath
),
)
try:
PDS_LOGGER.open("File changes detected")
if primary_files != latest_files:
PDS_LOGGER.log(
"info",
f"CHANGE DETECTED IN {dirpath}: {primary_files} != {latest_files}",
)
PDS_LOGGER.close()
return False
for filename in primary_files:
filepath = join(dirpath, relpath(filename))
if primary_fs.getbytes(filepath) != latest_version_fs.getbytes(
filepath
):
PDS_LOGGER.log(
"info", f"CHANGE DETECTED IN {filepath}; DIRPATH = {dirpath}"
)
PDS_LOGGER.close()
return False
except Exception as e:
PDS_LOGGER.exception(e)
finally:
PDS_LOGGER.close()
return True
for dirpath in primary_fs.walk.dirs(filter=["*\$$"], search="depth"):
lid = dirpath_to_lid(dirpath)
if _lid_is_primary(lid):
latest_lidvid = mv.latest_lidvid(lid)
if latest_version_fs.isdir(dirpath):
matches = files_match(dirpath) and dirs_match(dirpath)
result.set(lid, _next_vid(mv, lid, not matches), not matches)
else:
result.set(lid, _next_vid(mv, lid, True), True)
else:
pass
return result