def __init__(
self,
base_fs: FS,
additions_fs: Optional[FS] = None,
deletions_fs: Optional[FS] = None,
) -> None:
COWFS.__init__(self, base_fs, additions_fs, deletions_fs)
def __init__(
self,
base_fs: FS,
additions_fs: Optional[FS] = None,
deletions_fs: Optional[FS] = None,
) -> None:
if not isinstance(base_fs, Versioned):
raise TypeError(f"Base filesystem {base_fs} is not Versioned.")
if not base_fs.is_multiversioned_fs():
raise ValueError(f"Base filesystem {base_fs} is not multiversioned.")
COWFS.__init__(self, base_fs, additions_fs, deletions_fs)
def _populate_target_identification(changes_dict: ChangesDict, db: BundleDB,
sv_deltas: COWFS) -> None:
for lid, (vid, changed) in changes_dict.items():
if changed and lid.is_product_lid():
lidvid = LIDVID.create_from_lid_and_vid(lid, vid)
product_path = lid_to_dirpath(lidvid.lid())
# Get a list of SHM/SPT/SHP fits files
fits_files = [
fits_file for fits_file in sv_deltas.listdir(product_path)
if (fs.path.splitext(fits_file)[1].lower() == ".fits"
and has_suffix_shm_spt_shf(
fs.path.splitext(fits_file)[0].lower()))
]
# Pass the path of SHM/SPT/SHP fits files to create a record in
# target identification table
for fits_file in fits_files:
fits_file_path = fs.path.join(product_path, fits_file)
fits_os_path = sv_deltas.getsyspath(fits_file_path)
db.create_target_identification(fits_os_path)
def _populate_products(changes_dict: ChangesDict, db: BundleDB,
sv_deltas: COWFS) -> None:
for lid, (vid, changed) in changes_dict.items():
if lid.is_product_lid():
lidvid = LIDVID.create_from_lid_and_vid(lid, vid)
collection_lidvid = changes_dict.parent_lidvid(lidvid)
if changed:
product_path = lid_to_dirpath(lidvid.lid())
if collection_lidvid.lid().collection_id == "document":
db.create_document_product(str(lidvid),
str(collection_lidvid))
doc_files = [
doc_file
for doc_file in sv_deltas.listdir(product_path)
if (fs.path.splitext(doc_file)[1].lower() in
DOCUMENT_SUFFIXES)
]
for doc_file in doc_files:
sys_filepath = sv_deltas.getsyspath(
fs.path.join(product_path, doc_file))
db.create_document_file(sys_filepath, doc_file,
str(lidvid))
else:
db.create_fits_product(str(lidvid), str(collection_lidvid))
fits_files = [
fits_file
for fits_file in sv_deltas.listdir(product_path)
if fs.path.splitext(fits_file)[1].lower() == ".fits"
]
for fits_file in fits_files:
fits_file_path = fs.path.join(product_path, fits_file)
fits_os_path = sv_deltas.getsyspath(fits_file_path)
populate_database_from_fits_file(
db, fits_os_path, str(lidvid))
else:
if changes_dict.changed(collection_lidvid.lid()):
db.create_collection_product_link(str(collection_lidvid),
str(lidvid))
def test_listdir(self) -> None:
fs = MemoryFS()
fs.makedirs("/b$")
fs.makedirs("/b$/dir1")
fs.makedirs("/b$/dir2")
fs.writetext("/b$/file1.txt", "file1")
fs.writetext("/b$/file2.txt", "file2")
fs.writetext("/b$/dir1/file1.txt", "file1")
fs.writetext("/b$/dir1/file2.txt", "file2")
fs.writetext("/b$/dir2/file1.txt", "file1")
fs.writetext("/b$/dir2/file2.txt", "file2")
c = COWFS(fs)
path = "/b$/dir1/file2.txt"
c.writetext(path, "xxxx")
# Now the COW version is different. But it should still have
# the old unchanged files.
self.assertTrue(c.exists("/b$/dir1/file1.txt")) # Yes, but...
self.assertEqual({"dir1", "dir2", "file1.txt", "file2.txt"},
set(c.listdir("/b$")))
def test_update_from_single_version(self) -> None:
fs = MemoryFS()
mv = Multiversioned(fs)
d = {
"file1.txt": "file1",
"file2.txt": "file2",
"dir1": {
"file1.txt": "file1",
"file2.txt": "file2"
},
"dir2": {
"file1.txt": "file1",
"file2.txt": "file2"
},
}
bundle_lidvid = LIDVID("urn:nasa:pds:b::1.0")
bundle_lid = bundle_lidvid.lid()
no_lidvids: Set[LIDVID] = set()
def create_bundle() -> None:
lidvids = [create_collection(bundle_lid, c) for c in ["c1", "c2"]]
contents = dictionary_to_contents(set(lidvids), d)
mv.add_contents_if(is_new, bundle_lid, contents)
def create_collection(bundle_lid: LID, c: str) -> LIDVID:
lid = bundle_lid.extend_lid(c)
lidvids = [create_product(lid, p) for p in ["p1", "p2"]]
contents = dictionary_to_contents(set(lidvids), d)
return mv.add_contents_if(is_new, lid, contents)
def create_product(coll_lid: LID, p: str) -> LIDVID:
lid = coll_lid.extend_lid(p)
contents = dictionary_to_contents(no_lidvids, d)
return mv.add_contents_if(is_new, lid, contents)
create_bundle()
vv = VersionView(mv, bundle_lidvid)
c = COWFS(vv, MemoryFS(), MemoryFS())
path = "/b$/c2$/p1$/dir1/file2.txt"
c.writetext(path, "xxxx")
latest_lidvid = mv.latest_lidvid(LID("urn:nasa:pds:b"))
# Update from the COWFS.
mv.update_from_single_version(is_new, c)
self.assertNotEqual(latest_lidvid,
mv.latest_lidvid(LID("urn:nasa:pds:b")))
latest_lidvid = mv.latest_lidvid(LID("urn:nasa:pds:b"))
# changed files are changed
self.assertEqual("file2", fs.readtext("b/c2/p1/v$1.0/dir1/file2.txt"))
self.assertEqual("xxxx", fs.readtext("b/c2/p1/v$2.0/dir1/file2.txt"))
# unchanged files are unchanged
self.assertEqual("file1", fs.readtext("b/c2/p1/v$1.0/dir1/file1.txt"))
self.assertEqual("file1", fs.readtext("b/c2/p1/v$2.0/dir1/file1.txt"))
# Change started in b/c2/p1. Check which versions are affected.
self.assertEqual(
VID("2.0"),
cast(LIDVID, mv.latest_lidvid(LID("urn:nasa:pds:b"))).vid())
self.assertEqual(
VID("1.0"),
cast(LIDVID, mv.latest_lidvid(LID("urn:nasa:pds:b:c1"))).vid())
self.assertEqual(
VID("2.0"),
cast(LIDVID, mv.latest_lidvid(LID("urn:nasa:pds:b:c2"))).vid())
self.assertEqual(
VID("2.0"),
cast(LIDVID, mv.latest_lidvid(LID("urn:nasa:pds:b:c2:p1"))).vid(),
)
self.assertEqual(
VID("1.0"),
cast(LIDVID, mv.latest_lidvid(LID("urn:nasa:pds:b:c2:p2"))).vid(),
)
# Now try updating again. Nothing should change.
mv.update_from_single_version(is_new, c)
self.assertEqual(latest_lidvid,
mv.latest_lidvid(LID("urn:nasa:pds:b")))
def make_fs(self) -> COWFS:
self.tempfs = TempFS()
return COWFS(self.tempfs)
def _build_browse_collection(
db: BundleDB,
changes_dict: ChangesDict,
browse_deltas: COWFS,
bundle_lidvid: LIDVID,
data_collection_lidvid: LIDVID,
bundle_path: str,
) -> None:
bundle_segment = bundle_lidvid.lid().parts()[0]
collection_segment = data_collection_lidvid.lid().parts()[1]
browse_collection_lid = data_collection_lidvid.lid().to_browse_lid()
collection_path = f"{bundle_path}{collection_segment}$/"
browse_collection_segment = browse_collection_lid.collection_id
browse_collection_path = f"{bundle_path}{browse_collection_segment}$/"
browse_collection_vid = data_collection_lidvid.vid()
browse_collection_lidvid = LIDVID.create_from_lid_and_vid(
browse_collection_lid, browse_collection_vid)
changes_dict.set(browse_collection_lid, browse_collection_vid, True)
browse_deltas.makedirs(browse_collection_path, recreate=True)
db.create_other_collection(str(browse_collection_lidvid),
str(bundle_lidvid))
db.create_bundle_collection_link(str(bundle_lidvid),
str(browse_collection_lidvid))
product_segments = [
str(prod[:-1]) for prod in browse_deltas.listdir(collection_path)
if "$" in prod
]
for product_segment in product_segments:
# These product_segments are from the data_collection
product_lid = LID.create_from_parts(
[bundle_segment, collection_segment, product_segment])
product_vid = changes_dict.vid(product_lid)
product_path = f"{collection_path}{product_segment}$/"
browse_product_path = f"{browse_collection_path}{product_segment}$/"
browse_product_lidvid = _extend_lidvid(browse_collection_lid,
product_vid, product_segment)
if changes_dict.changed(product_lid):
fits_product_lidvid = _extend_lidvid(
data_collection_lidvid.lid(),
data_collection_lidvid.vid(),
product_segment,
)
bpl = LIDVID(browse_product_lidvid)
changes_dict.set(bpl.lid(), bpl.vid(), True)
browse_deltas.makedirs(browse_product_path, recreate=True)
db.create_browse_product(
browse_product_lidvid,
fits_product_lidvid,
str(browse_collection_lidvid),
)
db.create_collection_product_link(str(browse_collection_lidvid),
browse_product_lidvid)
for fits_file in browse_deltas.listdir(product_path):
fits_filepath = fs.path.join(product_path, fits_file)
fits_os_filepath = browse_deltas.getsyspath(fits_filepath)
browse_file = fs.path.splitext(fits_file)[0] + ".jpg"
browse_filepath = fs.path.join(browse_product_path,
browse_file)
# In a COWFS, a directory does not have a
# syspath, only files. So we write a stub
# file into the directory, find its syspath
# and its directory's syspath. Then we remove
# the stub file.
browse_deltas.touch(browse_filepath)
browse_product_os_filepath = browse_deltas.getsyspath(
browse_filepath)
browse_deltas.remove(browse_filepath)
browse_product_os_dirpath = fs.path.dirname(
browse_product_os_filepath)
# Picmaker expects a list of strings. If you give it
# str, it'll index into it and complain about '/'
# not being a file. So don't do that!
try:
picmaker.ImagesToPics(
[str(fits_os_filepath)],
browse_product_os_dirpath,
filter="None",
percentiles=(1, 99),
)
except IndexError as e:
tb = traceback.format_exc()
message = f"File {fits_file}: {e}\n{tb}"
raise Exception(message)
browse_os_filepath = fs.path.join(browse_product_os_dirpath,
browse_file)
size = os.stat(browse_os_filepath).st_size
db.create_browse_file(browse_os_filepath, browse_file,
browse_product_lidvid, size)
else:
bpl = LIDVID(browse_product_lidvid)
changes_dict.set(bpl.lid(), bpl.vid(), False)
db.create_collection_product_link(str(browse_collection_lidvid),
browse_product_lidvid)