def _testAddTaleCreator(self):
from server.lib.manifest import Manifest
manifest_doc = Manifest(self.tale, self.user)
self.assertTrue(len(manifest_doc.manifest["schema:author"]))
def _testDataSet(self):
from server.lib.manifest import Manifest
# Test that all of the files in the dataSet are added
reference_aggregates = [
{
"uri": "doi:10.5065/D6862DM8",
"bundledAs": {
"folder":
"../data/Humans and Hydrology at High Latitudes: Water Use Information/"
},
"size": 28848454,
},
{
"uri":
"https://cn.dataone.org/cn/v2/resolve/urn:uuid:01a53103-8db1-46b3-967c-b42acf69ae08",
"bundledAs": {
"folder": "../data/",
"filename": "usco2005.xls"
},
"schema:isPartOf": "doi:10.5065/D6862DM8",
"size": 6427136,
},
{
"uri":
"globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec//published/publication_113/data/D_whites_darks_AJS.hdf",
"bundledAs": {
"folder": "../data/",
"filename": "D_whites_darks_AJS.hdf",
},
"schema:isPartOf": "doi:10.18126/M2301J",
"size": 8786120536,
},
{
"uri":
"globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec//published/publication_1106/data/Dmax",
"bundledAs": {
"folder": "../data/Dmax/"
},
"schema:isPartOf": "doi:10.18126/M2662X",
"size": 105050561,
},
{
"uri":
"https://www.gw-openscience.org/s/events/BBH_events_v3.json",
"bundledAs": {
"folder": "../data/",
"filename": "BBH_events_v3.json"
},
"size": 2202,
},
{
"uri":
"https://www.gw-openscience.org/s/events/GW170104/GW170104_4_template.hdf5",
"bundledAs": {
"folder": "../data/GW170104/",
"filename": "GW170104_4_template.hdf5",
},
"size": 1056864,
},
{
"uri":
"https://www.gw-openscience.org/s/events/GW170104/H-H1_LOSC_4_V1-1167559920-32.hdf5",
"bundledAs": {
"folder": "../data/GW170104/",
"filename": "H-H1_LOSC_4_V1-1167559920-32.hdf5",
},
"size": 1033609,
},
{
"uri":
"https://www.gw-openscience.org/s/events/GW170104/L-L1_LOSC_4_V1-1167559920-32.hdf5",
"bundledAs": {
"folder": "../data/GW170104/",
"filename": "L-L1_LOSC_4_V1-1167559920-32.hdf5",
},
"size": 1005007,
},
{
"uri": "../LICENSE",
"schema:license": "CC-BY-4.0"
},
]
from operator import itemgetter
reference_aggregates = sorted(reference_aggregates,
key=itemgetter("uri"))
manifest_doc = Manifest(self.tale, self.user)
for i, aggregate in enumerate(
sorted(manifest_doc.manifest["aggregates"],
key=itemgetter("uri"))):
self.assertDictEqual(aggregate, reference_aggregates[i])
# Check the datasets
reference_datasets = [
{
"@id": "doi:10.18126/M2662X",
"@type": "Dataset",
"name":
"A Machine Learning Approach for Engineering Bulk Metallic Glass Alloys",
"identifier": "doi:10.18126/M2662X",
},
{
"@id": "doi:10.18126/M2301J",
"@type": "Dataset",
"name": "Twin-mediated Crystal Growth: an Enigma Resolved",
"identifier": "doi:10.18126/M2301J",
},
{
"@id": "doi:10.5065/D6862DM8",
"@type": "Dataset",
"name":
"Humans and Hydrology at High Latitudes: Water Use Information",
"identifier": "doi:10.5065/D6862DM8",
},
]
reference_datasets = sorted(reference_datasets, key=itemgetter("@id"))
for i, dataset in enumerate(
sorted(manifest_doc.manifest["Datasets"],
key=itemgetter("@id"))):
self.assertDictEqual(dataset, reference_datasets[i])
def copyVersionsAndRuns(event: events.Event) -> None:
def get_dir_path(root_id_key, tale):
if root_id_key == "versionsRootId":
return util.getTaleVersionsDirPath(tale)
elif root_id_key == "runsRootId":
return util.getTaleRunsDirPath(tale)
old_tale, new_tale, target_version_id, shallow = event.info
if shallow and not target_version_id:
return
creator = User().load(new_tale["creatorId"], force=True)
versions_map = {}
for root_id_key in ("versionsRootId", "runsRootId"):
old_root = Folder().load(old_tale[root_id_key],
user=creator,
level=AccessType.READ)
new_root = Folder().load(new_tale[root_id_key],
user=creator,
level=AccessType.WRITE)
old_root_path = get_dir_path(root_id_key, old_tale)
new_root_path = get_dir_path(root_id_key, new_tale)
for src in Folder().childFolders(old_root, "folder", user=creator):
if shallow and str(src["_id"]) != target_version_id:
continue
dst = Folder().createFolder(new_root, src["name"], creator=creator)
if root_id_key == "versionsRootId":
versions_map[str(src["_id"])] = str(dst["_id"])
filtered_folder = Folder().filter(dst, creator)
for key in src:
if key not in filtered_folder and key not in dst:
dst[key] = copy.deepcopy(src[key])
src_path = old_root_path / str(src["_id"])
dst_path = new_root_path / str(dst["_id"])
dst_path.mkdir(parents=True)
shutil.copytree(src_path,
dst_path,
dirs_exist_ok=True,
symlinks=True)
dst.update({
"fsPath": dst_path.absolute().as_posix(),
"isMapping": True,
"created": src["created"], # preserve timestamps
"updated": src["updated"],
})
if root_id_key == "runsRootId":
current_version = dst_path / "version"
new_version_id = versions_map[current_version.resolve().name]
new_version_path = (
"../../../../versions/"
f"{str(new_tale['_id'])[:2]}/{new_tale['_id']}/{new_version_id}"
)
current_version.unlink()
current_version.symlink_to(new_version_path, True)
dst["runVersionId"] = ObjectId(new_version_id)
dst = Folder().save(dst, validate=False, triggerEvents=False)
# update the time on root
Folder().updateFolder(new_root)
versions_root = Folder().load(new_tale["versionsRootId"],
user=creator,
level=AccessType.WRITE)
for version in Folder().childFolders(versions_root, "folder",
user=creator):
tale = copy.deepcopy(new_tale)
tale.update(VersionHierarchyModel().restoreTaleFromVersion(version))
manifest = Manifest(tale,
creator,
versionId=version["_id"],
expand_folders=False)
dst_path = pathlib.Path(version["fsPath"])
with open(dst_path / "manifest.json", "w") as fp:
fp.write(manifest.dump_manifest())
Folder().updateFolder(versions_root)
if target_version_id:
new_version_id = versions_map[str(target_version_id)]
target_version = Folder().load(new_version_id,
level=AccessType.READ,
user=creator)
VersionHierarchyModel().restore(new_tale, target_version, creator)