def testTaleImportBinderFail(self):
with mock.patch("girder.plugins.wholetale.lib.pids_to_entities") as mock_pids:
mock_pids.side_effect = ValueError
resp = self.request(
path="/tale/import",
method="POST",
user=self.user,
params={
"url": "http://use.yt/upload/ef4cd901",
"spawn": False,
"imageId": self.image["_id"],
"asTale": True,
"taleKwargs": json.dumps({"title": "tale should fail"}),
},
)
self.assertStatusOk(resp)
tale = resp.json
job = Job().findOne({"type": "wholetale.import_binder"})
self.assertEqual(
json.loads(job["kwargs"])["taleId"]["$oid"], tale["_id"]
)
for i in range(300):
if job["status"] in {JobStatus.SUCCESS, JobStatus.ERROR}:
break
time.sleep(0.1)
job = Job().load(job["_id"], force=True)
self.assertEqual(job["status"], JobStatus.ERROR)
Job().remove(job)
tale = Tale().load(tale["_id"], force=True)
self.assertEqual(tale["status"], TaleStatus.ERROR)
Tale().remove(tale)
def testSingletonDataverse(self):
from girder.plugins.wholetale.models.tale import Tale
from bson import ObjectId
tale = Tale().createTale(
{"_id": ObjectId()},
[],
creator=self.user,
title="Some Tale",
relatedIdentifiers=[{
"identifier": "doi:10.7910/DVN/TJCLKP",
"relation": "IsDerivedFrom"
}],
)
resp = self.request(
"/integration/dataverse",
method="GET",
params={
"datasetId": "3035124",
"siteUrl": "https://dataverse.harvard.edu",
"fullDataset": False,
},
user=self.user,
isJson=False,
)
self.assertStatus(resp, 303)
self.assertEqual(
urlparse(resp.headers["Location"]).path,
"/run/{}".format(tale["_id"]))
Tale().remove(tale)
def testCopyWorkspaceFail(self):
tale = Tale().createTale(
self.image,
[],
creator=self.admin,
title="tale one",
public=True,
config={"memLimit": "2g"},
)
job = Job().createLocalJob(
title='Copy "{title}" workspace'.format(**tale),
user=self.user,
type="wholetale.copy_workspace",
public=False,
async=True,
module="girder.plugins.wholetale.tasks.copy_workspace",
args=(tale["workspaceId"], "non_existing"),
kwargs={"user": self.user, "tale": tale},
)
Job().scheduleJob(job)
for i in range(300):
if job["status"] in {JobStatus.SUCCESS, JobStatus.ERROR}:
break
time.sleep(0.1)
job = Job().load(job["_id"], force=True)
self.assertEqual(job["status"], JobStatus.ERROR)
Job().remove(job)
tale = Tale().load(tale["_id"], force=True)
self.assertEqual(tale["status"], TaleStatus.ERROR)
Tale().remove(tale)
def test09TaleUpdateEventHandler(self):
dataSet = self.makeDataSet([{
'_id': self.testFolder['_id'],
'name': 'fldr'
}],
objectids=False)
from girder.plugins.wholetale.models.tale import Tale
tale = Tale().createTale({'_id': ObjectId()},
dataSet,
title='test09',
creator=self.user)
resp = self.request(path='/dm/session',
method='POST',
user=self.user,
params={'taleId': str(tale['_id'])})
self.assertStatusOk(resp)
session = resp.json
self.assertEqual(session['dataSet'], dataSet)
tale['dataSet'].pop(0)
tale = Tale().save(tale)
resp = self.request(path='/dm/session/{_id}'.format(**session),
method='GET',
user=self.user)
self.assertStatusOk(resp)
session = resp.json
self.assertEqual(session['dataSet'], tale['dataSet'])
Tale().remove(tale) # TODO: This should fail, since the session is up
resp = self.request(path='/dm/session/{_id}'.format(**session),
method='DELETE',
user=self.user)
self.assertStatusOk(resp)
def test_binder_heuristics(self):
from girder.plugins.wholetale.tasks.import_binder import sanitize_binder
tale = Tale().createTale(self.image, [], creator=self.user, title="Binder")
token = Token().createToken(user=self.user, days=0.25)
tmpdir = tempfile.mkdtemp()
with open(tmpdir + "/i_am_a_binder", "w") as fobj:
fobj.write("but well hidden!")
with tarfile.open(tmpdir + "/tale.tar.gz", "w:gz") as tar:
tar.add(tmpdir + "/i_am_a_binder", arcname="dir_in_tar/i_am_a_binder")
os.remove(tmpdir + "/i_am_a_binder")
with zipfile.ZipFile(tmpdir + "/tale.zip", "w") as myzip:
myzip.write(tmpdir + "/tale.tar.gz", arcname="dir_in_zip/tale.tar.gz")
os.remove(tmpdir + "/tale.tar.gz")
os.makedirs(tmpdir + "/hidden_binder")
os.rename(tmpdir + "/tale.zip", tmpdir + "/hidden_binder" + "/tale.zip")
girder_root = "http://localhost:{}".format(
config.getConfig()["server.socket_port"]
)
with WebDAVFS(
girder_root,
login=self.user["login"],
password="******".format(**token),
root="/tales/{_id}".format(**tale),
) as destination_fs, OSFS(tmpdir) as source_fs:
copy_fs(source_fs, destination_fs)
sanitize_binder(destination_fs)
self.assertEqual(destination_fs.listdir("/"), ["i_am_a_binder"])
shutil.rmtree(tmpdir)
Tale().remove(tale)
def _testRelatedIdentifiers(self):
from girder.plugins.wholetale.lib.manifest import Manifest
from girder.plugins.wholetale.models.tale import Tale
tale = copy.deepcopy(self.tale)
tale.pop("_id")
tale["relatedIdentifiers"] = [{
"identifier": "urn:some_urn",
"relation": "cites"
}]
with pytest.raises(ValidationException) as exc:
tale = Tale().save(tale)
self.assertTrue(str(exc.value).startswith("'cites' is not one of"))
tale["relatedIdentifiers"] = [
{
"identifier": "urn:some_urn",
"relation": "Cites"
},
{
"identifier": "doi:some_doi",
"relation": "IsDerivedFrom"
},
{
"identifier": "https://some.url",
"relation": "IsIdenticalTo"
},
]
tale = Tale().save(tale)
manifest = Manifest(tale, self.user)
attrs = manifest.create_related_identifiers()
self.assertIn("DataCite:relatedIdentifiers", attrs)
self.assertEqual(
attrs["DataCite:relatedIdentifiers"],
[
{
"DataCite:relatedIdentifier": {
"@id": "urn:some_urn",
"DataCite:relationType": "DataCite:Cites",
"DataCite:relatedIdentifierType": "DataCite:URN",
}
},
{
"DataCite:relatedIdentifier": {
"@id": "doi:some_doi",
"DataCite:relationType": "DataCite:IsDerivedFrom",
"DataCite:relatedIdentifierType": "DataCite:DOI",
}
},
{
"DataCite:relatedIdentifier": {
"@id": "https://some.url",
"DataCite:relationType": "DataCite:IsIdenticalTo",
"DataCite:relatedIdentifierType": "DataCite:URL",
}
},
],
)
Tale().remove(tale)
def _createAuxFolder(tale, name, rootProp, creator):
folder = Tale()._createAuxFolder(tale, name, creator=creator)
folder.update({'seq': 0, 'taleId': tale['_id']})
Folder().save(folder, False)
rootDir = util.getTaleDirPath(tale, rootProp)
rootDir.mkdir(parents=True, exist_ok=True)
trashDir = rootDir / '.trash'
trashDir.mkdir(exist_ok=True)
return (folder, rootDir)
def testTaleCopy(self):
from girder.plugins.wholetale.models.tale import Tale
from girder.plugins.wholetale.constants import TaleStatus
from girder.plugins.jobs.models.job import Job
from girder.plugins.jobs.constants import JobStatus
tale = Tale().createTale(self.image, [],
creator=self.admin,
public=True)
workspace = Tale().createWorkspace(tale)
# Below workarounds a bug, it will be addressed elsewhere.
workspace = Folder().setPublic(workspace, True, save=True)
adapter = assetstore_utilities.getAssetstoreAdapter(self.ws_assetstore)
size = 101
data = BytesIO(b' ' * size)
files = []
files.append(Upload().uploadFromFile(data,
size,
'file01.txt',
parentType='folder',
parent=workspace,
assetstore=self.ws_assetstore))
fullPath = adapter.fullPath(files[0])
# Create a copy
resp = self.request(path='/tale/{_id}/copy'.format(**tale),
method='POST',
user=self.user)
self.assertStatusOk(resp)
new_tale = resp.json
self.assertFalse(new_tale['public'])
self.assertEqual(new_tale['dataSet'], tale['dataSet'])
self.assertEqual(new_tale['copyOfTale'], str(tale['_id']))
self.assertEqual(new_tale['imageId'], str(tale['imageId']))
self.assertEqual(new_tale['creatorId'], str(self.user['_id']))
self.assertEqual(new_tale['status'], TaleStatus.PREPARING)
copied_file_path = re.sub(workspace['name'], new_tale['_id'], fullPath)
job = Job().findOne({'type': 'wholetale.copy_workspace'})
for i in range(10):
job = Job().load(job['_id'], force=True)
if job['status'] == JobStatus.SUCCESS:
break
time.sleep(0.1)
self.assertTrue(os.path.isfile(copied_file_path))
resp = self.request(path='/tale/{_id}'.format(**new_tale),
method='GET',
user=self.user)
self.assertStatusOk(resp)
new_tale = resp.json
self.assertEqual(new_tale['status'], TaleStatus.READY)
Tale().remove(new_tale)
Tale().remove(tale)
def setTaleFolderMapping(event: events.Event):
tale = event.info
root_path = Setting().get(PluginSettings.TALE_DIRS_ROOT)
creator = User().load(tale["creatorId"], force=True)
workspace = Tale()._createAuxFolder(tale, WORKSPACE_NAME, creator=creator)
absDir = "%s/%s" % (root_path, TalePathMapper().davToPhysical("/" + str(tale["_id"])))
absDir = pathlib.Path(absDir)
absDir.mkdir(parents=True, exist_ok=True)
workspace.update({'fsPath': absDir.as_posix(), 'isMapping': True})
Folder().save(workspace, validate=True, triggerEvents=False)
tale["workspaceId"] = workspace["_id"]
tale = Tale().save(tale)
event.addResponse(tale)
def migrate_item(item):
_file = list(Item().childFiles(item))[0]
url = _file['linkUrl']
if url.startswith('https://dashboard.wholetale.org'):
return 0
creator = User().load(item['creatorId'], force=True)
# register url
entity = Entity(url.strip(), creator)
provider = HTTPImportProvider()
try:
dataMap = provider.lookup(entity)
except Exception:
print(" -> Failed to resolve {} as HTTP".format(url))
print(" -> item_id = {}".format(str(item["_id"])))
return 0
ds = dataMap.toDict()
if not ds['name']:
print(" -> Item has no name!!!")
print(ds)
return 0
with ProgressContext(True, user=creator,
title='Registering resources') as ctx:
objType, new_item = provider.register(CAT_ROOT,
'folder',
ctx,
creator,
dataMap,
base_url=base_url)
# find userData and replace with new id
for user in User().find({'myData': item['_id']}):
print(' Updating {} in myData for user "{}"'.format(
item['name'], user['login']))
user['myData'][user['myData'].index(item['_id'])] = new_item['_id']
user = User().save(user)
# find tale dataset and switch id
for tale in Tale().find({'dataSet.itemId': str(item['_id'])}):
print(' Updating {} in dataSet of Tale: "{}"'.format(
item['name'], tale['title']))
for i, ds in enumerate(tale['dataSet']):
if ds['itemId'] == str(item['_id']):
tale['dataSet'][i]['itemId'] = str(new_item['_id'])
Tale().save(tale)
return 1
def update_parents(self, event: events.Event):
vrfolder_id = event.info.get("id")
user = self.getCurrentUser()
vrfolder = Folder().load(vrfolder_id,
user=user,
level=AccessType.WRITE)
if vrfolder:
root = Folder().load(vrfolder['parentId'],
user=user,
level=AccessType.WRITE)
Folder().updateFolder(root)
tale = Tale().load(root["meta"]["taleId"],
user=user,
level=AccessType.WRITE)
Tale().updateTale(tale)
def restore(self, tale: dict, version: dict):
user = self.getCurrentUser()
version_root = Folder().load(version["parentId"],
user=user,
level=AccessType.READ)
workspace = Folder().load(tale["workspaceId"], force=True)
workspace_path = Path(workspace["fsPath"])
version = Folder().load(version["_id"], force=True, fields=["fsPath"])
version_workspace_path = Path(version["fsPath"]) / "workspace"
if not Version._setCriticalSectionFlag(version_root):
raise RestException(
'Another operation is in progress. Try again later.', 409)
try:
# restore workspace
shutil.rmtree(workspace_path)
workspace_path.mkdir()
self._snapshotRecursive(None, version_workspace_path,
workspace_path)
# restore Tale
tale.update(self._restoreTaleFromVersion(version))
return Tale().save(tale)
finally:
# probably need a better way to deal with hard crashes here
Version._resetCriticalSectionFlag(version_root)
def create(self,
tale: dict,
name: str = None,
force: bool = False,
allowRename: bool = False) -> dict:
if not name:
name = self._generateName()
user = self.getCurrentUser()
root = self._getRootFromTale(tale, user=user, level=AccessType.WRITE)
name = self._checkNameSanity(name, root, allow_rename=allowRename)
if not Version._setCriticalSectionFlag(root):
raise RestException(
'Another operation is in progress. Try again later.', 409)
try:
rootDir = util.getTaleVersionsDirPath(tale)
return self._create(tale,
name,
rootDir,
root,
user=user,
force=force)
finally:
# probably need a better way to deal with hard crashes here
Version._resetCriticalSectionFlag(root)
Tale().updateTale(tale)
def load(info):
setDefaults()
settings = Setting()
homeDirsRoot = settings.get(PluginSettings.HOME_DIRS_ROOT)
logger.info('WT Home Dirs root: %s' % homeDirsRoot)
startDAVServer(homeDirsRoot, HomeDirectoryInitializer, HomeAuthorizer, HomePathMapper())
taleDirsRoot = settings.get(PluginSettings.TALE_DIRS_ROOT)
logger.info('WT Tale Dirs root: %s' % taleDirsRoot)
startDAVServer(taleDirsRoot, TaleDirectoryInitializer, TaleAuthorizer, TalePathMapper())
runsDirsRoot = settings.get(PluginSettings.RUNS_DIRS_ROOT)
if runsDirsRoot:
logger.info('WT Runs Dirs root: %s' % runsDirsRoot)
startDAVServer(runsDirsRoot, RunsDirectoryInitializer, RunsAuthorizer, RunsPathMapper())
events.unbind('model.user.save.created', CoreEventHandler.USER_DEFAULT_FOLDERS)
events.bind('model.user.save.created', 'wt_home_dirs', setHomeFolderMapping)
events.bind('model.tale.save.created', 'wt_home_dirs', setTaleFolderMapping)
events.bind('model.tale.remove', 'wt_home_dirs', deleteWorkspace)
hdp = Homedirpass()
info['apiRoot'].homedirpass = hdp
info['apiRoot'].homedirpass.route('GET', ('generate',), hdp.generatePassword)
info['apiRoot'].homedirpass.route('PUT', ('set',), hdp.setPassword)
Tale().exposeFields(level=AccessType.READ, fields={"workspaceId"})
def startRun(self, run, entrypoint):
user = self.getCurrentUser()
if not entrypoint:
entrypoint = "run.sh"
runRoot = Folder().load(run['parentId'],
user=user,
level=AccessType.WRITE)
tale = Tale().load(runRoot['meta']['taleId'],
user=user,
level=AccessType.READ)
resource = {
'type': 'wt_recorded_run',
'tale_id': tale['_id'],
'tale_title': tale['title']
}
token = Token().createToken(user=user, days=0.5)
notification = init_progress(resource, user, 'Recorded run',
'Initializing', RECORDED_RUN_STEP_TOTAL)
rrTask = recorded_run.signature(
args=[str(run['_id']),
str(tale['_id']), entrypoint],
girder_job_other_fields={
'wt_notification_id': str(notification['_id']),
},
girder_client_token=str(token['_id']),
).apply_async()
return Job().filter(rrTask.job, user=user)
def test15WorkspaceRemoval(self):
from girder.plugins.wholetale.models.tale import Tale
from girder.models.folder import Folder
workspace = Folder().load(self.publicTale["workspaceId"], force=True)
self.assertTrue(os.path.isdir(workspace["fsPath"]))
Tale().remove(self.publicTale)
self.assertFalse(os.path.isdir(workspace["fsPath"]))
workspace = Folder().load(workspace["_id"], force=True)
self.assertEqual(workspace, None)
def testTaleImportZipFail(self):
image = Image().createImage(
name="Jupyter Classic",
creator=self.user,
public=True,
config=dict(
template="base.tpl",
buildpack="PythonBuildPack",
user="******",
port=8888,
urlPath="",
),
)
with mock.patch("girder.plugins.wholetale.lib.pids_to_entities") as mock_pids:
mock_pids.side_effect = ValueError
with open(
os.path.join(DATA_PATH, "5c92fbd472a9910001fbff72.zip"), "rb"
) as fp:
resp = self.request(
path="/tale/import",
method="POST",
user=self.user,
type="application/zip",
body=fp.read(),
)
self.assertStatusOk(resp)
tale = resp.json
job = Job().findOne({"type": "wholetale.import_tale"})
self.assertEqual(
json.loads(job["kwargs"])["taleId"]["$oid"], tale["_id"]
)
for i in range(300):
if job["status"] in {JobStatus.SUCCESS, JobStatus.ERROR}:
break
time.sleep(0.1)
job = Job().load(job["_id"], force=True)
self.assertEqual(job["status"], JobStatus.ERROR)
Job().remove(job)
tale = Tale().load(tale["_id"], force=True)
self.assertEqual(tale["status"], TaleStatus.ERROR)
Tale().remove(tale)
Image().remove(image)
def create(
self, version: dict, name: Optional[str], user: dict, allowRename: bool = False
) -> dict:
if not name:
name = self.generateName()
versionsRoot = Folder().load(
version["parentId"], user=user, level=AccessType.WRITE
)
taleId = versionsRoot["taleId"]
tale = Tale().load(taleId, user=user, level=AccessType.WRITE)
root = self.getRootFromTale(tale, user=user, level=AccessType.WRITE)
name = self.checkNameSanity(name, root, allow_rename=allowRename)
rootDir = util.getTaleRunsDirPath(tale)
runFolder = self.createSubdir(rootDir, root, name, user=user)
runFolder["runVersionId"] = version["_id"]
runFolder[FIELD_STATUS_CODE] = RunStatus.UNKNOWN.code
Folder().save(runFolder, False)
# Structure is:
# @version -> ../Versions/<version> (link handled manually by FS)
# @workspace -> version/workspace (same)
# .status
# .stdout (created using stream() above)
# .stderr (-''-)
runDir = Path(runFolder["fsPath"])
tale_id = runDir.parts[-2]
# TODO: a lot assumptions hardcoded below...
(runDir / "version").symlink_to(
f"../../../../versions/{tale_id[:2]}/{tale_id}/{version['_id']}", True
)
(runDir / "workspace").mkdir()
self.snapshotRecursive(
None, (runDir / "version" / "workspace"), (runDir / "workspace")
)
self.write_status(runDir, RunStatus.UNKNOWN)
Tale().updateTale(tale)
VersionHierarchyModel().incrementReferenceCount(version)
return runFolder
def rename(self, vfolder: dict, name: str, allowRename: bool) -> dict:
renamed_version = super().rename(vfolder,
name,
allow_rename=allowRename)
user = self.getCurrentUser()
root = Folder().load(vfolder["parentId"],
user=user,
level=AccessType.WRITE)
tale = Tale().load(root["taleId"], user=user, level=AccessType.WRITE)
manifest = Manifest(tale,
user,
versionId=renamed_version["_id"],
expand_folders=False)
version_path = Path(renamed_version["fsPath"])
with open((version_path / "manifest.json").as_posix(), "w") as fp:
fp.write(manifest.dump_manifest())
Tale().updateTale(Tale().load(root["taleId"], force=True))
return renamed_version
def delete(self, vfolder: dict) -> None:
root = Folder().load(vfolder['parentId'], force=True)
Version._setCriticalSectionFlag(root)
try:
# make sure we use information protected by the critical section
vfolder = Folder().load(vfolder['_id'], force=True)
if FIELD_REFERENCE_COUNTER in vfolder and vfolder[
FIELD_REFERENCE_COUNTER] > 0:
raise RestException(
'Version is in use by a run and cannot be deleted.', 461)
finally:
Version._resetCriticalSectionFlag(root)
path = Path(vfolder['fsPath'])
trashDir = path.parent / '.trash'
Folder().remove(vfolder)
shutil.move(path.as_posix(), trashDir)
Tale().updateTale(Tale().load(root["taleId"], force=True))
def _restoreTaleFromVersion(self, version, annotate=True):
version_path = Path(version["fsPath"])
with open((version_path / "manifest.json").as_posix(), "r") as fp:
manifest = json.load(fp)
with open((version_path / "environment.json").as_posix(), "r") as fp:
env = json.load(fp)
restored_tale = Tale().restoreTale(manifest, env)
if annotate:
restored_tale["restoredFrom"] = version["_id"]
return restored_tale
def addVersionsAndRuns(event: events.Event) -> None:
tale = event.info
creator = User().load(tale['creatorId'], force=True)
versions_root, _ = _createAuxFolder(tale, Constants.VERSIONS_ROOT_DIR_NAME,
PluginSettings.VERSIONS_DIRS_ROOT,
creator)
tale["versionsRootId"] = versions_root["_id"]
runs_root, _ = _createAuxFolder(tale, Constants.RUNS_ROOT_DIR_NAME,
PluginSettings.RUNS_DIRS_ROOT, creator)
tale["runsRootId"] = runs_root["_id"]
tale = Tale().save(tale)
event.addResponse(tale)
def remove(self, version: dict, user: dict) -> None:
root = Folder().load(version["parentId"],
user=user,
level=AccessType.WRITE)
self.setCriticalSectionFlag(root)
try:
# make sure we use information protected by the critical section
version = Folder().load(version["_id"],
user=user,
level=AccessType.ADMIN)
if version.get(self.field_reference_counter, 0) > 0:
raise RestException(
"Version is in use by a run and cannot be deleted.", 461)
finally:
self.resetCriticalSectionFlag(root)
path = Path(version["fsPath"])
trashDir = path.parent / ".trash"
Folder().remove(version)
shutil.move(path.as_posix(), trashDir)
Tale().updateTale(Tale().load(root["taleId"], force=True))
def create(self,
version: dict,
name: str = None,
allowRename: bool = False) -> dict:
if not name:
name = self._generateName()
user = self.getCurrentUser()
versionsRoot = Folder().load(version['parentId'],
user=user,
level=AccessType.WRITE)
taleId = versionsRoot['taleId']
tale = Tale().load(taleId, user=user, level=AccessType.WRITE)
root = self._getRootFromTale(tale, user=user, level=AccessType.WRITE)
name = self._checkNameSanity(name, root, allow_rename=allowRename)
rootDir = util.getTaleRunsDirPath(tale)
run = self._create(version, name, root, rootDir)
Tale().updateTale(tale)
Version._incrementReferenceCount(version)
return run
def migrateIframe():
# Set iframe attributes on images and tales
for img in Image().find():
img['iframe'] = True
Image().save(img, validate=True)
for tale in Tale().find({}):
tale['iframe'] = True
Tale().save(tale, validate=True)
# Add config to Jupyter images
for image in Image().find():
if 'rstudio' in image['fullName']:
image['config'] = {
"command": "/init",
"environment": ["CSP_HOSTS=dashboard.stage.wholetale.org"],
"port": 8787,
"targetMount": "/home/rstudio/work",
"urlPath": "",
"user": "******"
}
Image().save(image)
else:
image['config'] = {
"command":
"jupyter notebook --no-browser --port {port} --ip=0.0.0.0 --NotebookApp.token={token} --NotebookApp.base_url=/{base_path} --NotebookApp.port_retries=0",
"memLimit": "2048m",
"port": 8888,
"targetMount": "/home/jovyan/work",
"urlPath": "?token={token}",
"user": "******",
"environment": ["CSP_HOSTS=\"dashboard.stage.wholetale.org\""]
}
Image().save(image)
def load(info):
setDefaults()
createIndex()
resetCrashedCriticalSections()
events.bind('model.tale.save.created', 'wt_versioning', addVersionsAndRuns)
events.bind('model.tale.remove', 'wt_versioning', removeVersionsAndRuns)
events.bind('wholetale.tale.copied', 'wt_versioning', copyVersionsAndRuns)
Tale().exposeFields(
level=AccessType.READ,
fields={"versionsRootId", "runsRootId", "restoredFrom"})
Folder().exposeFields(level=AccessType.READ,
fields={"runVersionId", FIELD_STATUS_CODE})
info['apiRoot'].version = Version(info["apiRoot"].tale)
info['apiRoot'].run = Run()
def load(info):
SettingDefault.defaults[
PluginSettings.VERSIONS_DIRS_ROOT] = '/tmp/wt/versions'
SettingDefault.defaults[PluginSettings.RUNS_DIRS_ROOT] = '/tmp/wt/runs'
Folder().ensureIndex('created')
VersionHierarchyModel().resetCrashedCriticalSections()
events.bind('model.tale.save.created', 'wt_versioning', addVersionsAndRuns)
events.bind('model.tale.remove', 'wt_versioning', removeVersionsAndRuns)
events.bind('wholetale.tale.copied', 'wt_versioning', copyVersionsAndRuns)
Tale().exposeFields(
level=AccessType.READ,
fields={"versionsRootId", "runsRootId", "restoredFrom"})
Folder().exposeFields(level=AccessType.READ,
fields={"runVersionId", FIELD_STATUS_CODE})
info['apiRoot'].version = Version(info["apiRoot"].tale)
info['apiRoot'].run = Run()
def _is_same(self, tale, version, user):
workspace = Folder().load(tale["workspaceId"], force=True)
tale_workspace_path = Path(workspace["fsPath"])
version_path = None if version is None else Path(version["fsPath"])
version_workspace_path = None if version_path is None else version_path / "workspace"
manifest_obj = Manifest(tale, user)
manifest = json.loads(manifest_obj.dump_manifest())
environment = json.loads(manifest_obj.dump_environment())
tale_restored_from_wrk = Tale().restoreTale(manifest, environment)
tale_restored_from_ver = \
self._restoreTaleFromVersion(version, annotate=False) if version else None
if self._sameTaleMetadata(tale_restored_from_ver, tale_restored_from_wrk) and \
self._sameTree(version_workspace_path, tale_workspace_path):
raise RestException('Not modified',
code=303,
extra=str(version["_id"]))
def finalizeInstance(event):
job = event.info['job']
if job.get("instance_id"):
instance = Instance().load(job["instance_id"], force=True)
if (instance["status"] == InstanceStatus.LAUNCHING
and job["status"] == JobStatus.ERROR # noqa
):
instance["status"] = InstanceStatus.ERROR
Instance().updateInstance(instance)
if job['title'] == 'Spawn Instance' and job.get('status') is not None:
status = int(job['status'])
instance_id = job['args'][0]['instanceId']
instance = Instance().load(instance_id, force=True, exc=True)
update = True
if (status == JobStatus.SUCCESS
and instance["status"] == InstanceStatus.LAUNCHING # noqa
):
service = getCeleryApp().AsyncResult(job['celeryTaskId']).get()
valid_keys = set(containerInfoSchema['properties'].keys())
containerInfo = {key: service.get(key, '') for key in valid_keys}
url = service.get('url', 'https://google.com')
_wait_for_server(url)
# Since _wait_for_server can potentially take some time,
# we need to refresh the state of the instance
instance = Instance().load(instance_id, force=True, exc=True)
if instance["status"] != InstanceStatus.LAUNCHING:
return # bail
# Preserve the imageId / current digest in containerInfo
tale = Tale().load(instance['taleId'], force=True)
containerInfo['imageId'] = tale['imageId']
containerInfo['digest'] = tale['imageInfo']['digest']
instance.update({
'url': url,
'status': InstanceStatus.RUNNING,
'containerInfo': containerInfo,
})
if "sessionId" in service:
instance["sessionId"] = ObjectId(service["sessionId"])
elif (status == JobStatus.ERROR
and instance["status"] != InstanceStatus.ERROR # noqa
):
instance['status'] = InstanceStatus.ERROR
elif (status in (JobStatus.QUEUED, JobStatus.RUNNING)
and instance["status"] != InstanceStatus.LAUNCHING # noqa
):
instance['status'] = InstanceStatus.LAUNCHING
else:
update = False
if update:
msg = "Updating instance ({_id}) in finalizeInstance".format(
**instance)
msg += " for job(id={_id}, status={status})".format(**job)
logger.debug(msg)
Instance().updateInstance(instance)
def _testSessionApi(self, dataSet, item):
session = self.apiroot.dm.createSession(self.user, dataSet)
sessions = list(
self.model('session', 'wt_data_manager').list(self.user))
self.assertEqual(len(sessions), 1)
self._testItemWithSession(session, item)
resp = self.request(path='/dm/session/{_id}/object'.format(**session),
method='GET',
user=self.user,
params={'path': '/non_existent_path'})
self.assertStatus(resp, 400)
resp = self.request(path='/dm/session/{_id}/object'.format(**session),
method='GET',
user=self.user,
params={'path': '/filetest__4'})
self.assertStatusOk(resp)
self.assertEqual(resp.json['object']['_id'],
str(self.gfiles[3]['_id']))
dataSet.append({
'itemId': str(self.testFolder2['_id']),
'mountPath': '/' + self.testFolder2['name']
})
dataSet = [{
'itemId': str(_['itemId']),
'mountPath': _['mountPath']
} for _ in dataSet]
resp = self.request(path='/dm/session/{_id}'.format(**session),
method='PUT',
user=self.user,
params={'dataSet': json.dumps(dataSet)})
self.assertStatusOk(resp)
session = resp.json
self.assertEqual(session['seq'], 1)
resp = self.request(path='/dm/session/{_id}/object'.format(**session),
method='GET',
user=self.user,
params={
'path': '/' + self.testFolder2['name'],
'children': True
})
self.assertStatusOk(resp)
children = resp.json['children']
leafFile = next(
(_ for _ in children if _['name'] == self.gfiles2[-1]['name']),
None)
self.assertEqual(leafFile['_id'], str(self.gfiles2[-1]['_id']))
resp = self.request(path='/dm/session/{_id}/object'.format(**session),
method='GET',
user=self.user,
params={
'path':
'/' + self.testFolder2['name'] + '/' +
leafFile['name'] + '_blah',
'children':
True
})
self.assertStatus(resp, 400)
resp = self.request(path='/dm/session/{_id}/object'.format(**session),
method='GET',
user=self.user,
params={
'path':
'/' + self.testFolder2['name'] + '/' +
leafFile['name'],
'children':
True
})
self.assertStatusOk(resp)
resp = self.request(path='/dm/session/{_id}'.format(**session),
method='DELETE',
user=self.user2)
self.assertStatus(resp, 403)
resp = self.request(path='/dm/session/{_id}'.format(**session),
method='DELETE',
user=self.admin)
self.assertStatusOk(resp)
from girder.plugins.wholetale.models.tale import Tale
tale = Tale().createTale({'_id': ObjectId()},
dataSet,
title='blah',
creator=self.user)
resp = self.request(path='/dm/session',
method='POST',
user=self.user,
params={'taleId': str(tale['_id'])})
self.assertStatusOk(resp)
session = resp.json
self.assertEqual(session['dataSet'], dataSet)
Tale().remove(tale) # TODO: This should fail, since the session is up
resp = self.request(path='/dm/session/{_id}'.format(**session),
method='DELETE',
user=self.user)
self.assertStatusOk(resp)
