def __init__(self, realm, pathMapper, app, user: dict, token, realmDir):
Adapter.__init__(self, realm, pathMapper, app, user)
url = 'http://127.0.0.1:%s' % os.environ['GIRDER_PORT']
password = '******' % token['_id']
self.root = '/%s/%s' % (realm, realmDir)
self.handle = WebDAVFS(url,
login=self.userName,
password=password,
root=self.root)
self.handle.makedir('test')
if not self.handle.isdir('test'):
raise Exception('Basic DAV test failed')
self.handle.removedir('test')
self.name = 'DAV'
def _configure_backing_store(self):
try:
backing_stores = []
for bs in self.config['Backing Store']:
if 'Type' in bs:
for key, item in bs.items():
bs[key] = _get_from_env(item)
if bs['Type'].lower() == 's3':
backing_stores.append(S3FS(
bs['Bucket'],
strict=False,
aws_access_key_id=bs.get('Key ID', None),
aws_secret_access_key=bs.get('Secret Key', None),
endpoint_url=bs.get('Endpoint URL', None)
))
elif 'dav' in bs['Type'].lower():
if not webdav_available:
raise exceptions.NoWebdav("no webdavfs module was found")
if bs['Root'][0] != '/':
bs['Root'] = '/' + bs['Root']
backing_stores.append(WebDAVFS(
url=bs['Base URL'],
login=bs['Username'],
password=bs['Password'],
root=bs['Root']
))
else:
_config_error("Unknown filesystem type.")
else:
backing_stores.append(fs.open_fs(bs['URI'], create=True))
except (KeyError, OSError, CreateFailed) as err:
_config_error(err)
return backing_stores
def test_binder_heuristics(self):
from girder.plugins.wholetale.tasks.import_binder import sanitize_binder
tale = Tale().createTale(self.image, [], creator=self.user, title="Binder")
token = Token().createToken(user=self.user, days=0.25)
tmpdir = tempfile.mkdtemp()
with open(tmpdir + "/i_am_a_binder", "w") as fobj:
fobj.write("but well hidden!")
with tarfile.open(tmpdir + "/tale.tar.gz", "w:gz") as tar:
tar.add(tmpdir + "/i_am_a_binder", arcname="dir_in_tar/i_am_a_binder")
os.remove(tmpdir + "/i_am_a_binder")
with zipfile.ZipFile(tmpdir + "/tale.zip", "w") as myzip:
myzip.write(tmpdir + "/tale.tar.gz", arcname="dir_in_zip/tale.tar.gz")
os.remove(tmpdir + "/tale.tar.gz")
os.makedirs(tmpdir + "/hidden_binder")
os.rename(tmpdir + "/tale.zip", tmpdir + "/hidden_binder" + "/tale.zip")
girder_root = "http://localhost:{}".format(
config.getConfig()["server.socket_port"]
)
with WebDAVFS(
girder_root,
login=self.user["login"],
password="******".format(**token),
root="/tales/{_id}".format(**tale),
) as destination_fs, OSFS(tmpdir) as source_fs:
copy_fs(source_fs, destination_fs)
sanitize_binder(destination_fs)
self.assertEqual(destination_fs.listdir("/"), ["i_am_a_binder"])
shutil.rmtree(tmpdir)
Tale().remove(tale)
def test14RunDir(self):
resp = self.request(path="/version",
method="POST",
user=self.user,
params={"taleId": self.privateTale["_id"]})
self.assertStatusOk(resp)
version = resp.json
resp = self.request(
path="/run",
method="POST",
user=self.user,
params={
"versionId": version["_id"],
"name": "test run"
},
)
self.assertStatusOk(resp)
run_folder = Folder().load(resp.json["_id"], force=True)
url = f"http://127.0.0.1:{os.environ['GIRDER_PORT']}"
root = f"/runs/{run_folder['_id']}"
password = f"token:{self.token['_id']}"
time.sleep(1)
with WebDAVFS(url,
login=self.user["login"],
password=password,
root=root) as handle:
self.assertEqual(list(handle.listdir('.')), [])
handle.makedir('ala')
# exists in WebDAV
self.assertEqual(list(handle.listdir('.')), ['ala'])
# exists on the backend
physDirPath = pathlib.Path(
run_folder["fsPath"]) / "workspace" / "ala"
self.assertTrue(os.path.isdir(physDirPath))
handle.removedir('ala')
# gone from WebDAV
self.assertEqual(list(handle.listdir('.')), [])
# gone from the backend
self.assertFalse(os.path.isdir(physDirPath))
def _open_fs(self, user_context):
props = self._serialization_props(user_context)
handle = WebDAVFS(**props)
return handle
def get_fs(options):
from webdavfs.webdavfs import WebDAVFS
return WebDAVFS(**options)
def run(job):
jobModel = Job()
jobModel.updateJob(job, status=JobStatus.RUNNING)
lookup_kwargs, = job["args"]
user = User().load(job["userId"], force=True)
tale = Tale().load(job["kwargs"]["taleId"], user=user)
spawn = job["kwargs"]["spawn"]
asTale = job["kwargs"]["asTale"]
token = Token().createToken(user=user, days=0.5)
progressTotal = 3 + int(spawn)
progressCurrent = 0
try:
# 0. Spawn instance in the background
if spawn:
instance = Instance().createInstance(tale, user, token, spawn=spawn)
# 1. Register data using url
progressCurrent += 1
jobModel.updateJob(
job,
status=JobStatus.RUNNING,
progressTotal=progressTotal,
progressCurrent=progressCurrent,
progressMessage="Registering external data",
)
dataIds = lookup_kwargs.pop("dataId")
base_url = lookup_kwargs.get("base_url", DataONELocations.prod_cn)
dataMap = pids_to_entities(
dataIds, user=user, base_url=base_url, lookup=True
) # DataONE shouldn't be here
imported_data = register_dataMap(
dataMap,
getOrCreateRootFolder(CATALOG_NAME),
"folder",
user=user,
base_url=base_url,
)
if dataMap[0]["repository"].lower().startswith("http"):
resource = Item().load(imported_data[0], user=user, level=AccessType.READ)
resourceType = "item"
else:
resource = Folder().load(imported_data[0], user=user, level=AccessType.READ)
resourceType = "folder"
data_set = [
{
"itemId": imported_data[0],
"mountPath": resource["name"],
"_modelType": resourceType,
}
]
if asTale:
if resourceType == "folder":
# Create a dataset with the content of root ds folder,
# so that it looks nicely and it's easy to copy to workspace later on
workspace_data_set = [
{
"itemId": folder["_id"],
"mountPath": folder["name"],
"_modelType": "folder ",
}
for folder in Folder().childFolders(
parentType="folder", parent=resource, user=user
)
]
workspace_data_set += [
{
"itemId": item["_id"],
"mountPath": item["name"],
"_modelType": "item",
}
for item in Folder().childItems(resource)
]
else:
workspace_data_set = data_set
# 2. Create a session
# TODO: yay circular dependencies! IMHO we really should merge
# wholetale and wt_data_manager plugins...
from girder.plugins.wt_data_manager.models.session import Session
# Session is created so that we can easily copy files to workspace,
# without worrying about how to handler transfers. DMS will do that for us <3
session = Session().createSession(user, dataSet=workspace_data_set)
# 3. Copy data to the workspace using WebDAVFS
progressCurrent += 1
jobModel.updateJob(
job,
status=JobStatus.RUNNING,
log="Copying files to workspace",
progressTotal=progressTotal,
progressCurrent=progressCurrent,
progressMessage="Copying files to workspace",
)
girder_root = "http://localhost:{}".format(
config.getConfig()["server.socket_port"]
)
with WebDAVFS(
girder_root,
login=user["login"],
password="******".format(**token),
root="/tales/{_id}".format(**tale),
) as destination_fs, DMSFS(
str(session["_id"]), girder_root + "/api/v1", str(token["_id"])
) as source_fs:
copy_fs(source_fs, destination_fs)
sanitize_binder(destination_fs)
Session().deleteSession(user, session)
else:
# 3. Update Tale's dataSet
update_citations = {_["itemId"] for _ in tale["dataSet"]} ^ {
_["itemId"] for _ in data_set
}
tale["dataSet"] = data_set
tale = Tale().updateTale(tale)
if update_citations:
eventParams = {"tale": tale, "user": user}
event = events.trigger("tale.update_citation", eventParams)
if len(event.responses):
tale = Tale().updateTale(event.responses[-1])
# Tale is ready to be built
tale = Tale().load(tale["_id"], user=user) # Refresh state
tale["status"] = TaleStatus.READY
tale = Tale().updateTale(tale)
# 4. Wait for container to show up
if spawn:
progressCurrent += 1
jobModel.updateJob(
job,
status=JobStatus.RUNNING,
log="Waiting for a Tale container",
progressTotal=progressTotal,
progressCurrent=progressCurrent,
progressMessage="Waiting for a Tale container",
)
sleep_step = 10
timeout = 15 * 60
while instance["status"] == InstanceStatus.LAUNCHING and timeout > 0:
time.sleep(sleep_step)
instance = Instance().load(instance["_id"], user=user)
timeout -= sleep_step
if timeout <= 0:
raise RuntimeError(
"Failed to launch instance {}".format(instance["_id"])
)
else:
instance = None
except Exception:
tale = Tale().load(tale["_id"], user=user) # Refresh state
tale["status"] = TaleStatus.ERROR
tale = Tale().updateTale(tale)
t, val, tb = sys.exc_info()
log = "%s: %s\n%s" % (t.__name__, repr(val), traceback.extract_tb(tb))
jobModel.updateJob(
job,
progressTotal=progressTotal,
progressCurrent=progressTotal,
progressMessage="Task failed",
status=JobStatus.ERROR,
log=log,
)
raise
# To get rid of ObjectId's, dates etc.
tale = json.loads(
json.dumps(tale, sort_keys=True, allow_nan=False, cls=JsonEncoder)
)
instance = json.loads(
json.dumps(instance, sort_keys=True, allow_nan=False, cls=JsonEncoder)
)
jobModel.updateJob(
job,
status=JobStatus.SUCCESS,
log="Tale created",
progressTotal=progressTotal,
progressCurrent=progressTotal,
progressMessage="Tale created",
otherFields={"result": {"tale": tale, "instance": instance}},
)
def run(job):
jobModel = Job()
jobModel.updateJob(job, status=JobStatus.RUNNING)
tale_dir, manifest_file = job["args"]
user = User().load(job["userId"], force=True)
tale = Tale().load(job["kwargs"]["taleId"], user=user)
token = Token().createToken(user=user,
days=0.5,
scope=(TokenScope.USER_AUTH,
REST_CREATE_JOB_TOKEN_SCOPE))
progressTotal = 3
progressCurrent = 0
try:
os.chdir(tale_dir)
with open(manifest_file, "r") as manifest_fp:
manifest = json.load(manifest_fp)
# 1. Register data
progressCurrent += 1
jobModel.updateJob(
job,
status=JobStatus.RUNNING,
progressTotal=progressTotal,
progressCurrent=progressCurrent,
progressMessage="Registering external data",
)
dataIds = [obj["identifier"] for obj in manifest["Datasets"]]
dataIds += [
obj["uri"] for obj in manifest["aggregates"]
if obj["uri"].startswith("http")
]
if dataIds:
dataMap = pids_to_entities(
dataIds,
user=user,
base_url=DataONELocations.prod_cn,
lookup=True) # DataONE shouldn't be here
register_dataMap(
dataMap,
getOrCreateRootFolder(CATALOG_NAME),
"folder",
user=user,
base_url=DataONELocations.prod_cn,
)
# 2. Construct the dataSet
dataSet = []
for obj in manifest["aggregates"]:
if "bundledAs" not in obj:
continue
uri = obj["uri"]
fobj = File().findOne(
{"linkUrl": uri}) # TODO: That's expensive, use something else
if fobj:
dataSet.append({
"itemId": fobj["itemId"],
"_modelType": "item",
"mountPath": obj["bundledAs"]["filename"],
})
# TODO: handle folders
# 3. Update Tale's dataSet
update_citations = {_["itemId"]
for _ in tale["dataSet"]
} ^ {_["itemId"]
for _ in dataSet}
tale["dataSet"] = dataSet
tale = Tale().updateTale(tale)
if update_citations:
eventParams = {"tale": tale, "user": user}
event = events.trigger("tale.update_citation", eventParams)
if len(event.responses):
tale = Tale().updateTale(event.responses[-1])
# 4. Copy data to the workspace using WebDAVFS (if it exists)
progressCurrent += 1
jobModel.updateJob(
job,
status=JobStatus.RUNNING,
progressTotal=progressTotal,
progressCurrent=progressCurrent,
progressMessage="Copying files to workspace",
)
orig_tale_id = pathlib.Path(manifest_file).parts[0]
for workdir in ("workspace", "data/workspace", None):
if workdir:
workdir = os.path.join(orig_tale_id, workdir)
if os.path.isdir(workdir):
break
if workdir:
password = "******".format(**token)
root = "/tales/{_id}".format(**tale)
url = "http://localhost:{}".format(
config.getConfig()["server.socket_port"])
with WebDAVFS(url,
login=user["login"],
password=password,
root=root) as webdav_handle:
copy_fs(OSFS(workdir), webdav_handle)
# Tale is ready to be built
tale = Tale().load(tale["_id"], user=user) # Refresh state
tale["status"] = TaleStatus.READY
tale = Tale().updateTale(tale)
progressCurrent += 1
jobModel.updateJob(
job,
status=JobStatus.SUCCESS,
log="Tale created",
progressTotal=progressTotal,
progressCurrent=progressCurrent,
progressMessage="Tale created",
)
except Exception:
tale = Tale().load(tale["_id"], user=user) # Refresh state
tale["status"] = TaleStatus.ERROR
tale = Tale().updateTale(tale)
t, val, tb = sys.exc_info()
log = "%s: %s\n%s" % (t.__name__, repr(val), traceback.extract_tb(tb))
jobModel.updateJob(job, status=JobStatus.ERROR, log=log)
raise
def test13HomeDir(self):
resp = self.request(
path='/resource/lookup',
method='GET',
user=self.user,
params={'path': '/user/{login}/Home/ala'.format(**self.user)})
url = 'http://127.0.0.1:%s' % os.environ['GIRDER_PORT']
root = '/homes/{login}'.format(**self.user)
password = '******'.format(**self.token)
time.sleep(1)
with WebDAVFS(url,
login=self.user['login'],
password=password,
root=root) as handle:
self.assertEqual(list(handle.listdir('.')), [])
handle.makedir('ala')
# exists in WebDAV
self.assertEqual(list(handle.listdir('.')), ['ala'])
# exists on the backend
physDirPath = self.homesPhysicalPath(self.user['login'], 'ala')
self.assertTrue(os.path.isdir(physDirPath))
# exists in Girder
resp = self.request(
path='/resource/lookup',
method='GET',
user=self.user,
params={'path': '/user/{login}/Home/ala'.format(**self.user)})
self.assertStatusOk(resp)
self.assertEqual(resp.json['_modelType'], 'folder')
self.assertEqual(resp.json['name'], 'ala')
handle.removedir('ala')
# gone from WebDAV
self.assertEqual(list(handle.listdir('.')), [])
# gone from the backend
self.assertFalse(os.path.isdir(physDirPath))
# gone from Girder
resp = self.request(
path='/resource/lookup',
method='GET',
user=self.user,
params={'path': '/user/{login}/Home/ala'.format(**self.user)})
self.assertStatus(resp, 400)
self.assertEqual(
resp.json, {
'type':
'validation',
'message': ('Path not found: '
'user/{login}/Home/ala'.format(**self.user))
})
with WebDAVFS(url,
login=self.user['login'],
password=f"key:{self.api_key['key']}",
root=root) as handle:
self.assertEqual(list(handle.listdir('.')), [])
handle.makedir('test_dir')
with handle.open('test_dir/test_file.txt', 'w') as fp:
fsize = fp.write('Hello world!')
self.assertEqual(list(handle.listdir('.')), ['test_dir'])
self.assertEqual(list(handle.listdir('test_dir')),
['test_file.txt'])
fAbsPath = self.homesPhysicalPath(self.user['login'],
'test_dir/test_file.txt')
fAbsPathCopy = self.homesPhysicalPath(
self.user['login'], 'test_dir/test_file.txt (1)')
self.assertTrue(os.path.isfile(fAbsPath))
gabspath = '/user/{login}/Home/test_dir/test_file.txt'
resp = self.request(path='/resource/lookup',
method='GET',
user=self.user,
params={'path': gabspath.format(**self.user)})
self.assertStatusOk(resp)
self.assertEqual(resp.json['_modelType'], 'item')
self.assertEqual(resp.json['name'], 'test_file.txt')
self.assertEqual(resp.json['size'], fsize)
item = resp.json
resp = self.request(path='/item/{_id}/files'.format(**item),
method='GET',
user=self.user)
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 1)
gfile = resp.json[0]
self.assertEqual(gfile['size'], fsize)
resp = self.request(path='/item/{_id}/download'.format(**item),
method='GET',
user=self.user,
params={'contentDisposition': 'inline'},
isJson=False)
self.assertStatusOk(resp)
with open(fAbsPath, 'r') as fp:
self.assertEqual(self.getBody(resp), fp.read())
resp = self.request(path='/resource/copy',
method='POST',
user=self.user,
params={
'resources':
'{"item": ["%s"]}' % item['_id'],
'parentType': 'folder',
'parentId': item['folderId'],
'progress': False
})
self.assertStatusOk(resp)
self.assertTrue(os.path.isfile(fAbsPathCopy))
gabspath = '/user/{login}/Home/test_dir/test_file.txt (1)'
resp = self.request(path='/resource/lookup',
method='GET',
user=self.user,
params={'path': gabspath.format(**self.user)})
self.assertStatusOk(resp)
self.assertEqual(resp.json['_modelType'], 'item')
self.assertEqual(resp.json['name'], 'test_file.txt (1)')
self.assertEqual(resp.json['size'], fsize)
resp = self.request(path='/item/{_id}'.format(**resp.json),
method='DELETE',
user=self.user)
self.assertStatusOk(resp)
self.assertFalse(os.path.isfile(fAbsPathCopy))
resp = self.request(path='/item/{_id}'.format(**item),
method='DELETE',
user=self.user)
self.assertStatusOk(resp)
self.assertFalse(os.path.isfile(fAbsPath))
fAbsPath = os.path.dirname(fAbsPath)
self.assertTrue(os.path.isdir(fAbsPath))
resp = self.request(path='/folder/{folderId}'.format(**item),
method='DELETE',
user=self.user)
self.assertStatusOk(resp)
self.assertFalse(os.path.isdir(fAbsPath))
class DAVAdapter(Adapter):
def __init__(self, realm, pathMapper, app, user: dict, token, realmDir):
Adapter.__init__(self, realm, pathMapper, app, user)
url = 'http://127.0.0.1:%s' % os.environ['GIRDER_PORT']
password = '******' % token['_id']
self.root = '/%s/%s' % (realm, realmDir)
self.handle = WebDAVFS(url,
login=self.userName,
password=password,
root=self.root)
self.handle.makedir('test')
if not self.handle.isdir('test'):
raise Exception('Basic DAV test failed')
self.handle.removedir('test')
self.name = 'DAV'
def mkdir(self, path):
self.handle.makedir(path)
def rmdir(self, path):
self.handle.removedir(path)
def isdir(self, path):
return self.handle.isdir(path)
def isfile(self, path):
return self.handle.isfile(path)
def exists(self, path):
return self.handle.exists(path)
def size(self, path):
return self.handle.getsize(path)
def renamedir(self, src, dst):
# see mvdir
src = self.handle.validatepath(src)
dst = self.handle.validatepath(dst)
self.handle.client.move(src, dst)
def mvdir(self, dir, dst):
# WebDAVFS is being silly. Doesn't allow you to move directories (move requires
# the source to be a file, while movedir is not implemented, so it defaults to
# whatever FS does). Anyway, bypass it.
dir = self.handle.validatepath(dir)
dst = self.handle.validatepath(dst)
# handle.client.move() does not check reply for errors!
self.handle.client.move(dir, dst)
def mvfile(self, src, dst):
self.handle.move(src, dst, overwrite=False)
def mkfile(self, path, data):
with self.handle.open(path, 'w') as fp:
fsize = fp.write(data)
if fsize != len(data):
raise Exception('Could not write all data to DAV file')
def getfile(self, path):
with self.handle.open(path) as f:
return f.read()
def rm(self, path):
self.handle.remove(path)
def renamefile(self, src, dst):
self.handle.move(src, dst, overwrite=False)