def __init__(self, realm, pathMapper, app, user: dict, token, realmDir): Adapter.__init__(self, realm, pathMapper, app, user) url = 'http://127.0.0.1:%s' % os.environ['GIRDER_PORT'] password = '******' % token['_id'] self.root = '/%s/%s' % (realm, realmDir) self.handle = WebDAVFS(url, login=self.userName, password=password, root=self.root) self.handle.makedir('test') if not self.handle.isdir('test'): raise Exception('Basic DAV test failed') self.handle.removedir('test') self.name = 'DAV'
def _configure_backing_store(self): try: backing_stores = [] for bs in self.config['Backing Store']: if 'Type' in bs: for key, item in bs.items(): bs[key] = _get_from_env(item) if bs['Type'].lower() == 's3': backing_stores.append(S3FS( bs['Bucket'], strict=False, aws_access_key_id=bs.get('Key ID', None), aws_secret_access_key=bs.get('Secret Key', None), endpoint_url=bs.get('Endpoint URL', None) )) elif 'dav' in bs['Type'].lower(): if not webdav_available: raise exceptions.NoWebdav("no webdavfs module was found") if bs['Root'][0] != '/': bs['Root'] = '/' + bs['Root'] backing_stores.append(WebDAVFS( url=bs['Base URL'], login=bs['Username'], password=bs['Password'], root=bs['Root'] )) else: _config_error("Unknown filesystem type.") else: backing_stores.append(fs.open_fs(bs['URI'], create=True)) except (KeyError, OSError, CreateFailed) as err: _config_error(err) return backing_stores
def test_binder_heuristics(self): from girder.plugins.wholetale.tasks.import_binder import sanitize_binder tale = Tale().createTale(self.image, [], creator=self.user, title="Binder") token = Token().createToken(user=self.user, days=0.25) tmpdir = tempfile.mkdtemp() with open(tmpdir + "/i_am_a_binder", "w") as fobj: fobj.write("but well hidden!") with tarfile.open(tmpdir + "/tale.tar.gz", "w:gz") as tar: tar.add(tmpdir + "/i_am_a_binder", arcname="dir_in_tar/i_am_a_binder") os.remove(tmpdir + "/i_am_a_binder") with zipfile.ZipFile(tmpdir + "/tale.zip", "w") as myzip: myzip.write(tmpdir + "/tale.tar.gz", arcname="dir_in_zip/tale.tar.gz") os.remove(tmpdir + "/tale.tar.gz") os.makedirs(tmpdir + "/hidden_binder") os.rename(tmpdir + "/tale.zip", tmpdir + "/hidden_binder" + "/tale.zip") girder_root = "http://localhost:{}".format( config.getConfig()["server.socket_port"] ) with WebDAVFS( girder_root, login=self.user["login"], password="******".format(**token), root="/tales/{_id}".format(**tale), ) as destination_fs, OSFS(tmpdir) as source_fs: copy_fs(source_fs, destination_fs) sanitize_binder(destination_fs) self.assertEqual(destination_fs.listdir("/"), ["i_am_a_binder"]) shutil.rmtree(tmpdir) Tale().remove(tale)
def test14RunDir(self): resp = self.request(path="/version", method="POST", user=self.user, params={"taleId": self.privateTale["_id"]}) self.assertStatusOk(resp) version = resp.json resp = self.request( path="/run", method="POST", user=self.user, params={ "versionId": version["_id"], "name": "test run" }, ) self.assertStatusOk(resp) run_folder = Folder().load(resp.json["_id"], force=True) url = f"http://127.0.0.1:{os.environ['GIRDER_PORT']}" root = f"/runs/{run_folder['_id']}" password = f"token:{self.token['_id']}" time.sleep(1) with WebDAVFS(url, login=self.user["login"], password=password, root=root) as handle: self.assertEqual(list(handle.listdir('.')), []) handle.makedir('ala') # exists in WebDAV self.assertEqual(list(handle.listdir('.')), ['ala']) # exists on the backend physDirPath = pathlib.Path( run_folder["fsPath"]) / "workspace" / "ala" self.assertTrue(os.path.isdir(physDirPath)) handle.removedir('ala') # gone from WebDAV self.assertEqual(list(handle.listdir('.')), []) # gone from the backend self.assertFalse(os.path.isdir(physDirPath))
def _open_fs(self, user_context): props = self._serialization_props(user_context) handle = WebDAVFS(**props) return handle
def get_fs(options): from webdavfs.webdavfs import WebDAVFS return WebDAVFS(**options)
def run(job): jobModel = Job() jobModel.updateJob(job, status=JobStatus.RUNNING) lookup_kwargs, = job["args"] user = User().load(job["userId"], force=True) tale = Tale().load(job["kwargs"]["taleId"], user=user) spawn = job["kwargs"]["spawn"] asTale = job["kwargs"]["asTale"] token = Token().createToken(user=user, days=0.5) progressTotal = 3 + int(spawn) progressCurrent = 0 try: # 0. Spawn instance in the background if spawn: instance = Instance().createInstance(tale, user, token, spawn=spawn) # 1. Register data using url progressCurrent += 1 jobModel.updateJob( job, status=JobStatus.RUNNING, progressTotal=progressTotal, progressCurrent=progressCurrent, progressMessage="Registering external data", ) dataIds = lookup_kwargs.pop("dataId") base_url = lookup_kwargs.get("base_url", DataONELocations.prod_cn) dataMap = pids_to_entities( dataIds, user=user, base_url=base_url, lookup=True ) # DataONE shouldn't be here imported_data = register_dataMap( dataMap, getOrCreateRootFolder(CATALOG_NAME), "folder", user=user, base_url=base_url, ) if dataMap[0]["repository"].lower().startswith("http"): resource = Item().load(imported_data[0], user=user, level=AccessType.READ) resourceType = "item" else: resource = Folder().load(imported_data[0], user=user, level=AccessType.READ) resourceType = "folder" data_set = [ { "itemId": imported_data[0], "mountPath": resource["name"], "_modelType": resourceType, } ] if asTale: if resourceType == "folder": # Create a dataset with the content of root ds folder, # so that it looks nicely and it's easy to copy to workspace later on workspace_data_set = [ { "itemId": folder["_id"], "mountPath": folder["name"], "_modelType": "folder ", } for folder in Folder().childFolders( parentType="folder", parent=resource, user=user ) ] workspace_data_set += [ { "itemId": item["_id"], "mountPath": item["name"], "_modelType": "item", } for item in Folder().childItems(resource) ] else: workspace_data_set = data_set # 2. Create a session # TODO: yay circular dependencies! IMHO we really should merge # wholetale and wt_data_manager plugins... from girder.plugins.wt_data_manager.models.session import Session # Session is created so that we can easily copy files to workspace, # without worrying about how to handler transfers. DMS will do that for us <3 session = Session().createSession(user, dataSet=workspace_data_set) # 3. Copy data to the workspace using WebDAVFS progressCurrent += 1 jobModel.updateJob( job, status=JobStatus.RUNNING, log="Copying files to workspace", progressTotal=progressTotal, progressCurrent=progressCurrent, progressMessage="Copying files to workspace", ) girder_root = "http://localhost:{}".format( config.getConfig()["server.socket_port"] ) with WebDAVFS( girder_root, login=user["login"], password="******".format(**token), root="/tales/{_id}".format(**tale), ) as destination_fs, DMSFS( str(session["_id"]), girder_root + "/api/v1", str(token["_id"]) ) as source_fs: copy_fs(source_fs, destination_fs) sanitize_binder(destination_fs) Session().deleteSession(user, session) else: # 3. Update Tale's dataSet update_citations = {_["itemId"] for _ in tale["dataSet"]} ^ { _["itemId"] for _ in data_set } tale["dataSet"] = data_set tale = Tale().updateTale(tale) if update_citations: eventParams = {"tale": tale, "user": user} event = events.trigger("tale.update_citation", eventParams) if len(event.responses): tale = Tale().updateTale(event.responses[-1]) # Tale is ready to be built tale = Tale().load(tale["_id"], user=user) # Refresh state tale["status"] = TaleStatus.READY tale = Tale().updateTale(tale) # 4. Wait for container to show up if spawn: progressCurrent += 1 jobModel.updateJob( job, status=JobStatus.RUNNING, log="Waiting for a Tale container", progressTotal=progressTotal, progressCurrent=progressCurrent, progressMessage="Waiting for a Tale container", ) sleep_step = 10 timeout = 15 * 60 while instance["status"] == InstanceStatus.LAUNCHING and timeout > 0: time.sleep(sleep_step) instance = Instance().load(instance["_id"], user=user) timeout -= sleep_step if timeout <= 0: raise RuntimeError( "Failed to launch instance {}".format(instance["_id"]) ) else: instance = None except Exception: tale = Tale().load(tale["_id"], user=user) # Refresh state tale["status"] = TaleStatus.ERROR tale = Tale().updateTale(tale) t, val, tb = sys.exc_info() log = "%s: %s\n%s" % (t.__name__, repr(val), traceback.extract_tb(tb)) jobModel.updateJob( job, progressTotal=progressTotal, progressCurrent=progressTotal, progressMessage="Task failed", status=JobStatus.ERROR, log=log, ) raise # To get rid of ObjectId's, dates etc. tale = json.loads( json.dumps(tale, sort_keys=True, allow_nan=False, cls=JsonEncoder) ) instance = json.loads( json.dumps(instance, sort_keys=True, allow_nan=False, cls=JsonEncoder) ) jobModel.updateJob( job, status=JobStatus.SUCCESS, log="Tale created", progressTotal=progressTotal, progressCurrent=progressTotal, progressMessage="Tale created", otherFields={"result": {"tale": tale, "instance": instance}}, )
def run(job): jobModel = Job() jobModel.updateJob(job, status=JobStatus.RUNNING) tale_dir, manifest_file = job["args"] user = User().load(job["userId"], force=True) tale = Tale().load(job["kwargs"]["taleId"], user=user) token = Token().createToken(user=user, days=0.5, scope=(TokenScope.USER_AUTH, REST_CREATE_JOB_TOKEN_SCOPE)) progressTotal = 3 progressCurrent = 0 try: os.chdir(tale_dir) with open(manifest_file, "r") as manifest_fp: manifest = json.load(manifest_fp) # 1. Register data progressCurrent += 1 jobModel.updateJob( job, status=JobStatus.RUNNING, progressTotal=progressTotal, progressCurrent=progressCurrent, progressMessage="Registering external data", ) dataIds = [obj["identifier"] for obj in manifest["Datasets"]] dataIds += [ obj["uri"] for obj in manifest["aggregates"] if obj["uri"].startswith("http") ] if dataIds: dataMap = pids_to_entities( dataIds, user=user, base_url=DataONELocations.prod_cn, lookup=True) # DataONE shouldn't be here register_dataMap( dataMap, getOrCreateRootFolder(CATALOG_NAME), "folder", user=user, base_url=DataONELocations.prod_cn, ) # 2. Construct the dataSet dataSet = [] for obj in manifest["aggregates"]: if "bundledAs" not in obj: continue uri = obj["uri"] fobj = File().findOne( {"linkUrl": uri}) # TODO: That's expensive, use something else if fobj: dataSet.append({ "itemId": fobj["itemId"], "_modelType": "item", "mountPath": obj["bundledAs"]["filename"], }) # TODO: handle folders # 3. Update Tale's dataSet update_citations = {_["itemId"] for _ in tale["dataSet"] } ^ {_["itemId"] for _ in dataSet} tale["dataSet"] = dataSet tale = Tale().updateTale(tale) if update_citations: eventParams = {"tale": tale, "user": user} event = events.trigger("tale.update_citation", eventParams) if len(event.responses): tale = Tale().updateTale(event.responses[-1]) # 4. Copy data to the workspace using WebDAVFS (if it exists) progressCurrent += 1 jobModel.updateJob( job, status=JobStatus.RUNNING, progressTotal=progressTotal, progressCurrent=progressCurrent, progressMessage="Copying files to workspace", ) orig_tale_id = pathlib.Path(manifest_file).parts[0] for workdir in ("workspace", "data/workspace", None): if workdir: workdir = os.path.join(orig_tale_id, workdir) if os.path.isdir(workdir): break if workdir: password = "******".format(**token) root = "/tales/{_id}".format(**tale) url = "http://localhost:{}".format( config.getConfig()["server.socket_port"]) with WebDAVFS(url, login=user["login"], password=password, root=root) as webdav_handle: copy_fs(OSFS(workdir), webdav_handle) # Tale is ready to be built tale = Tale().load(tale["_id"], user=user) # Refresh state tale["status"] = TaleStatus.READY tale = Tale().updateTale(tale) progressCurrent += 1 jobModel.updateJob( job, status=JobStatus.SUCCESS, log="Tale created", progressTotal=progressTotal, progressCurrent=progressCurrent, progressMessage="Tale created", ) except Exception: tale = Tale().load(tale["_id"], user=user) # Refresh state tale["status"] = TaleStatus.ERROR tale = Tale().updateTale(tale) t, val, tb = sys.exc_info() log = "%s: %s\n%s" % (t.__name__, repr(val), traceback.extract_tb(tb)) jobModel.updateJob(job, status=JobStatus.ERROR, log=log) raise
def test13HomeDir(self): resp = self.request( path='/resource/lookup', method='GET', user=self.user, params={'path': '/user/{login}/Home/ala'.format(**self.user)}) url = 'http://127.0.0.1:%s' % os.environ['GIRDER_PORT'] root = '/homes/{login}'.format(**self.user) password = '******'.format(**self.token) time.sleep(1) with WebDAVFS(url, login=self.user['login'], password=password, root=root) as handle: self.assertEqual(list(handle.listdir('.')), []) handle.makedir('ala') # exists in WebDAV self.assertEqual(list(handle.listdir('.')), ['ala']) # exists on the backend physDirPath = self.homesPhysicalPath(self.user['login'], 'ala') self.assertTrue(os.path.isdir(physDirPath)) # exists in Girder resp = self.request( path='/resource/lookup', method='GET', user=self.user, params={'path': '/user/{login}/Home/ala'.format(**self.user)}) self.assertStatusOk(resp) self.assertEqual(resp.json['_modelType'], 'folder') self.assertEqual(resp.json['name'], 'ala') handle.removedir('ala') # gone from WebDAV self.assertEqual(list(handle.listdir('.')), []) # gone from the backend self.assertFalse(os.path.isdir(physDirPath)) # gone from Girder resp = self.request( path='/resource/lookup', method='GET', user=self.user, params={'path': '/user/{login}/Home/ala'.format(**self.user)}) self.assertStatus(resp, 400) self.assertEqual( resp.json, { 'type': 'validation', 'message': ('Path not found: ' 'user/{login}/Home/ala'.format(**self.user)) }) with WebDAVFS(url, login=self.user['login'], password=f"key:{self.api_key['key']}", root=root) as handle: self.assertEqual(list(handle.listdir('.')), []) handle.makedir('test_dir') with handle.open('test_dir/test_file.txt', 'w') as fp: fsize = fp.write('Hello world!') self.assertEqual(list(handle.listdir('.')), ['test_dir']) self.assertEqual(list(handle.listdir('test_dir')), ['test_file.txt']) fAbsPath = self.homesPhysicalPath(self.user['login'], 'test_dir/test_file.txt') fAbsPathCopy = self.homesPhysicalPath( self.user['login'], 'test_dir/test_file.txt (1)') self.assertTrue(os.path.isfile(fAbsPath)) gabspath = '/user/{login}/Home/test_dir/test_file.txt' resp = self.request(path='/resource/lookup', method='GET', user=self.user, params={'path': gabspath.format(**self.user)}) self.assertStatusOk(resp) self.assertEqual(resp.json['_modelType'], 'item') self.assertEqual(resp.json['name'], 'test_file.txt') self.assertEqual(resp.json['size'], fsize) item = resp.json resp = self.request(path='/item/{_id}/files'.format(**item), method='GET', user=self.user) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 1) gfile = resp.json[0] self.assertEqual(gfile['size'], fsize) resp = self.request(path='/item/{_id}/download'.format(**item), method='GET', user=self.user, params={'contentDisposition': 'inline'}, isJson=False) self.assertStatusOk(resp) with open(fAbsPath, 'r') as fp: self.assertEqual(self.getBody(resp), fp.read()) resp = self.request(path='/resource/copy', method='POST', user=self.user, params={ 'resources': '{"item": ["%s"]}' % item['_id'], 'parentType': 'folder', 'parentId': item['folderId'], 'progress': False }) self.assertStatusOk(resp) self.assertTrue(os.path.isfile(fAbsPathCopy)) gabspath = '/user/{login}/Home/test_dir/test_file.txt (1)' resp = self.request(path='/resource/lookup', method='GET', user=self.user, params={'path': gabspath.format(**self.user)}) self.assertStatusOk(resp) self.assertEqual(resp.json['_modelType'], 'item') self.assertEqual(resp.json['name'], 'test_file.txt (1)') self.assertEqual(resp.json['size'], fsize) resp = self.request(path='/item/{_id}'.format(**resp.json), method='DELETE', user=self.user) self.assertStatusOk(resp) self.assertFalse(os.path.isfile(fAbsPathCopy)) resp = self.request(path='/item/{_id}'.format(**item), method='DELETE', user=self.user) self.assertStatusOk(resp) self.assertFalse(os.path.isfile(fAbsPath)) fAbsPath = os.path.dirname(fAbsPath) self.assertTrue(os.path.isdir(fAbsPath)) resp = self.request(path='/folder/{folderId}'.format(**item), method='DELETE', user=self.user) self.assertStatusOk(resp) self.assertFalse(os.path.isdir(fAbsPath))
class DAVAdapter(Adapter): def __init__(self, realm, pathMapper, app, user: dict, token, realmDir): Adapter.__init__(self, realm, pathMapper, app, user) url = 'http://127.0.0.1:%s' % os.environ['GIRDER_PORT'] password = '******' % token['_id'] self.root = '/%s/%s' % (realm, realmDir) self.handle = WebDAVFS(url, login=self.userName, password=password, root=self.root) self.handle.makedir('test') if not self.handle.isdir('test'): raise Exception('Basic DAV test failed') self.handle.removedir('test') self.name = 'DAV' def mkdir(self, path): self.handle.makedir(path) def rmdir(self, path): self.handle.removedir(path) def isdir(self, path): return self.handle.isdir(path) def isfile(self, path): return self.handle.isfile(path) def exists(self, path): return self.handle.exists(path) def size(self, path): return self.handle.getsize(path) def renamedir(self, src, dst): # see mvdir src = self.handle.validatepath(src) dst = self.handle.validatepath(dst) self.handle.client.move(src, dst) def mvdir(self, dir, dst): # WebDAVFS is being silly. Doesn't allow you to move directories (move requires # the source to be a file, while movedir is not implemented, so it defaults to # whatever FS does). Anyway, bypass it. dir = self.handle.validatepath(dir) dst = self.handle.validatepath(dst) # handle.client.move() does not check reply for errors! self.handle.client.move(dir, dst) def mvfile(self, src, dst): self.handle.move(src, dst, overwrite=False) def mkfile(self, path, data): with self.handle.open(path, 'w') as fp: fsize = fp.write(data) if fsize != len(data): raise Exception('Could not write all data to DAV file') def getfile(self, path): with self.handle.open(path) as f: return f.read() def rm(self, path): self.handle.remove(path) def renamefile(self, src, dst): self.handle.move(src, dst, overwrite=False)