class HdfsAssetstoreResource(Resource): def __init__(self): super(HdfsAssetstoreResource, self).__init__() self.resourceName = 'hdfs_assetstore' self.route('PUT', (':id', 'import'), self.importData) self.folderModel = Folder() # Save to avoid many lookups self.itemModel = Item() self.fileModel = File() def _importFile(self, parent, name, user, assetstore, node): item = self.itemModel.findOne({ 'folderId': parent['_id'], 'name': name }) if item is None: item = self.itemModel.createItem( name=name, creator=user, folder=parent) file = self.fileModel.findOne({ 'name': name, 'itemId': item['_id'] }) if file is None: file = self.fileModel.createFile( creator=user, item=item, name=name, size=node['length'], assetstore=assetstore, mimeType=None, saveFile=False) file['hdfs'] = { 'imported': True, 'path': node['path'] } self.fileModel.save(file) def _importData(self, parentType, parent, assetstore, client, path, ctx, user): for node in client.ls([path]): ctx.update(message='Importing ' + node['path']) name = posixpath.basename(node['path']) if node['file_type'] == 'd': folder = self.folderModel.findOne({ 'parentId': parent['_id'], 'name': name, 'parentCollection': parentType }) if folder is None: folder = self.folderModel.createFolder( parent, name, parentType=parentType, creator=user) self._importData('folder', folder, assetstore, client, node['path'], ctx, user) elif node['file_type'] == 'f' and parentType == 'folder': self._importFile(parent, name, user, assetstore, node) @access.admin @loadmodel(model='assetstore') @describeRoute( Description('Import a data hierarchy from an HDFS instance.') .notes('Only site administrators may use this endpoint.') .param('id', 'The ID of the assetstore representing the HDFS instance.', paramType='path') .param('parentId', 'The ID of the parent object in the Girder data ' 'hierarchy under which to import the files.') .param('parentType', 'The type of the parent object to import into.', enum=('folder', 'user', 'collection'), required=False) .param('path', 'Root of the directory structure (relative to the root ' 'of the HDFS) to import.') .param('progress', 'Whether to record progress on this operation (' 'default=False)', required=False, dataType='boolean') .errorResponse() .errorResponse('You are not an administrator.', 403) ) def importData(self, assetstore, params): self.requireParams(('parentId', 'path'), params) user = self.getCurrentUser() parentType = params.get('parentType', 'folder') if parentType not in ('user', 'collection', 'folder'): raise RestException('Invalid parentType.') parent = self.model(parentType).load(params['parentId'], force=True, exc=True) progress = self.boolParam('progress', params, default=False) client = HdfsClient( host=assetstore['hdfs']['host'], port=assetstore['hdfs']['port'], use_trash=False) path = params['path'] with ProgressContext(progress, user=user, title='Importing data from HDFS') as ctx: try: self._importData(parentType, parent, assetstore, client, path, ctx, user) except FileNotFoundException: raise RestException('File not found: %s.' % path)
class LabelResource(Resource): def __init__(self): super().__init__() self.resourceName = 'label' self.ann_file_name = "annotation.json" self.coll_m = Collection() self.file_m = File() self.folder_m = Folder() self.item_m = Item() self.upload_m = Upload() self.asset_m = Assetstore() self.setupRoutes() def setupRoutes(self): self.route('GET', (), handler=self.getLabelList) self.route('GET', (':label_id',), self.getLabel) self.route('GET', ('meta',), self.getLabelMeta) self.route('GET', ('create',), self.createLabelFile) self.route('GET', ('by_name',), self.getLabelByName) self.route('POST', (), self.postLabel) # ############# PUBLIC METHODS ################## @access.public @autoDescribeRoute( Description('Get label list').param('assign_id', 'assignment folder id')) @rest.rawResponse @trace def getLabelList(self, assign_id): files = self.folder_m.fileList(assign_id, user=self.getCurrentUser(), data=False, includeMetadata=True, mimeFilter=['application/json']) files = list(files) cherrypy.response.headers["Content-Type"] = "application/json" return dumps(files) @access.public @autoDescribeRoute( Description('Create a new label file inside the label folder if it doesnt exist') .param('assign_id', 'the parent folder id') .param('name', 'image name for which we are creating this label file')) @rest.rawResponse @trace def createLabelFile(self, assign_id, name): p_folder = self.folder_m.load(id=assign_id, user=self.getCurrentUser(), level=AccessType.WRITE) label_folder = find_folder(p_folder=p_folder, name=name, user=self.getCurrentUser(), desc="Label Folder", create=True) file = find_file(p_folder=label_folder, name=self.ann_file_name, user=self.getCurrentUser(), assetstore=self.asset_m.getCurrent(), create=False) if not file: file = create_new_file(p_folder=label_folder, name=self.ann_file_name, user=self.getCurrentUser(), assetstore=self.asset_m.getCurrent()) config_file = self.__find_config(assign_id) if not config_file: printFail("No config file found") return errorMessage("No config file found") else: res = copy_file(src_file=config_file, dest_file=file, user=self.getCurrentUser()) return dumps({ "label_id": res['fileId'] }) return dumps({ "label_id": file['_id'] }) @access.public @autoDescribeRoute( Description('Get labels by file_name') .param('name', 'label file name') .param('assign_id', 'the assignment id')) @rest.rawResponse @trace def getLabelByName(self, name, assign_id): p_folder = self.folder_m.load(assign_id, user=self.getCurrentUser(), level=AccessType.READ) label_folder = find_folder(p_folder=p_folder, name=name, user=self.getCurrentUser(), desc="Label Folder", create=True) # this file is created in <assign_folder>/<label_folder>/assignment.json file = find_file(p_folder=label_folder, name=self.ann_file_name, user=self.getCurrentUser(), assetstore=self.asset_m.getCurrent(), create=False) cherrypy.response.headers["Content-Type"] = "application/json" if file: return self.file_m.download(file) else: return dumps({}) @access.public @autoDescribeRoute( Description('Get label file by id') .param('label_id', 'label file id')) @rest.rawResponse @trace def getLabel(self, label_id): file = self.file_m.load(label_id, level=AccessType.READ, user=self.getCurrentUser()) printOk2(file) cherrypy.response.headers["Content-Type"] = "application/json" return self.file_m.download(file) @access.public @autoDescribeRoute( Description('Get label meta data by id') .param('label_id', 'label file id')) @trace def getLabelMeta(self, label_id): file = self.file_m.load(label_id, level=AccessType.READ, user=self.getCurrentUser()) cherrypy.response.headers["Content-Type"] = "application/json" return dumps(file) @access.public @autoDescribeRoute( Description('Post to label file by id') .param('label_id', 'label file id') .param('labels', 'labels to be updated')) @rest.rawResponse @trace def postLabel(self, label_id, labels): file = self.file_m.load(label_id, level=AccessType.WRITE, user=self.getCurrentUser()) cherrypy.response.headers["Content-Type"] = "application/json" params = {'labels': json.loads(labels)} data = json.dumps(params, indent=2, sort_keys=True) upload = writeData(self.getCurrentUser(), file, data) printOk2(file) printOk(upload) return dumps(upload) # ############# PRIVATE METHODS ################## def __create_new_file(self, folder, file_name): item = self.item_m.createItem(file_name, creator=self.getCurrentUser(), folder=folder, description='label file', reuseExisting=False) file = self.file_m.createFile(size=0, item=item, name=file_name, creator=self.getCurrentUser(), assetstore=self.asset_m.getCurrent(), mimeType="application/json") return file @staticmethod def __get_owner_id(folder): aclList = Folder().getFullAccessList(folder) for acl in aclList['users']: if acl['level'] == AccessType.ADMIN: return str(acl['id']) return None def __get_config_folder(self, label_folder_id): label_folder = Folder().load(label_folder_id, user=self.getCurrentUser(), level=AccessType.READ) ownerId = self.__get_owner_id(label_folder) config_folder = self.folder_m.load(label_folder['meta'][ownerId], level=AccessType.READ, user=self.getCurrentUser()) return config_folder def __find_config(self, folder_id): folder = self.__get_config_folder(folder_id) printOk2("Config folder {}".format(folder)) files = self.folder_m.fileList(folder, self.getCurrentUser(), data=False) for file_path, file in files: printOk(file) if file['name'] == "config.json": return file def __findFolder(self, p_folder, name, desc="", create=False): """ Find folder by name. If not found create the folder :param p_folder: parent folder :param name: name of the folder you want to find inside the parent folder :return: folder doc """ folder = list(self.folder_m.find({'folderId': p_folder['_id'], 'name': name}).limit(1)) if not folder: # check if you are allowed to create, else return nothing if create: folder = self.folder_m.createFolder(parent=p_folder, name=name, creator=self.getCurrentUser(), description=desc, reuseExisting=True) else: return None return folder def __findFile(self, folder, file_name): item = list(self.item_m.find({'folderId': folder['_id'], 'name': file_name}).limit(1)) if not item: return None item = item[0] file = list(self.file_m.find({'itemId': item['_id']}).limit(1)) if not file: return None return file[0]
class LabelResource(Resource): def __init__(self): super().__init__() self.resourceName = 'label' self.coll_m = Collection() self.file_m = File() self.folder_m = Folder() self.item_m = Item() self.upload_m = Upload() self.asset_m = Assetstore() self.setupRoutes() def setupRoutes(self): self.route('GET', (), handler=self.getLabelList) self.route('GET', (':label_id',), self.getLabel) self.route('GET', ('meta',), self.getLabelMeta) self.route('GET', ('create',), self.createLabelFile) self.route('GET', ('by_name',), self.getLabelByName) self.route('POST', (), self.postLabel) def createNewFile(self, folder, file_name): item = self.item_m.createItem(file_name, creator=self.getCurrentUser(), folder=folder, description='label file', reuseExisting=False) file = self.file_m.createFile(size=0, item=item, name=file_name, creator=self.getCurrentUser(), assetstore=self.asset_m.getCurrent(), mimeType="application/json") return file def copy(self, srcFile, destFile): upload = self.upload_m.createUploadToFile(destFile, self.getCurrentUser(), srcFile['size']) self.upload_m.handleChunk(upload=upload, chunk=RequestBodyStream(self.file_m.open(srcFile), size=destFile['size']), user=self.getCurrentUser()) return upload @access.public @autoDescribeRoute( Description('Get label list')) @rest.rawResponse def getLabelList(self): printOk('getLabelsList() was called!') try: collection = list(self.coll_m.list(user=self.getCurrentUser(), offset=0, limit=1))[0] files = self.coll_m.fileList(collection, user=self.getCurrentUser(), data=False, includeMetadata=True, mimeFilter=['application/json']) files = list(files) cherrypy.response.headers["Content-Type"] = "application/json" return dumps(files) except: printFail(traceback.print_exc) @staticmethod def getOwnerId(folder): aclList = Folder().getFullAccessList(folder) for acl in aclList['users']: if acl['level'] == AccessType.ADMIN: return str(acl['id']) return None def getConfigFolder(self, label_folder_id): label_folder = Folder().load(label_folder_id, user=self.getCurrentUser(), level=AccessType.READ) ownerId = self.getOwnerId(label_folder) config_folder = self.folder_m.load(label_folder['meta'][ownerId], level=AccessType.READ, user=self.getCurrentUser()) return config_folder def findConfig(self, folder_id): folder = self.getConfigFolder(folder_id) printOk2("Config folder {}".format(folder)) files = self.folder_m.fileList(folder, self.getCurrentUser(), data=False) for file_path, file in files: printOk(file) if file['name'] == "config.json": return file def __findFile(self, folder, file_name): item = list(self.item_m.find({'folderId': folder['_id'], 'name': file_name}).limit(1)) if not item: return None item = item[0] file = list(self.file_m.find({'itemId': item['_id']}).limit(1)) if not file: return None return file[0] @access.public @autoDescribeRoute( Description('Create a new label file if it doesnt exist') .param('file_name', 'label file name').param('folder_id', 'the parent folder id')) @rest.rawResponse def createLabelFile(self, file_name, folder_id): try: folder = self.folder_m.load(folder_id, user=self.getCurrentUser(), level=AccessType.WRITE) file = self.__findFile(folder, file_name) if not file: file = self.createNewFile(folder, file_name) config_file = self.findConfig(folder_id) if not config_file: printFail("No config file found") return errorMessage("No config file found") else: res = self.copy(config_file, file) return dumps({ "label_id": res['fileId'] }) return dumps({ "label_id": file['_id'] }) except: printFail(traceback.print_exc) cherrypy.response.status = 500 @access.public @autoDescribeRoute( Description('Get labels by file_name') .param('file_name', 'label file name').param('folder_id', 'the parent folder id')) @rest.rawResponse def getLabelByName(self, file_name, folder_id): try: folder = self.folder_m.load(folder_id, user=self.getCurrentUser(), level=AccessType.READ) file = self.__findFile(folder, file_name) cherrypy.response.headers["Content-Type"] = "application/json" if file: return self.file_m.download(file) else: return dumps({}) except: printFail(traceback.print_exc) cherrypy.response.status = 500 @access.public @autoDescribeRoute( Description('Get label by id') .param('label_id', 'label file id')) @rest.rawResponse def getLabel(self, label_id): try: file = self.file_m.load(label_id, level=AccessType.READ, user=self.getCurrentUser()) printOk2(file) cherrypy.response.headers["Content-Type"] = "application/json" return self.file_m.download(file) except: # Unknown slug printFail(traceback.print_exc) cherrypy.response.status = 404 @access.public @autoDescribeRoute( Description('Get label by id') .param('label_id', 'label file id')) def getLabelMeta(self, label_id): try: file = self.file_m.load(label_id, level=AccessType.READ, user=self.getCurrentUser()) cherrypy.response.headers["Content-Type"] = "application/json" return dumps(file) except: # Unknown slug printFail(traceback.print_exc) cherrypy.response.status = 404 @access.public @autoDescribeRoute( Description('Post label by id') .param('label_id', 'label file id')) @rest.rawResponse def postLabel(self, label_id, params): try: file = self.file_m.load(label_id, level=AccessType.WRITE, user=self.getCurrentUser()) cherrypy.response.headers["Content-Type"] = "application/json" params['labels'] = json.loads(params['labels']) data = json.dumps(params, indent=2, sort_keys=True) upload = writeData(self.getCurrentUser(), file, data) printOk2(file) printOk(upload) return dumps(upload) except: # Unknown slug printFail(traceback.print_exc) cherrypy.response.status = 404 @access.public @autoDescribeRoute( Description('Post label by id') .param('label_id', 'label file id')) @rest.rawResponse def strokeToOutline(self, strokes): pass