def updateItemLicense(event):
"""
REST event handler to update item with license parameter, if provided.
"""
params = event.info['params']
if 'license' not in params:
return
itemModel = Item()
item = itemModel.load(event.info['returnVal']['_id'], force=True, exc=True)
newLicense = validateString(params['license'])
if item['license'] == newLicense:
return
# Ensure that new license name is in configured list of licenses.
#
# Enforcing this here, instead of when validating the item, avoids an extra
# database lookup (for the settings) on every future item save.
if newLicense:
licenseSetting = Setting().get(PluginSettings.LICENSES)
validLicense = any(
license['name'] == newLicense
for group in licenseSetting
for license in group['licenses'])
if not validLicense:
raise ValidationException(
'License name must be in configured list of licenses.', 'license')
item['license'] = newLicense
item = itemModel.save(item)
event.preventDefault()
event.addResponse(item)
def testFileProcessHandler(self):
admin, user = self.users
# Create a collection, folder, and item
collection = Collection().createCollection('collection1', admin, public=True)
folder = Folder().createFolder(collection, 'folder1', parentType='collection', public=True)
item = Item().createItem('item1', admin, folder)
# Upload non-DICOM files
self._uploadNonDicomFiles(item, admin)
nonDicomItem = Item().load(item['_id'], force=True)
self.assertIsNone(nonDicomItem.get('dicom'))
# Upload DICOM files
self._uploadDicomFiles(item, admin)
# Check if the 'dicomItem' is well processed
dicomItem = Item().load(item['_id'], force=True)
self.assertIn('dicom', dicomItem)
self.assertHasKeys(dicomItem['dicom'], ['meta', 'files'])
# Check if the files list contain the good keys and all the file are well sorted
for i in range(0, 4):
self.assertTrue('_id' in dicomItem['dicom']['files'][i])
self.assertTrue('name' in dicomItem['dicom']['files'][i])
self.assertEqual(dicomItem['dicom']['files'][i]['name'], 'dicomFile{}.dcm'.format(i))
self.assertTrue('SeriesNumber' in dicomItem['dicom']['files'][i]['dicom'])
self.assertTrue('InstanceNumber' in dicomItem['dicom']['files'][i]['dicom'])
self.assertTrue('SliceLocation' in dicomItem['dicom']['files'][i]['dicom'])
# Check the common metadata
self.assertIsNotNone(dicomItem['dicom']['meta'])
def testMakeDicomItem(self):
admin, user = self.users
# create a collection, folder, and item
collection = Collection().createCollection('collection2', admin, public=True)
folder = Folder().createFolder(collection, 'folder2', parentType='collection', public=True)
item = Item().createItem('item2', admin, folder)
# Upload files
self._uploadDicomFiles(item, admin)
# Check the endpoint 'parseDicom' for an admin user
dicomItem = Item().load(item['_id'], force=True)
dicomItem = self._purgeDicomItem(dicomItem)
path = '/item/%s/parseDicom' % dicomItem.get('_id')
resp = self.request(path=path, method='POST', user=admin)
self.assertStatusOk(resp)
dicomItem = Item().load(item['_id'], force=True)
self.assertIn('dicom', dicomItem)
self.assertHasKeys(dicomItem['dicom'], ['meta', 'files'])
# Check the endpoint 'parseDicom' for an non admin user
dicomItem = Item().load(item['_id'], force=True)
dicomItem = self._purgeDicomItem(dicomItem)
path = '/item/%s/parseDicom' % dicomItem.get('_id')
resp = self.request(path=path, method='POST', user=user)
self.assertStatus(resp, 403)
def _uploadComplete(event):
"""
Called after an upload finishes. We check if our current token is a special
authorized upload token, and if so, delete it.
TODO we could alternatively keep a reference count inside each token that authorized
more than a single upload at a time, and just decrement it here.
"""
token = getCurrentToken()
if token and 'authorizedUploadId' in token:
user = User().load(token['userId'], force=True)
item = Item().load(event.info['file']['itemId'], force=True)
# Save the metadata on the item
item['description'] = token['authorizedUploadDescription']
item['authorizedUploadEmail'] = token['authorizedUploadEmail']
Item().save(item)
text = mail_utils.renderTemplate('authorized_upload.uploadFinished.mako', {
'itemId': item['_id'],
'itemName': item['name'],
'itemDescription': item.get('description', '')
})
mail_utils.sendEmail(to=user['email'], subject='Authorized upload complete', text=text)
Token().remove(token)
def testCreationOnUpload(self):
resp = self.request(
path='/file', method='POST', user=self.admin, params={
'parentType': 'folder',
'parentId': self.publicFolder['_id'],
'name': 'test.png',
'size': len(self.image),
'reference': json.dumps({
'thumbnail': {
'width': 100
}
})
})
self.assertStatusOk(resp)
resp = self.request(
path='/file/chunk', method='POST', user=self.admin, body=self.image, params={
'offset': 0,
'uploadId': resp.json['_id']
}, type='image/png')
self.assertStatusOk(resp)
self.assertIn('itemId', resp.json)
itemId = resp.json['itemId']
start = time.time()
while time.time() - start < 15:
# Wait for thumbnail creation
item = Item().load(itemId, force=True)
if item.get('_thumbnails'):
break
time.sleep(0.1)
self.assertEqual(len(item['_thumbnails']), 1)
file = File().load(item['_thumbnails'][0], force=True)
with File().open(file) as fh:
self.assertEqual(fh.read(2), b'\xff\xd8') # jpeg magic number
def __init__(self):
super(HdfsAssetstoreResource, self).__init__()
self.resourceName = 'hdfs_assetstore'
self.route('PUT', (':id', 'import'), self.importData)
self.folderModel = Folder() # Save to avoid many lookups
self.itemModel = Item()
self.fileModel = File()
def __init__(self):
super().__init__()
self.resourceName = 'label'
self.coll_m = Collection()
self.file_m = File()
self.folder_m = Folder()
self.item_m = Item()
self.upload_m = Upload()
self.asset_m = Assetstore()
self.setupRoutes()
def _virtualChildItems(self, event):
params = event.info['params']
if 'folderId' not in params:
return # This is not a child listing request
user = self.getCurrentUser()
folder = Folder().load(params['folderId'], user=user, level=AccessType.READ)
if not folder.get('isVirtual') or 'virtualItemsQuery' not in folder:
return # Parent is not a virtual folder, proceed as normal
limit, offset, sort = self.getPagingParameters(params, defaultSortField='name')
q = json_util.loads(folder['virtualItemsQuery'])
if 'virtualItemsSort' in folder:
sort = json.loads(folder['virtualItemsSort'])
item = Item()
# These items may reside in folders that the user cannot read, so we must filter
items = item.filterResultsByPermission(
item.find(q, sort=sort), user, level=AccessType.READ, limit=limit, offset=offset)
event.preventDefault().addResponse([item.filter(i, user) for i in items])
def __init__(self):
super(Item, self).__init__()
self.resourceName = 'item'
self._model = ItemModel()
self.route('DELETE', (':id',), self.deleteItem)
self.route('GET', (), self.find)
self.route('GET', (':id',), self.getItem)
self.route('GET', (':id', 'files'), self.getFiles)
self.route('GET', (':id', 'download'), self.download)
self.route('GET', (':id', 'rootpath'), self.rootpath)
self.route('POST', (), self.createItem)
self.route('PUT', (':id',), self.updateItem)
self.route('POST', (':id', 'copy'), self.copyItem)
self.route('PUT', (':id', 'metadata'), self.setMetadata)
self.route('DELETE', (':id', 'metadata'), self.deleteMetadata)
def process_metadata(event):
"""Add metadata to an item on a ``data.process`` event"""
results = _itemFromEvent(event, 'ItemMetadata', AccessType.WRITE)
if not results:
return
file = File().load(event.info.get('file', {}).get('_id'),
level=AccessType.READ,
user=results['user'])
if not file:
logger.error('Could not load models from the database')
return
try:
data = json.loads(b''.join(File().download(file)()).decode('utf8'))
except Exception:
logger.error('Could not parse metadata file')
raise
item = results['item']
Item().setMetadata(item, data, allowNull=False)
def load(self, info):
getPlugin('worker').load(info)
events.bind('model.item.remove', __name__,
lambda e: _removeThumbnails(e.info))
events.bind('model.file.finalizeUpload.after', __name__, _handleUpload)
File().ensureIndex(([('interactive_thumbnails_uid', 1),
('attachedToId', 1)], {
'sparse': True
}))
File().exposeFields(level=AccessType.READ,
fields={'interactive_thumbnails_info'})
Item().exposeFields(level=AccessType.READ,
fields={'hasInteractiveThumbnail'})
info['apiRoot'].item.route('GET',
(':id', 'interactive_thumbnail', ':uid'),
_getThumbnail)
info['apiRoot'].item.route('POST', (':id', 'interactive_thumbnail'),
_createThumbnail)
def testMigrateAnnotationAccessControl(self, user, admin):
publicFolder = utilities.namedFolder(admin, 'Public')
# create an annotation
item = Item().createItem('userItem', user, publicFolder)
annot = Annotation().createAnnotation(item, admin, sampleAnnotation)
# assert ACL's work
with pytest.raises(AccessException):
Annotation().load(annot['_id'], user=user, level=AccessType.WRITE)
# remove the access control properties and save back to the database
del annot['access']
del annot['public']
Annotation().save(annot)
# load the annotation and assert access properties were added
annot = Annotation().load(annot['_id'], force=True)
assert annot['access'] == publicFolder['access']
assert annot['public'] is True
def test_grid_geometa(server, admin, fsAssetstore, testFile, expected):
name = os.path.basename(testFile)
public = server.request(path='/folder',
user=admin,
method='GET',
params={
'parentId': admin['_id'],
'parentType': 'user',
'name': 'Public'
})
assertStatusOk(public)
with open(testFile, 'rb') as f:
uploadedFile = server.uploadFile(name, f.read(), admin, public.json[0])
document = Item().load(uploadedFile['itemId'], user=admin)
with open(expected, 'r') as f:
expectedJson = json.load(f)
assert list(document['geometa']).sort() == list(expectedJson).sort()
def testAnnotationGroup(self, admin):
publicFolder = utilities.namedFolder(admin, 'Public')
item = Item().createItem('sample', admin, publicFolder)
elements = [{
'type': 'rectangle',
'center': [20.0, 25.0, 0],
'width': 14.0,
'height': 15.0,
'group': 'a'
}, {
'type': 'rectangle',
'center': [40.0, 15.0, 0],
'width': 5.0,
'height': 5.0
}]
annotationWithGroup = {'name': 'groups', 'elements': elements}
annot = Annotation().createAnnotation(item, admin, annotationWithGroup)
result = Annotation().load(annot['_id'], user=admin)
assert result['annotation']['elements'][0]['group'] == 'a'
def _fixBaseParents(self, progress):
fixes = 0
models = [Folder(), Item()]
steps = sum(model.find().count() for model in models)
progress.update(total=steps, current=0)
for model in models:
for doc in model.find():
progress.update(increment=1)
baseParent = model.parentsToRoot(doc, force=True)[0]
baseParentType = baseParent['type']
baseParentId = baseParent['object']['_id']
if (doc['baseParentType'] != baseParentType or
doc['baseParentId'] != baseParentId):
model.update({'_id': doc['_id']}, update={
'$set': {
'baseParentType': baseParentType,
'baseParentId': baseParentId
}})
fixes += 1
return fixes
def getAggregation(property):
pipeline = [
*getMatchConditions(filter), {
"$group": {
"_id": "$meta.meta.{}".format(property),
"count": {
"$sum": 1
}
}
}, {
"$project": {
"_id": 0,
"key": {
"$ifNull": ["$_id", "other"]
},
"count": 1
}
}
]
return list(Item().collection.aggregate(pipeline))
def _getBaseResource(self, model, resource):
"""
Get the base resource for something pertaining to quota policies. If
the base resource has no quota policy, return (None, None).
:param model: the initial model type. Could be file, item, folder,
user, or collection.
:param resource: the initial resource document.
:returns: A pair ('model', 'resource'), where 'model' is the base model
type, either 'user' or 'collection'., and 'resource' is the
base resource document or the id of that document.
"""
if isinstance(resource, six.string_types + (ObjectId, )):
try:
resource = ModelImporter.model(model).load(id=resource,
force=True)
except ImportError:
return None, None
if model == 'file':
model = 'item'
resource = Item().load(id=resource['itemId'], force=True)
if model in ('folder', 'item'):
if ('baseParentType' not in resource
or 'baseParentId' not in resource):
resource = ModelImporter.model(model).load(id=resource['_id'],
force=True)
if ('baseParentType' not in resource
or 'baseParentId' not in resource):
return None, None
model = resource['baseParentType']
resourceId = resource['baseParentId']
resource = ModelImporter.model(model).load(id=resourceId,
force=True)
if model in ('user', 'collection') and resource:
# Ensure the base resource has a quota field so we can use the
# default quota if appropriate
if QUOTA_FIELD not in resource:
resource[QUOTA_FIELD] = {}
if not resource or QUOTA_FIELD not in resource:
return None, None
return model, resource
def testAnnotationWithGirderIdHandlerAltOrder(self, server, fsAssetstore,
admin):
file = utilities.uploadExternalFile('Easy1.png', admin, fsAssetstore)
item = Item().load(file['itemId'], user=admin)
utilities.uploadTestFile('sample_girder_id.anot',
admin,
fsAssetstore,
reference=json.dumps({
'identifier':
'IsAnAnnotationFile',
'uuid':
'12346',
'userId':
str(admin['_id']),
'itemId':
str(item['_id']),
'fileId':
str(file['_id']),
}))
assert Annotation().findOne({'itemId': item['_id']}) is None
utilities.uploadExternalFile('Easy1.png',
admin,
fsAssetstore,
reference=json.dumps({
'identifier':
'ImageRecord1',
'uuid':
'12346',
'userId':
str(admin['_id']),
'itemId':
str(item['_id']),
'fileId':
str(file['_id']),
}))
starttime = time.time()
while time.time() < starttime + 10:
if Annotation().findOne({'itemId': item['_id']}) is not None:
break
time.sleep(0.1)
assert Annotation().findOne({'itemId': item['_id']}) is not None
def testGetVersion(self, user, admin):
privateFolder = utilities.namedFolder(admin, 'Private')
Setting().set(constants.PluginSettings.LARGE_IMAGE_ANNOTATION_HISTORY, True)
item = Item().createItem('sample', admin, privateFolder)
annot = Annotation().createAnnotation(item, admin, copy.deepcopy(sampleAnnotation))
annot['annotation']['name'] = 'First Change'
annot['annotation']['elements'].extend([
{'type': 'point', 'center': [20.0, 25.0, 0]},
{'type': 'point', 'center': [10.0, 24.0, 0]},
{'type': 'point', 'center': [25.5, 23.0, 0]},
])
annot = Annotation().save(annot)
annot['annotation']['name'] = 'Second Change'
annot['annotation']['elements'].pop(2)
annot = Annotation().save(annot)
versions = list(Annotation().versionList(annot['_id'], user=admin))
with pytest.raises(AccessException):
Annotation().getVersion(annot['_id'], versions[0]['_version'], user=user)
assert len(Annotation().getVersion(
annot['_id'],
versions[0]['_version'],
user=admin)['annotation']['elements']) == 3
assert len(Annotation().getVersion(
annot['_id'],
versions[1]['_version'],
user=admin)['annotation']['elements']) == 4
assert len(Annotation().getVersion(
annot['_id'],
versions[2]['_version'],
user=admin)['annotation']['elements']) == 1
# We can get a version by its own id
assert len(Annotation().getVersion(
str(versions[1]['_id']),
versions[1]['_version'],
user=admin)['annotation']['elements']) == 4
# Asking for an invalid version gets us None
assert Annotation().getVersion(
annot['_id'],
versions[0]['_version'] + 1,
user=admin) is None
def _importTar(self, path, folder, progress, user):
if not os.path.isabs(path):
path = os.path.join(self.assetstore['root'], path)
if not os.path.isfile(path):
raise ValidationException('Error: %s is not a file.' % path)
folderCache = {}
def _resolveFolder(name):
if name in {'.', ''}: # This file is at the top level
return folder
if name not in folderCache:
tokens = name.split('/')
sub = folder
for token in tokens:
if token.strip() in {'.', ''}:
continue
sub = Folder().createFolder(sub, token, creator=user, reuseExisting=True)
folderCache[name] = sub
return folderCache[name]
with tarfile.open(path, 'r') as tar:
for entry in tar:
if entry.isreg():
dir, name = os.path.split(entry.name)
progress.update(message=entry.name)
parent = _resolveFolder(dir)
if not Folder().hasAccess(parent, user, AccessType.WRITE):
raise AccessException('Write access denied for folder: %s' % folder['_id'])
item = Item().createItem(
name=name, creator=user, folder=parent, reuseExisting=True)
file = File().createFile(
name=name, creator=user, item=item, reuseExisting=True,
assetstore=self.assetstore, size=entry.size, saveFile=False)
file['path'] = ''
file['tarPath'] = path
file['imported'] = True
file['pathInTarfile'] = entry.name
File().save(file)
def downloadFiles(assetstoreId):
ids = []
print("Downloading all files from assetstore %s" % assetstoreId)
files = getFiles(assetstoreId)
i = 0
for file in files:
for user in User().find({'_id': file["creatorId"]}):
if user['login'] == 'admin':
continue
if file['itemId'] is not None:
item = Item().load(file['itemId'], force=True)
if item['baseParentType'] != "user":
continue
if 'copied' in file:
continue
print("Downloading files for user %s" % (user))
print("File %s" % file)
fullpath = getResourcePath("file", file, user=user)
fullpath = os.path.dirname(fullpath)
if not fullpath.startswith("/user"):
continue
path = os.path.dirname(fullpath)
os.makedirs(path, exist_ok=True)
print("Downloading file %s" % (fullpath))
for data in File().download(file, headers=False)():
f = open(fullpath, "wb")
f.write(data)
f.close()
i = i + 1
ids.append(file["_id"])
else:
print("Item id is None for %s" % file['_id'])
print("Downloaded %d of %d files" % (i, len(files)))
return ids
def test_union_subdataset_bounds(server, admin, fsAssetstore):
testFile = 'tests/data/sresa1b_ncar_ccsm3-example.nc'
name = os.path.basename(testFile)
public = server.request(path='/folder',
user=admin,
method='GET',
params={
'parentId': admin['_id'],
'parentType': 'user',
'name': 'Public'
})
assertStatusOk(public)
with open(testFile, 'rb') as f:
uploadedFile = server.uploadFile(name, f.read(), admin, public.json[0])
document = Item().load(uploadedFile['itemId'], user=admin)
if document['geometa']['subDatasets'] == 1:
assert document['geometa']['bounds'] == document['geometa'][
'subDatasetInfo'][0]['bounds']
def convertImage(self, item, params):
if 'concurrent' in params:
params['_concurrency'] = params.pop('concurrent')
largeImageFileId = params.get('fileId')
if largeImageFileId is None:
files = list(Item().childFiles(item=item, limit=2))
if len(files) == 1:
largeImageFileId = str(files[0]['_id'])
if not largeImageFileId:
raise RestException('Missing "fileId" parameter.')
largeImageFile = File().load(largeImageFileId, force=True, exc=True)
user = self.getCurrentUser()
token = self.getCurrentToken()
params.pop('notify', None)
localJob = self.boolParam('localJob', params, default=True)
params.pop('localJob', None)
try:
return self.imageItemModel.convertImage(
item, largeImageFile, user, token, localJob=localJob, **params)
except TileGeneralException as e:
raise RestException(e.args[0])
def extract(path):
resources = []
if '/' in path:
resources += list(Collection().find())
resources += list(User().find())
elif '/collection' in path:
resources += list(Collection().find())
elif '/user' in path:
resources += list(User().find())
else:
resources += [lookUpPath(p)['document'] for p in path]
for resource in resources:
items = list(Item().find({'baseParentId': resource['_id']}))
for item in items:
files = list(File().find({'itemId': item['_id']}))
for file in files:
create_geometa(item, file)
def testGetLargeImagePath(server, admin, fsAssetstore):
file = utilities.uploadExternalFile('data/sample_image.ptif.sha512', admin, fsAssetstore)
itemId = str(file['itemId'])
item = Item().load(itemId, user=admin)
ts = ImageItem().tileSource(item)
with mock.patch.object(File(), 'getGirderMountFilePath', return_value='mockmount'):
path = ts._getLargeImagePath()
abspath = os.path.abspath(path)
assert path != file['path']
assert path.endswith(file['path'])
ts._mayHaveAdjacentFiles = True
path = ts._getLargeImagePath()
assert path == 'mockmount'
origFile = file
file['imported'] = True
file['path'] = abspath
file = File().save(file)
path = ts._getLargeImagePath()
assert path == abspath
file = File().save(origFile)
def testGridHeatmapAnnotation(self, db, admin, fsAssetstore):
item = Item().createItem('sample', admin, utilities.namedFolder(admin, 'Public'))
annotation = {
'name': 'testAnnotation',
'elements': [{
'type': 'griddata',
'interpretation': 'heatmap',
'origin': [30, 40, 50],
'dx': 3,
'dy': 4,
'gridWidth': 128,
'values': [random.random() for _ in range(10240)]
}]
}
annot = Annotation().createAnnotation(item, admin, annotation)
assert Annotation().load(annot['_id'], user=admin) is not None
Setting().set(constants.PluginSettings.LARGE_IMAGE_ANNOTATION_HISTORY, False)
result = Annotation().remove(annot)
Setting().set(constants.PluginSettings.LARGE_IMAGE_ANNOTATION_HISTORY, True)
assert result.deleted_count == 1
assert Annotation().load(annot['_id'], user=admin) is None
def feature_vs_material(self, filter, params):
return list(Item().collection.aggregate([
*getMatchConditions(filter), {
"$group": {
"_id": {
"feature": "$meta.meta.feature",
"material": "$meta.meta.material"
},
"count": {
"$sum": 1
}
}
}, {
"$project": {
"_id": 0,
"feature": "$_id.feature",
"material": "$_id.material",
"count": 1
}
}
]))
def _uploadFile(self, name, item):
"""
Upload a random file to an item.
:param name: name of the file.
:param item: item to upload the file to.
:returns: file: the created file object
path: the path to the file within the parent hierarchy.
contents: the contents that were generated for the file.
"""
contents = os.urandom(1024)
file = self.uploadFile(name,
contents,
user=self.admin,
parent=item,
parentType='item')
parents = Item().parentsToRoot(item, user=self.admin)
path = os.path.join(*([
part['object'].get('name', part['object'].get('login', ''))
for part in parents
] + [item['name'], name]))
return file, path, contents
def _getAll(self):
datasetItems = list(Item().find({
'$and': [{
'name': {
'$regex': '.NTF$'
}
}, {
'$or': [{
'geometa.driver': {
'$in': [
'GeoJSON', 'GeoTIFF', 'OBJ',
'National Imagery Transmission Format'
]
}
}, {
'geometa.subDatasets.driver':
'National Imagery Transmission Format'
}]
}]
}))
return self.filterInputNTF(datasetItems)
def _uploadFile(self, path, name=None, private=False):
"""
Upload the specified path to the admin user's public or private folder
and return the resulting item.
:param path: path to upload.
:param name: optional name for the file.
:param private: True to upload to the private folder, False for public.
'user' for the user's private folder.
:returns: file: the created file.
"""
if not name:
name = os.path.basename(path)
with open(path, 'rb') as file:
data = file.read()
if private == 'user':
folderId = self.userPrivateFolder['_id']
elif private:
folderId = self.privateFolder['_id']
else:
folderId = self.publicFolder['_id']
resp = self.request(
path='/file', method='POST', user=self.admin, params={
'parentType': 'folder',
'parentId': folderId,
'name': name,
'size': len(data)
})
self.assertStatusOk(resp)
uploadId = resp.json['_id']
fields = [('offset', 0), ('uploadId', uploadId)]
files = [('chunk', name, data)]
resp = self.multipartRequest(
path='/file/chunk', fields=fields, files=files, user=self.admin)
self.assertStatusOk(resp)
self.assertIn('itemId', resp.json)
file = File().load(resp.json['_id'], user=self.admin, exc=True)
item = Item().load(file['itemId'], user=self.admin, exc=True)
return file, item
def _onUpload(event):
"""
Thumbnail creation can be requested on file upload by passing a reference field
that is a JSON object of the following form:
{
"thumbnail": {
"width": 123,
"height": 123,
"crop": True
}
}
At least one of ``width`` or ``height`` must be passed. The ``crop`` parameter is optional.
"""
file = event.info['file']
if 'itemId' not in file:
return
try:
ref = json.loads(event.info.get('reference'))
except (ValueError, TypeError):
return
if not isinstance(ref, dict) or not isinstance(ref.get('thumbnail'), dict):
return
width = max(0, ref['thumbnail'].get('width', 0))
height = max(0, ref['thumbnail'].get('height', 0))
if not width and not height:
return
if not isinstance(width, int) or not isinstance(height, int):
return
item = Item().load(file['itemId'], force=True)
crop = bool(ref['thumbnail'].get('crop', True))
utils.scheduleThumbnailJob(
file=file, attachToType='item', attachToId=item['_id'], user=event.info['currentUser'],
width=width, height=height, crop=crop)
def create_new_file(p_folder, name, user, assetstore):
"""
create a new file
:param p_folder: parent folder
:param name: name of the file you want to create
:param user: user trying to create this file
:param assetstore: assetstore where this file is going to be created
:return: file doc
"""
item = Item().createItem(name,
creator=user,
folder=p_folder,
description='label file',
reuseExisting=False)
file = File().createFile(size=0,
item=item,
name=name,
creator=user,
assetstore=assetstore,
mimeType="application/json")
return file
def createTiles(self, item, params):
largeImageFileId = params.get('fileId')
if largeImageFileId is None:
files = list(Item().childFiles(item=item, limit=2))
if len(files) == 1:
largeImageFileId = str(files[0]['_id'])
if not largeImageFileId:
raise RestException('Missing "fileId" parameter.')
largeImageFile = File().load(largeImageFileId, force=True, exc=True)
user = self.getCurrentUser()
token = self.getCurrentToken()
try:
return self.imageItemModel.createImageItem(item,
largeImageFile,
user,
token,
notify=self.boolParam(
'notify',
params,
default=True))
except TileGeneralException as e:
raise RestException(e.args[0])
def setUp(self):
base.TestCase.setUp(self)
info = {
'email': '*****@*****.**',
'login': '******',
'firstName': 'Admin',
'lastName': 'Admin',
'password': '******',
'admin': True
}
self.admin = User().createUser(**info)
self.publicFolder = six.next(Folder().childFolders(
self.admin, parentType='user', force=True, filters={'name': 'Public'}))
self.item = Item().createItem('test', self.admin, self.publicFolder)
path = os.path.join(
ROOT_DIR, 'tests', 'cases', 'mount_test_files', 'file1a.txt')
file = File().createFile(
name='file1a.txt', creator=self.admin, item=self.item,
assetstore=self.assetstore, size=os.path.getsize(path))
file['imported'] = True
file['path'] = path
self.file = File().save(file)
def find_file(p_folder, name, user, assetstore, create=False):
"""
Find file by name. If not found create the file.
:param p_folder: parent folder
:param name: name of the file
:param user: user trying to access this file
:param assetstore: assetstore where this file exists
:param create: boolean, whether or not to create the file if not found
:return: file doc or None
"""
item = list(Item().find({'folderId': p_folder['_id'], 'name': name}).limit(1))
if not item:
# check if you are allowed to create, else return nothing
if create:
file = create_new_file(p_folder, name, user, assetstore)
else:
return None
else:
item = item[0]
file = list(File().find({'itemId': item['_id']}).limit(1))[0]
return file
def _importDataAsItem(self,
name,
user,
folder,
path,
files,
reuseExisting=True,
params=None):
params = params or {}
item = Item().createItem(name=name,
creator=user,
folder=folder,
reuseExisting=reuseExisting)
events.trigger('filesystem_assetstore_imported', {
'id': item['_id'],
'type': 'item',
'importPath': path
})
for fname in files:
fpath = os.path.join(path, fname)
if self.shouldImportFile(fpath, params):
self.importFile(item, fpath, user, name=fname)
def childItems(self,
folder,
limit=0,
offset=0,
sort=None,
filters=None,
includeVirtual=False,
**kwargs):
if not includeVirtual or not folder.get(
'isVirtual') or 'virtualItemsQuery' not in folder:
return Folder._childItemsBeforeHUI(self,
folder,
limit=limit,
offset=offset,
sort=sort,
filters=filters,
**kwargs)
q = json_util.loads(folder['virtualItemsQuery'])
if 'virtualItemsSort' in folder and sort is None:
sort = json.loads(folder['virtualItemsSort'])
q.update(filters or {})
return Item().find(q, limit=limit, offset=offset, sort=sort, **kwargs)
def testRevertVersion(self, admin):
privateFolder = utilities.namedFolder(admin, 'Private')
Setting().set(constants.PluginSettings.LARGE_IMAGE_ANNOTATION_HISTORY,
True)
item = Item().createItem('sample', admin, privateFolder)
annot = Annotation().createAnnotation(item, admin,
copy.deepcopy(sampleAnnotation))
annot['annotation']['name'] = 'First Change'
annot['annotation']['elements'].extend([
{
'type': 'point',
'center': [20.0, 25.0, 0]
},
{
'type': 'point',
'center': [10.0, 24.0, 0]
},
{
'type': 'point',
'center': [25.5, 23.0, 0]
},
])
annot = Annotation().save(annot)
annot['annotation']['name'] = 'Second Change'
annot['annotation']['elements'].pop(2)
annot = Annotation().save(annot)
versions = list(Annotation().versionList(annot['_id'], user=admin))
assert Annotation().revertVersion(
annot['_id'], versions[0]['_version'] + 1, user=admin) is None
assert len(Annotation().revertVersion(
annot['_id'], force=True)['annotation']['elements']) == 4
assert len(Annotation().revertVersion(
annot['_id'], force=True)['annotation']['elements']) == 3
assert len(Annotation().revertVersion(
annot['_id'], versions[2]['_version'],
force=True)['annotation']['elements']) == 1
Annotation().remove(annot)
assert len(Annotation().revertVersion(
annot['_id'], user=admin)['annotation']['elements']) == 1
def create_file(self, assetstore, params):
params = getBodyJson()
self.requireParams(('name', 'itemId', 'size', 'path'), params)
name = params['name']
item_id = params['itemId']
size = int(params['size'])
path = params['path']
user = self.getCurrentUser()
mime_type = params.get('mimeType')
item = Item().load(id=item_id, user=user,
level=AccessType.WRITE, exc=True)
file = File().createFile(
name=name, creator=user, item=item, reuseExisting=True,
assetstore=assetstore, mimeType=mime_type, size=size)
file['path'] = path
file['imported'] = True
File().save(file)
return File().filter(file)
class Item(Resource):
def __init__(self):
super(Item, self).__init__()
self.resourceName = 'item'
self._model = ItemModel()
self.route('DELETE', (':id',), self.deleteItem)
self.route('GET', (), self.find)
self.route('GET', (':id',), self.getItem)
self.route('GET', (':id', 'files'), self.getFiles)
self.route('GET', (':id', 'download'), self.download)
self.route('GET', (':id', 'rootpath'), self.rootpath)
self.route('POST', (), self.createItem)
self.route('PUT', (':id',), self.updateItem)
self.route('POST', (':id', 'copy'), self.copyItem)
self.route('PUT', (':id', 'metadata'), self.setMetadata)
self.route('DELETE', (':id', 'metadata'), self.deleteMetadata)
@access.public(scope=TokenScope.DATA_READ)
@filtermodel(model=ItemModel)
@autoDescribeRoute(
Description('List or search for items.')
.notes('You must pass either a "itemId" or "text" field '
'to specify how you are searching for items. '
'If you omit one of these parameters the request will fail and respond : '
'"Invalid search mode."')
.responseClass('Item', array=True)
.param('folderId', 'Pass this to list all items in a folder.',
required=False)
.param('text', 'Pass this to perform a full text search for items.',
required=False)
.param('name', 'Pass to lookup an item by exact name match. Must '
'pass folderId as well when using this.', required=False)
.pagingParams(defaultSort='lowerName')
.errorResponse()
.errorResponse('Read access was denied on the parent folder.', 403)
)
def find(self, folderId, text, name, limit, offset, sort):
"""
Get a list of items with given search parameters. Currently accepted
search modes are:
1. Searching by folderId, with optional additional filtering by the name
field (exact match) or using full text search within a single parent
folder. Pass a "name" parameter or "text" parameter to invoke these
additional filters.
2. Searching with full text search across all items in the system.
Simply pass a "text" parameter for this mode.
"""
user = self.getCurrentUser()
if folderId:
folder = Folder().load(
id=folderId, user=user, level=AccessType.READ, exc=True)
filters = {}
if text:
filters['$text'] = {
'$search': text
}
if name:
filters['name'] = name
return Folder().childItems(
folder=folder, limit=limit, offset=offset, sort=sort, filters=filters)
elif text is not None:
return self._model.textSearch(
text, user=user, limit=limit, offset=offset, sort=sort)
else:
raise RestException('Invalid search mode.')
@access.public(scope=TokenScope.DATA_READ)
@filtermodel(model=ItemModel)
@autoDescribeRoute(
Description('Get an item by ID.')
.responseClass('Item')
.modelParam('id', model=ItemModel, level=AccessType.READ)
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
def getItem(self, item):
return item
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(model=ItemModel)
@autoDescribeRoute(
Description('Create a new item.')
.responseClass('Item')
.modelParam('folderId', 'The ID of the parent folder.', model=Folder,
level=AccessType.WRITE, paramType='query')
.param('name', 'Name for the item.', strip=True)
.param('description', 'Description for the item.', required=False,
default='', strip=True)
.param('reuseExisting', 'Return existing item (by name) if it exists.',
required=False, dataType='boolean', default=False)
.jsonParam('metadata', 'A JSON object containing the metadata keys to add',
paramType='form', requireObject=True, required=False)
.errorResponse()
.errorResponse('Write access was denied on the parent folder.', 403)
)
def createItem(self, folder, name, description, reuseExisting, metadata):
newItem = self._model.createItem(
folder=folder, name=name, creator=self.getCurrentUser(), description=description,
reuseExisting=reuseExisting)
if metadata:
newItem = self._model.setMetadata(newItem, metadata)
return newItem
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(model=ItemModel)
@autoDescribeRoute(
Description('Edit an item or move it to another folder.')
.responseClass('Item')
.modelParam('id', model=ItemModel, level=AccessType.WRITE)
.param('name', 'Name for the item.', required=False, strip=True)
.param('description', 'Description for the item.', required=False)
.modelParam('folderId', 'Pass this to move the item to a new folder.', model=Folder,
required=False, paramType='query', level=AccessType.WRITE)
.jsonParam('metadata', 'A JSON object containing the metadata keys to add',
paramType='form', requireObject=True, required=False)
.errorResponse('ID was invalid.')
.errorResponse('Write access was denied for the item or folder.', 403)
)
def updateItem(self, item, name, description, folder, metadata):
if name is not None:
item['name'] = name
if description is not None:
item['description'] = description
self._model.updateItem(item)
if folder and folder['_id'] != item['folderId']:
self._model.move(item, folder)
if metadata:
item = self._model.setMetadata(item, metadata)
return item
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(model=ItemModel)
@autoDescribeRoute(
Description('Set metadata fields on an item.')
.responseClass('Item')
.notes('Set metadata fields to null in order to delete them.')
.modelParam('id', model=ItemModel, level=AccessType.WRITE)
.jsonParam('metadata', 'A JSON object containing the metadata keys to add',
paramType='body', requireObject=True)
.param('allowNull', 'Whether "null" is allowed as a metadata value.', required=False,
dataType='boolean', default=False)
.errorResponse(('ID was invalid.',
'Invalid JSON passed in request body.',
'Metadata key name was invalid.'))
.errorResponse('Write access was denied for the item.', 403)
)
def setMetadata(self, item, metadata, allowNull):
return self._model.setMetadata(item, metadata, allowNull=allowNull)
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(ItemModel)
@autoDescribeRoute(
Description('Delete metadata fields on an item.')
.responseClass('Item')
.modelParam('id', model=ItemModel, level=AccessType.WRITE)
.jsonParam(
'fields', 'A JSON list containing the metadata fields to delete',
paramType='body', schema={
'type': 'array',
'items': {
'type': 'string'
}
}
)
.errorResponse(('ID was invalid.',
'Invalid JSON passed in request body.',
'Metadata key name was invalid.'))
.errorResponse('Write access was denied for the item.', 403)
)
def deleteMetadata(self, item, fields):
return self._model.deleteMetadata(item, fields)
def _downloadMultifileItem(self, item, user):
setResponseHeader('Content-Type', 'application/zip')
setContentDisposition(item['name'] + '.zip')
def stream():
zip = ziputil.ZipGenerator(item['name'])
for (path, file) in self._model.fileList(item, subpath=False):
for data in zip.addFile(file, path):
yield data
yield zip.footer()
return stream
@access.public(scope=TokenScope.DATA_READ)
@filtermodel(model=File)
@autoDescribeRoute(
Description('Get the files within an item.')
.responseClass('File', array=True)
.modelParam('id', model=ItemModel, level=AccessType.READ)
.pagingParams(defaultSort='name')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
def getFiles(self, item, limit, offset, sort):
return self._model.childFiles(item=item, limit=limit, offset=offset, sort=sort)
@access.cookie
@access.public(scope=TokenScope.DATA_READ)
@autoDescribeRoute(
Description('Download the contents of an item.')
.modelParam('id', model=ItemModel, level=AccessType.READ)
.param('offset', 'Byte offset into the file.', dataType='int',
required=False, default=0)
.param('format', 'If unspecified, items with one file are downloaded '
'as that file, and other items are downloaded as a zip '
'archive. If \'zip\', a zip archive is always sent.',
required=False)
.param('contentDisposition', 'Specify the Content-Disposition response '
'header disposition-type value, only applied for single file '
'items.', required=False, enum=['inline', 'attachment'],
default='attachment')
.param('extraParameters', 'Arbitrary data to send along with the '
'download request, only applied for single file '
'items.', required=False)
# single file items could produce other types, too.
.produces(['application/zip', 'application/octet-stream'])
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
def download(self, item, offset, format, contentDisposition, extraParameters):
user = self.getCurrentUser()
files = list(self._model.childFiles(item=item, limit=2))
if format not in (None, '', 'zip'):
raise RestException('Unsupported format: %s.' % format)
if len(files) == 1 and format != 'zip':
if contentDisposition not in {None, 'inline', 'attachment'}:
raise RestException('Unallowed contentDisposition type "%s".' % contentDisposition)
return File().download(
files[0], offset, contentDisposition=contentDisposition,
extraParameters=extraParameters)
else:
return self._downloadMultifileItem(item, user)
@access.user(scope=TokenScope.DATA_WRITE)
@autoDescribeRoute(
Description('Delete an item by ID.')
.modelParam('id', model=ItemModel, level=AccessType.WRITE)
.errorResponse('ID was invalid.')
.errorResponse('Write access was denied for the item.', 403)
)
def deleteItem(self, item):
self._model.remove(item)
return {'message': 'Deleted item %s.' % item['name']}
@access.public(scope=TokenScope.DATA_READ)
@autoDescribeRoute(
Description('Get the path to the root of the item\'s hierarchy.')
.modelParam('id', model=ItemModel, level=AccessType.READ)
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the item.', 403)
)
def rootpath(self, item):
return self._model.parentsToRoot(item, self.getCurrentUser())
@access.user(scope=TokenScope.DATA_WRITE)
@filtermodel(model=ItemModel)
@autoDescribeRoute(
Description('Copy an item.')
.notes('If no folderId parameter is specified, creates a copy of the item in '
'its current containing folder.')
.responseClass('Item')
.modelParam('id', 'The ID of the original item.', model=ItemModel, level=AccessType.READ)
.modelParam('folderId', 'The ID of the parent folder.', required=False, model=Folder,
level=AccessType.WRITE)
.param('name', 'Name for the new item.', required=False, strip=True)
.param('description', 'Description for the new item.', required=False, strip=True)
.errorResponse(('A parameter was invalid.',
'ID was invalid.'))
.errorResponse('Read access was denied on the original item.\n\n'
'Write access was denied on the parent folder.', 403)
)
def copyItem(self, item, folder, name, description):
user = self.getCurrentUser()
if folder is None:
folder = Folder().load(
id=item['folderId'], user=user, level=AccessType.WRITE, exc=True)
return self._model.copyItem(
item, creator=user, name=name, folder=folder, description=description)
def testAccessFlags(self):
resp = self.request('/system/access_flag')
self.assertStatusOk(resp)
self.assertEqual(resp.json, {})
registerAccessFlag('my_key', name='hello', description='a custom flag')
resp = self.request('/system/access_flag')
self.assertStatusOk(resp)
self.assertEqual(resp.json, {
'my_key': {
'name': 'hello',
'description': 'a custom flag',
'admin': False
}
})
self.users[1] = User().load(self.users[1]['_id'], force=True)
user = self.users[1]
# Manage custom access flags on an access controlled resource
self.assertFalse(User().hasAccessFlags(user, user, flags=['my_key']))
# Admin should always have permission
self.assertTrue(User().hasAccessFlags(user, self.users[0], flags=['my_key']))
# Test the requireAccessFlags method
with self.assertRaises(AccessException):
User().requireAccessFlags(user, user=user, flags='my_key')
User().requireAccessFlags(user, user=self.users[0], flags='my_key')
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], [])
# Test loadmodel requiredFlags argument via REST endpoint
resp = self.request(
'/test_endpoints/loadmodel_with_flags/%s' % user['_id'], user=self.users[1])
self.assertStatus(resp, 403)
user = User().setAccessList(self.users[0], access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['my_key', 'not a registered flag']
}],
'groups': [{
'id': self.group['_id'],
'level': AccessType.ADMIN,
'flags': ['my_key']
}]
}, save=True)
resp = self.request(
'/test_endpoints/loadmodel_with_flags/%s' % user['_id'], user=self.users[1])
self.assertStatusOk(resp)
self.assertEqual(resp.json, 'success')
# Only registered flags should be stored
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], ['my_key'])
self.assertTrue(User().hasAccessFlags(user, user, flags=['my_key']))
# Create an admin-only access flag
registerAccessFlag('admin_flag', name='admin flag', admin=True)
# Non-admin shouldn't be able to set it
user = User().setAccessList(self.users[0], access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}],
'groups': []
}, save=True, user=self.users[1])
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], [])
# Admin user should be able to set it
user = User().setAccessList(self.users[1], access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}],
'groups': [{
'id': self.group['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}]
}, save=True, user=self.users[0])
acl = User().getFullAccessList(user)
self.assertEqual(acl['users'][0]['flags'], ['admin_flag'])
# An already-enabled admin-only flag should stay enabled for non-admin user
user = User().setAccessList(self.users[1], access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['my_key', 'admin_flag']
}],
'groups': [{
'id': self.group['_id'],
'level': AccessType.ADMIN,
'flags': ['admin_flag']
}]
}, save=True, user=self.users[1])
acl = User().getFullAccessList(user)
self.assertEqual(set(acl['users'][0]['flags']), {'my_key', 'admin_flag'})
self.assertEqual(acl['groups'][0]['flags'], ['admin_flag'])
# Test setting public flags on a collection and folder
collectionModel = Collection()
folderModel = Folder()
itemModel = Item()
collection = collectionModel.createCollection('coll', creator=self.users[0], public=True)
folder = folderModel.createFolder(
collection, 'folder', parentType='collection', creator=self.users[0])
# Add an item to the folder so we can test AclMixin flag behavior
item = itemModel.createItem(folder=folder, name='test', creator=self.users[0])
folder = folderModel.setUserAccess(
folder, self.users[1], level=AccessType.ADMIN, save=True, currentUser=self.users[0])
with self.assertRaises(AccessException):
collectionModel.requireAccessFlags(collection, user=None, flags='my_key')
# Test AclMixin flag behavior
with self.assertRaises(AccessException):
itemModel.requireAccessFlags(item, user=None, flags='my_key')
self.assertFalse(itemModel.hasAccessFlags(item, user=None, flags='my_key'))
collection = collectionModel.setAccessList(
collection, access=collection['access'], save=True, recurse=True, user=self.users[0],
publicFlags=['my_key'])
collectionModel.requireAccessFlags(collection, user=None, flags='my_key')
# Make sure recursive setting of public flags worked
folder = folderModel.load(folder['_id'], force=True)
self.assertEqual(folder['publicFlags'], ['my_key'])
itemModel.requireAccessFlags(item, user=None, flags='my_key')
# Non-admin shouldn't be able to set admin-only public flags
folder = folderModel.setPublicFlags(
folder, flags=['admin_flag'], user=self.users[1], save=True)
self.assertEqual(folder['publicFlags'], [])
# Admin users should be able to set admin-only public flags
folder = folderModel.setPublicFlags(
folder, flags=['admin_flag'], user=self.users[0], save=True, append=True)
self.assertEqual(folder['publicFlags'], ['admin_flag'])
# Non-admin users can set admin-only public flags if they are already enabled
folder = folderModel.setPublicFlags(
folder, flags=['admin_flag', 'my_key'], user=self.users[1], save=True)
self.assertEqual(set(folder['publicFlags']), {'admin_flag', 'my_key'})
# Test "force" options
folder = folderModel.setPublicFlags(folder, flags='admin_flag', force=True, save=True)
self.assertEqual(folder['publicFlags'], ['admin_flag'])
folder = folderModel.setAccessList(folder, access={
'users': [{
'id': self.users[1]['_id'],
'level': AccessType.ADMIN,
'flags': ['my_key', 'admin_flag']
}],
'groups': []
}, save=True, force=True)
folderModel.requireAccessFlags(folder, user=self.users[1], flags='my_key')
folder = folderModel.setUserAccess(
folder, self.users[1], level=AccessType.READ, save=True, force=True, flags=[])
self.assertFalse(folderModel.hasAccessFlags(folder, self.users[1], flags='my_key'))
folder = folderModel.setGroupAccess(
folder, self.group, level=AccessType.READ, save=True, force=True, flags='my_key')
folderModel.requireAccessFlags(folder, user=self.users[1], flags='my_key')
# Testing with flags=None should give sensible behavior
folderModel.requireAccessFlags(folder, user=None, flags=None)
# Test filtering results by access flags (both ACModel and AclMixin)
for model, doc in ((folderModel, folder), (itemModel, item)):
cursor = model.find({})
self.assertGreater(len(list(cursor)), 0)
cursor = model.find({})
filtered = list(model.filterResultsByPermission(
cursor, user=None, level=AccessType.READ, flags='my_key'))
self.assertEqual(len(filtered), 0)
cursor = model.find({})
filtered = list(model.filterResultsByPermission(
cursor, user=self.users[1], level=AccessType.READ, flags=('my_key', 'admin_flag')))
self.assertEqual(len(filtered), 1)
self.assertEqual(filtered[0]['_id'], doc['_id'])
def validate(self, doc):
# TODO: implement
# raise ValidationException
return Item.validate(self, doc)
def findOne(self, query=None, **kwargs):
annotation_query = self._find_query_filter(query)
return Item.findOne(self, annotation_query, **kwargs)
class HdfsAssetstoreResource(Resource):
def __init__(self):
super(HdfsAssetstoreResource, self).__init__()
self.resourceName = 'hdfs_assetstore'
self.route('PUT', (':id', 'import'), self.importData)
self.folderModel = Folder() # Save to avoid many lookups
self.itemModel = Item()
self.fileModel = File()
def _importFile(self, parent, name, user, assetstore, node):
item = self.itemModel.findOne({
'folderId': parent['_id'],
'name': name
})
if item is None:
item = self.itemModel.createItem(
name=name, creator=user, folder=parent)
file = self.fileModel.findOne({
'name': name,
'itemId': item['_id']
})
if file is None:
file = self.fileModel.createFile(
creator=user, item=item, name=name, size=node['length'],
assetstore=assetstore, mimeType=None, saveFile=False)
file['hdfs'] = {
'imported': True,
'path': node['path']
}
self.fileModel.save(file)
def _importData(self, parentType, parent, assetstore, client, path, ctx,
user):
for node in client.ls([path]):
ctx.update(message='Importing ' + node['path'])
name = posixpath.basename(node['path'])
if node['file_type'] == 'd':
folder = self.folderModel.findOne({
'parentId': parent['_id'],
'name': name,
'parentCollection': parentType
})
if folder is None:
folder = self.folderModel.createFolder(
parent, name, parentType=parentType, creator=user)
self._importData('folder', folder, assetstore, client,
node['path'], ctx, user)
elif node['file_type'] == 'f' and parentType == 'folder':
self._importFile(parent, name, user, assetstore, node)
@access.admin
@loadmodel(model='assetstore')
@describeRoute(
Description('Import a data hierarchy from an HDFS instance.')
.notes('Only site administrators may use this endpoint.')
.param('id', 'The ID of the assetstore representing the HDFS instance.',
paramType='path')
.param('parentId', 'The ID of the parent object in the Girder data '
'hierarchy under which to import the files.')
.param('parentType', 'The type of the parent object to import into.',
enum=('folder', 'user', 'collection'), required=False)
.param('path', 'Root of the directory structure (relative to the root '
'of the HDFS) to import.')
.param('progress', 'Whether to record progress on this operation ('
'default=False)', required=False, dataType='boolean')
.errorResponse()
.errorResponse('You are not an administrator.', 403)
)
def importData(self, assetstore, params):
self.requireParams(('parentId', 'path'), params)
user = self.getCurrentUser()
parentType = params.get('parentType', 'folder')
if parentType not in ('user', 'collection', 'folder'):
raise RestException('Invalid parentType.')
parent = self.model(parentType).load(params['parentId'], force=True, exc=True)
progress = self.boolParam('progress', params, default=False)
client = HdfsClient(
host=assetstore['hdfs']['host'], port=assetstore['hdfs']['port'], use_trash=False)
path = params['path']
with ProgressContext(progress, user=user, title='Importing data from HDFS') as ctx:
try:
self._importData(parentType, parent, assetstore, client, path, ctx, user)
except FileNotFoundException:
raise RestException('File not found: %s.' % path)
class LabelResource(Resource):
def __init__(self):
super().__init__()
self.resourceName = 'label'
self.coll_m = Collection()
self.file_m = File()
self.folder_m = Folder()
self.item_m = Item()
self.upload_m = Upload()
self.asset_m = Assetstore()
self.setupRoutes()
def setupRoutes(self):
self.route('GET', (), handler=self.getLabelList)
self.route('GET', (':label_id',), self.getLabel)
self.route('GET', ('meta',), self.getLabelMeta)
self.route('GET', ('create',), self.createLabelFile)
self.route('GET', ('by_name',), self.getLabelByName)
self.route('POST', (), self.postLabel)
def createNewFile(self, folder, file_name):
item = self.item_m.createItem(file_name,
creator=self.getCurrentUser(),
folder=folder,
description='label file',
reuseExisting=False)
file = self.file_m.createFile(size=0,
item=item,
name=file_name,
creator=self.getCurrentUser(),
assetstore=self.asset_m.getCurrent(),
mimeType="application/json")
return file
def copy(self, srcFile, destFile):
upload = self.upload_m.createUploadToFile(destFile, self.getCurrentUser(), srcFile['size'])
self.upload_m.handleChunk(upload=upload,
chunk=RequestBodyStream(self.file_m.open(srcFile), size=destFile['size']),
user=self.getCurrentUser())
return upload
@access.public
@autoDescribeRoute(
Description('Get label list'))
@rest.rawResponse
def getLabelList(self):
printOk('getLabelsList() was called!')
try:
collection = list(self.coll_m.list(user=self.getCurrentUser(), offset=0, limit=1))[0]
files = self.coll_m.fileList(collection, user=self.getCurrentUser(), data=False,
includeMetadata=True, mimeFilter=['application/json'])
files = list(files)
cherrypy.response.headers["Content-Type"] = "application/json"
return dumps(files)
except:
printFail(traceback.print_exc)
@staticmethod
def getOwnerId(folder):
aclList = Folder().getFullAccessList(folder)
for acl in aclList['users']:
if acl['level'] == AccessType.ADMIN:
return str(acl['id'])
return None
def getConfigFolder(self, label_folder_id):
label_folder = Folder().load(label_folder_id,
user=self.getCurrentUser(),
level=AccessType.READ)
ownerId = self.getOwnerId(label_folder)
config_folder = self.folder_m.load(label_folder['meta'][ownerId], level=AccessType.READ,
user=self.getCurrentUser())
return config_folder
def findConfig(self, folder_id):
folder = self.getConfigFolder(folder_id)
printOk2("Config folder {}".format(folder))
files = self.folder_m.fileList(folder, self.getCurrentUser(), data=False)
for file_path, file in files:
printOk(file)
if file['name'] == "config.json":
return file
def __findFile(self, folder, file_name):
item = list(self.item_m.find({'folderId': folder['_id'], 'name': file_name}).limit(1))
if not item:
return None
item = item[0]
file = list(self.file_m.find({'itemId': item['_id']}).limit(1))
if not file:
return None
return file[0]
@access.public
@autoDescribeRoute(
Description('Create a new label file if it doesnt exist')
.param('file_name', 'label file name').param('folder_id', 'the parent folder id'))
@rest.rawResponse
def createLabelFile(self, file_name, folder_id):
try:
folder = self.folder_m.load(folder_id, user=self.getCurrentUser(), level=AccessType.WRITE)
file = self.__findFile(folder, file_name)
if not file:
file = self.createNewFile(folder, file_name)
config_file = self.findConfig(folder_id)
if not config_file:
printFail("No config file found")
return errorMessage("No config file found")
else:
res = self.copy(config_file, file)
return dumps({
"label_id": res['fileId']
})
return dumps({
"label_id": file['_id']
})
except:
printFail(traceback.print_exc)
cherrypy.response.status = 500
@access.public
@autoDescribeRoute(
Description('Get labels by file_name')
.param('file_name', 'label file name').param('folder_id', 'the parent folder id'))
@rest.rawResponse
def getLabelByName(self, file_name, folder_id):
try:
folder = self.folder_m.load(folder_id, user=self.getCurrentUser(), level=AccessType.READ)
file = self.__findFile(folder, file_name)
cherrypy.response.headers["Content-Type"] = "application/json"
if file:
return self.file_m.download(file)
else:
return dumps({})
except:
printFail(traceback.print_exc)
cherrypy.response.status = 500
@access.public
@autoDescribeRoute(
Description('Get label by id')
.param('label_id', 'label file id'))
@rest.rawResponse
def getLabel(self, label_id):
try:
file = self.file_m.load(label_id, level=AccessType.READ, user=self.getCurrentUser())
printOk2(file)
cherrypy.response.headers["Content-Type"] = "application/json"
return self.file_m.download(file)
except:
# Unknown slug
printFail(traceback.print_exc)
cherrypy.response.status = 404
@access.public
@autoDescribeRoute(
Description('Get label by id')
.param('label_id', 'label file id'))
def getLabelMeta(self, label_id):
try:
file = self.file_m.load(label_id, level=AccessType.READ, user=self.getCurrentUser())
cherrypy.response.headers["Content-Type"] = "application/json"
return dumps(file)
except:
# Unknown slug
printFail(traceback.print_exc)
cherrypy.response.status = 404
@access.public
@autoDescribeRoute(
Description('Post label by id')
.param('label_id', 'label file id'))
@rest.rawResponse
def postLabel(self, label_id, params):
try:
file = self.file_m.load(label_id, level=AccessType.WRITE, user=self.getCurrentUser())
cherrypy.response.headers["Content-Type"] = "application/json"
params['labels'] = json.loads(params['labels'])
data = json.dumps(params, indent=2, sort_keys=True)
upload = writeData(self.getCurrentUser(), file, data)
printOk2(file)
printOk(upload)
return dumps(upload)
except:
# Unknown slug
printFail(traceback.print_exc)
cherrypy.response.status = 404
@access.public
@autoDescribeRoute(
Description('Post label by id')
.param('label_id', 'label file id'))
@rest.rawResponse
def strokeToOutline(self, strokes):
pass
