def __init__(self, assetstore):
"""
:param assetstore: The assetstore to act on.
"""
self.assetstore = assetstore
try:
self.chunkColl = getDbConnection(
assetstore.get('mongohost', None),
assetstore.get('replicaset', None))[assetstore['db']]['chunk']
except pymongo.errors.ConnectionFailure:
logger.error('Failed to connect to GridFS assetstore %s',
assetstore['db'])
self.chunkColl = 'Failed to connect'
self.unavailable = True
return
except pymongo.errors.ConfigurationError:
logger.exception('Failed to configure GridFS assetstore %s',
assetstore['db'])
self.chunkColl = 'Failed to configure'
self.unavailable = True
return
self.chunkColl.ensure_index([
('uuid', pymongo.ASCENDING),
('n', pymongo.ASCENDING)
], unique=True)
def validateInfo(doc):
"""
Makes sure the root field is a valid absolute path and is writeable.
"""
if 'prefix' not in doc:
doc['prefix'] = ''
while len(doc['prefix']) and doc['prefix'][0] == '/':
doc['prefix'] = doc['prefix'][1:]
while len(doc['prefix']) and doc['prefix'][-1] == '/':
doc['prefix'] = doc['prefix'][:-1]
if not doc.get('bucket'):
raise ValidationException('Bucket must not be empty.', 'bucket')
if not doc.get('secret'):
raise ValidationException('Secret key must not be empty.',
'secretKey')
if not doc.get('accessKeyId'):
raise ValidationException('Access key ID must not be empty.',
'accessKeyId')
# Make sure we can write into the given bucket using boto
try:
conn = boto.connect_s3(aws_access_key_id=doc['accessKeyId'],
aws_secret_access_key=doc['secret'])
bucket = conn.lookup(bucket_name=doc['bucket'], validate=False)
testKey = boto.s3.key.Key(bucket=bucket,
name=os.path.join(doc['prefix'], 'test'))
testKey.set_contents_from_string('')
except:
logger.exception('S3 assetstore validation exception')
raise ValidationException(
'Unable to write into bucket "{}".'.format(doc['bucket']),
'bucket')
return doc
def _getPath(self, path):
"""
Given a fuse path, return the associated resource.
:param path: path within the fuse.
:returns: a Girder resource dictionary.
"""
# If asked about a file in top level directory or the top directory,
# return that it doesn't exist. Other methods should handle '',
# '/user', and 'collection' before calling this method.
if '/' not in path.rstrip('/')[1:]:
raise fuse.FuseOSError(errno.ENOENT)
try:
# We can't filter the resource, since that removes files'
# assetstore information and users' size information.
resource = path_util.lookUpPath(path.rstrip('/'),
filter=False,
force=True)
except (path_util.NotFoundException, AccessException):
raise fuse.FuseOSError(errno.ENOENT)
except ValidationException:
raise fuse.FuseOSError(errno.EROFS)
except Exception:
logger.exception('ServerFuse server internal error')
raise fuse.FuseOSError(errno.EROFS)
return resource # {model, document}
def __call__(self, op, path, *args, **kwargs):
"""
Generically allow logging and error handling for any operation.
:param op: operation to perform.
:param path: path within the fuse (e.g., '', '/user', '/user/<name>',
etc.).
"""
logger.debug('-> %s %s %s', op, path, repr(args))
ret = '[exception]'
try:
ret = getattr(self, op)(path, *args, **kwargs)
return ret
except Exception as e:
# Log all exceptions and then reraise them
if getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES):
logger.debug('-- %s %r', op, e)
else:
logger.exception('-- %s', op)
raise e
finally:
if op != 'read':
logger.debug('<- %s %s', op, repr(ret))
else:
logger.debug('<- %s (length %d) %r', op, len(ret), ret[:16])
def _ldapAuth(event):
login, password = event.info['login'], event.info['password']
servers = Setting().get(PluginSettings.LDAP_SERVERS)
for server in servers:
try:
# ldap requires a uri complete with protocol.
# Append one if the user did not specify.
conn = ldap.initialize(server['uri'])
conn.set_option(ldap.OPT_TIMEOUT, _CONNECT_TIMEOUT)
conn.set_option(ldap.OPT_NETWORK_TIMEOUT, _CONNECT_TIMEOUT)
conn.bind_s(server['bindName'], server['password'],
ldap.AUTH_SIMPLE)
searchStr = '%s=%s' % (server['searchField'], login)
results = conn.search_s(server['baseDn'], ldap.SCOPE_ONELEVEL,
searchStr, _LDAP_ATTRS)
if results:
entry, attrs = results[0]
dn = attrs['distinguishedName'][0].decode('utf8')
try:
conn.bind_s(dn, password, ldap.AUTH_SIMPLE)
except ldap.LDAPError:
# Try other LDAP servers or fall back to core auth
continue
finally:
conn.unbind_s()
user = _getLdapUser(attrs, server)
if user:
event.stopPropagation().preventDefault().addResponse(user)
except ldap.LDAPError:
logger.exception('LDAP connection exception (%s).' % server['uri'])
continue
def __init__(self, assetstore):
"""
:param assetstore: The assetstore to act on.
"""
self.assetstore = assetstore
try:
self.chunkColl = getDbConnection(
assetstore.get('mongohost', None),
assetstore.get('replicaset', None))[assetstore['db']]['chunk']
except pymongo.errors.ConnectionFailure:
logger.error('Failed to connect to GridFS assetstore %s',
assetstore['db'])
self.chunkColl = 'Failed to connect'
self.unavailable = True
return
except pymongo.errors.ConfigurationError:
logger.exception('Failed to configure GridFS assetstore %s',
assetstore['db'])
self.chunkColl = 'Failed to configure'
self.unavailable = True
return
self.chunkColl.ensure_index([
('uuid', pymongo.ASCENDING),
('n', pymongo.ASCENDING)
], unique=True)
def _ldapAuth(event):
login, password = event.info['login'], event.info['password']
servers = Setting().get(PluginSettings.LDAP_SERVERS)
for server in servers:
try:
# ldap requires a uri complete with protocol.
# Append one if the user did not specify.
conn = ldap.initialize(server['uri'])
conn.set_option(ldap.OPT_TIMEOUT, _CONNECT_TIMEOUT)
conn.set_option(ldap.OPT_NETWORK_TIMEOUT, _CONNECT_TIMEOUT)
conn.bind_s(server['bindName'], server['password'], ldap.AUTH_SIMPLE)
searchStr = '%s=%s' % (server['searchField'], login)
# Add the searchStr to the attributes, keep local scope.
lattr = _LDAP_ATTRS + (server['searchField'],)
results = conn.search_s(server['baseDn'], ldap.SCOPE_SUBTREE, searchStr, lattr)
if results:
entry, attrs = results[0]
dn = attrs['distinguishedName'][0].decode('utf8')
try:
conn.bind_s(dn, password, ldap.AUTH_SIMPLE)
except ldap.LDAPError:
# Try other LDAP servers or fall back to core auth
continue
finally:
conn.unbind_s()
user = _getLdapUser(attrs, server)
if user:
event.stopPropagation().preventDefault().addResponse(user)
except ldap.LDAPError:
logger.exception('LDAP connection exception (%s).' % server['uri'])
continue
def cache_tile_frames_job(job):
from girder_jobs.constants import JobStatus
from girder_jobs.models.job import Job
from girder_large_image.models.image_item import ImageItem
from girder import logger
kwargs = job['kwargs']
item = ImageItem().load(kwargs.pop('itemId'), force=True)
job = Job().updateJob(job,
log='Started caching tile frames\n',
status=JobStatus.RUNNING)
try:
for entry in kwargs.get('tileFramesList'):
job = Job().load(job['_id'], force=True)
if job['status'] == JobStatus.CANCELED:
return
job = Job().updateJob(job, log='Caching %r\n' % entry)
ImageItem().tileFrames(item, checkAndCreate=True, **entry)
job = Job().updateJob(job,
log='Finished caching tile frames\n',
status=JobStatus.SUCCESS)
except Exception as exc:
logger.exception('Failed caching tile frames')
job = Job().updateJob(job,
log='Failed caching tile frames (%s)\n' % exc,
status=JobStatus.ERROR)
def CLIListEntrypoint(cli_list_spec_file=None):
if cli_list_spec_file is None:
cli_list_spec_file = os.path.join(os.getcwd(), 'slicer_cli_list.json')
# Parse CLI List spec
with open(cli_list_spec_file, 'rt') as f:
cli_list_spec = json.load(f)
# create command-line argument parser
cmdparser = argparse.ArgumentParser(
formatter_class=_MultilineHelpFormatter)
# add --cli_list
cmdparser.add_argument(
'--list_cli',
action=_make_print_cli_list_spec_action(cli_list_spec_file),
help='Prints the json file containing the list of CLIs present')
# add cl-rel-path argument
cmdparser.add_argument('cli',
help='CLI to run',
metavar='<cli>',
choices=cli_list_spec.keys())
args = cmdparser.parse_args(sys.argv[1:2])
args.cli = os.path.normpath(args.cli)
if cli_list_spec[args.cli]['type'] == 'python':
script_file = os.path.join(args.cli,
os.path.basename(args.cli) + '.py')
# python <cli-rel-path>/<cli-name>.py [<args>]
output_code = subprocess.call([sys.executable, script_file] +
sys.argv[2:])
elif cli_list_spec[args.cli]['type'] == 'cxx':
script_file = os.path.join('.', args.cli, os.path.basename(args.cli))
if os.path.isfile(script_file):
# ./<cli-rel-path>/<cli-name> [<args>]
output_code = subprocess.call([script_file] + sys.argv[2:])
else:
# assumes parent dir of CLI executable is in ${PATH}
output_code = subprocess.call([os.path.basename(args.cli)] +
sys.argv[2:])
else:
logger.exception('CLIs of type %s are not supported',
cli_list_spec[args.cli]['type'])
raise Exception('CLIs of type %s are not supported',
cli_list_spec[args.cli]['type'])
return output_code
def createThumbnailsJobTask(item, spec):
"""
For an individual item, check or create all of the appropriate thumbnails.
:param item: the image item.
:param spec: a list of thumbnail specifications.
:returns: a dictionary with the total status of the thumbnail job.
"""
status = {'checked': 0, 'created': 0, 'failed': 0}
for entry in spec:
try:
if entry.get('imageKey'):
result = ImageItem().getAssociatedImage(item,
checkAndCreate=True,
**entry)
else:
result = ImageItem().getThumbnail(item,
checkAndCreate=True,
**entry)
status['checked' if result is True else 'created'] += 1
except TileGeneralException as exc:
status['failed'] += 1
status['lastFailed'] = str(item['_id'])
logger.info('Failed to get thumbnail for item %s: %r' %
(item['_id'], exc))
except AttributeError:
raise
except Exception:
status['failed'] += 1
status['lastFailed'] = str(item['_id'])
logger.exception(
'Unexpected exception when trying to create a thumbnail for item %s'
% item['_id'])
return status
def endpointDecorator(self, *path, **params):
_setCommonCORSHeaders()
cherrypy.lib.caching.expires(0)
cherrypy.request.girderRequestUid = str(uuid.uuid4())
setResponseHeader('Girder-Request-Uid',
cherrypy.request.girderRequestUid)
try:
val = fun(self, path, params)
# If this is a partial response, we set the status appropriately
if 'Content-Range' in cherrypy.response.headers:
cherrypy.response.status = 206
val = _mongoCursorToList(val)
if callable(val):
# If the endpoint returned anything callable (function,
# lambda, functools.partial), we assume it's a generator
# function for a streaming response.
cherrypy.response.stream = True
_logRestRequest(self, path, params)
return val()
if isinstance(val, cherrypy.lib.file_generator):
# Don't do any post-processing of static files
return val
if isinstance(val, types.GeneratorType):
val = list(val)
except RestException as e:
val = _handleRestException(e)
except AccessException as e:
val = _handleAccessException(e)
except GirderException as e:
val = _handleGirderException(e)
except ValidationException as e:
val = _handleValidationException(e)
except cherrypy.HTTPRedirect:
raise
except Exception:
# These are unexpected failures; send a 500 status
logger.exception('500 Error')
cherrypy.response.status = 500
val = dict(type='internal', uid=cherrypy.request.girderRequestUid)
if config.getConfig()['server']['mode'] == 'production':
# Sanitize errors in production mode
val['message'] = 'An unexpected error occurred on the server.'
else:
# Provide error details in non-production modes
t, value, tb = sys.exc_info()
val['message'] = '%s: %s' % (t.__name__, repr(value))
val['trace'] = traceback.extract_tb(tb)
resp = _createResponse(val)
_logRestRequest(self, path, params)
return resp
def _addOptionalInputParamsToContainerArgs(opt_input_params, containerArgs,
hargs):
for param in opt_input_params:
if param.longflag:
curFlag = param.longflag
elif param.flag:
curFlag = param.flag
else:
continue
if _is_on_girder(param) and param.identifier() in hargs:
curValue = "$input{%s}" % param.identifier()
elif param.identifier() in hargs['params']:
try:
curValue = _getParamCommandLineValue(
param, hargs['params'][param.identifier()])
except Exception:
logger.exception(
'Error: Parameter value is not in json.dumps format\n'
' Parameter name = %r\n Parameter type = %r\n'
' Value passed = %r', param.identifier(), param.typ,
hargs['params'][param.identifier()])
raise
else:
continue
containerArgs.append(curFlag)
containerArgs.append(curValue)
def _addAssociatedImage(self, largeImagePath, directoryNum):
"""
Check if the specfied TIFF directory contains a non-tiled image with a
sensible image description that can be used as an ID. If so, and if
the image isn't too large, add this image as an associated image.
:param largeImagePath: path to the TIFF file.
:param directoryNum: libtiff directory number of the image.
"""
try:
associated = TiledTiffDirectory(largeImagePath, directoryNum,
False)
id = associated._tiffInfo.get('imagedescription').strip().split(
None, 1)[0].lower()
if not isinstance(id, six.text_type):
id = id.decode('utf8')
# Only use this as an associated image if the parsed id is
# a reasonable length, alphanumeric characters, and the
# image isn't too large.
if (id.isalnum() and len(id) > 3 and len(id) <= 20
and associated._pixelInfo['width'] <= 8192
and associated._pixelInfo['height'] <= 8192):
self._associatedImages[id] = associated._tiffFile.read_image()
except (TiffException, AttributeError):
# If we can't validate or read an associated image or it has no
# useful imagedescription, fail quietly without adding an
# associated image.
pass
except Exception:
# If we fail for other reasons, don't raise an exception, but log
# what happened.
logger.exception(
'Could not use non-tiled TIFF image as an associated image.')
def updateAnnotation(self, annotation, params):
# Set the response time limit to a very long value
setResponseTimeLimit(86400)
user = self.getCurrentUser()
item = Item().load(annotation.get('itemId'), force=True)
if item is not None:
Item().requireAccess(item, user=user, level=AccessType.WRITE)
# If we have a content length, then we have replacement JSON. If
# elements are not included, don't replace them
returnElements = True
if cherrypy.request.body.length:
oldElements = annotation.get('annotation', {}).get('elements')
annotation['annotation'] = self.getBodyJson()
if 'elements' not in annotation['annotation'] and oldElements:
annotation['annotation']['elements'] = oldElements
returnElements = False
if params.get('itemId'):
newitem = Item().load(params['itemId'], force=True)
Item().requireAccess(newitem, user=user, level=AccessType.WRITE)
annotation['itemId'] = newitem['_id']
try:
annotation = Annotation().updateAnnotation(annotation,
updateUser=user)
except ValidationException as exc:
logger.exception('Failed to validate annotation')
raise RestException(
"Validation Error: JSON doesn't follow schema (%r)." %
(exc.args, ))
if not returnElements and 'elements' in annotation['annotation']:
del annotation['annotation']['elements']
return annotation
def getDockerOutput(imgName, command, client):
"""
Data from each docker image is collected by executing the equivalent of a
docker run <imgName> <command/args>
and collecting the output to standard output
:param imgName: The name of the docker image
:param command: The commands/ arguments to be passed to the docker image
:param client: The docker python client
"""
cont = None
try:
cont = client.containers.create(image=imgName, command=command)
cont.start()
ret_code = cont.wait()
if isinstance(ret_code, dict):
ret_code = ret_code['StatusCode']
logs = cont.logs(stdout=True, stderr=False, stream=False)
cont.remove()
except Exception as err:
if cont:
try:
cont.remove()
except Exception:
pass
logger.exception('Attempt to docker run %s %s failed', imgName,
command)
raise DockerImageError(
'Attempt to docker run %s %s failed ' % (imgName, command) +
str(err), imgName)
if ret_code != 0:
raise DockerImageError(
'Attempt to docker run %s %s failed' % (imgName, command), imgName)
return logs
def _parseParamValue(param, value, user, token):
if isinstance(value, six.binary_type):
value = value.decode('utf8')
param_id = param.identifier()
if is_on_girder(param):
girder_type = SLICER_TYPE_TO_GIRDER_MODEL_MAP[param.typ]
curModel = ModelImporter.model(girder_type)
loaded = curModel.load(value, level=AccessType.READ, user=user)
if not loaded:
raise RestException('Invalid %s id (%s).' %
(curModel.name, str(value)))
return loaded
try:
if param.isVector():
return '%s' % ', '.join(map(str, json.loads(value)))
elif param.typ in OPENAPI_DIRECT_TYPES or param.typ == 'string-enumeration':
return str(value)
else: # json
return str(json.loads(value))
except json.JSONDecodeError:
msg = 'Error: Parameter value is not in json.dumps format\n' \
' Parameter name = %r\n Parameter type = %r\n' \
' Value passed = %r' % (param_id, param.typ, value)
logger.exception(msg)
raise RestException(msg)
def createThumb(self, file, challenge, params):
self.requireParams('size', params)
size = int(params['size'])
user = self.getCurrentUser()
if challenge.get('thumbnailSourceId') != file['_id']:
challenge['thumbnails'] = []
challenge['thumbnailSourceId'] = file['_id']
i = 0
for thumbnail in challenge['thumbnails']:
if thumbnail['size'] == size:
return File().filter(
File().load(thumbnail['fileId'], force=True),
user)
elif size < thumbnail['size']:
break
i += 1
try:
newThumb = createThumbnail(
width=size, height=size, crop=True, fileId=file['_id'],
attachToType='item', attachToId=file['itemId'])
except IOError:
logger.exception('Thumbnail creation IOError')
raise RestException('Could not create thumbnail from the file.')
challenge['thumbnails'].insert(i, {
'size': size,
'fileId': newThumb['_id']
})
Challenge().save(challenge)
return File().filter(newThumb, user)
def getCliData(name, client, jobModel, job):
try:
cli_dict = getDockerOutput(name, '--list_cli', client)
# contains nested dict
# {<cliname>:{type:<type>}}
if isinstance(cli_dict, six.binary_type):
cli_dict = cli_dict.decode('utf8')
cli_dict = json.loads(cli_dict)
for key, info in six.iteritems(cli_dict):
desc_type = info.get('desc-type', 'xml')
cli_desc = getDockerOutput(name, '%s --%s' % (key, desc_type),
client)
if isinstance(cli_desc, six.binary_type):
cli_desc = cli_desc.decode('utf8')
cli_dict[key][desc_type] = cli_desc
jobModel.updateJob(
job,
log='Got image %s, cli %s metadata\n' % (name, key),
status=JobStatus.RUNNING,
)
return cli_dict
except Exception as err:
logger.exception('Error getting %s cli data from image', name)
raise DockerImageError('Error getting %s cli data from image ' %
(name) + str(err))
def createThumb(self, file, challenge, params):
self.requireParams('size', params)
size = int(params['size'])
user = self.getCurrentUser()
if challenge.get('thumbnailSourceId') != file['_id']:
challenge['thumbnails'] = []
challenge['thumbnailSourceId'] = file['_id']
i = 0
for thumbnail in challenge['thumbnails']:
if thumbnail['size'] == size:
return self.model('file').filter(
self.model('file').load(thumbnail['fileId'], force=True),
user)
elif size < thumbnail['size']:
break
i += 1
try:
newThumb = createThumbnail(
width=size, height=size, crop=True, fileId=file['_id'],
attachToType='item', attachToId=file['itemId'])
except IOError:
logger.exception('Thumbnail creation IOError')
raise RestException('Could not create thumbnail from the file.')
challenge['thumbnails'].insert(i, {
'size': size,
'fileId': newThumb['_id']
})
self.model('challenge', 'challenge').save(challenge)
return self.model('file').filter(newThumb, user)
def _getPath(self, path):
"""
Given a fuse path, return the associated resource.
:param path: path within the fuse.
:returns: a Girder resource dictionary.
"""
# If asked about a file in top level directory or the top directory,
# return that it doesn't exist. Other methods should handle '',
# '/user', and 'collection' before calling this method.
if '/' not in path.rstrip('/')[1:]:
raise fuse.FuseOSError(errno.ENOENT)
try:
# We can't filter the resource, since that removes files'
# assetstore information and users' size information.
resource = path_util.lookUpPath(
path.rstrip('/'), filter=False, force=True)
except (path_util.NotFoundException, AccessException):
raise fuse.FuseOSError(errno.ENOENT)
except ValidationException:
raise fuse.FuseOSError(errno.EROFS)
except Exception:
logger.exception('ServerFuse server internal error')
raise fuse.FuseOSError(errno.EROFS)
return resource # {model, document}
def updateAnnotation(self, annotation, params):
user = self.getCurrentUser()
item = self.model('item').load(annotation.get('itemId'), force=True)
if item is not None:
self.model('item').requireAccess(item,
user=user,
level=AccessType.WRITE)
# If we have a content length, then we have replacement JSON.
if cherrypy.request.body.length:
annotation['annotation'] = self.getBodyJson()
if params.get('itemId'):
newitem = self.model('item').load(params['itemId'], force=True)
self.model('item').requireAccess(newitem,
user=user,
level=AccessType.WRITE)
annotation['itemId'] = newitem['_id']
try:
self.model('annotation',
'large_image').updateAnnotation(annotation,
updateUser=user)
except ValidationException as exc:
logger.exception('Failed to validate annotation')
raise RestException(
'Validation Error: JSON doesn\'t follow schema (%s).' %
(exc.message, ))
return annotation
def __call__(self, op, path, *args, **kwargs):
"""
Generically allow logging and error handling for any operation.
:param op: operation to perform.
:param path: path within the fuse (e.g., '', '/user', '/user/<name>',
etc.).
"""
logger.debug('-> %s %s %s', op, path, repr(args))
ret = '[exception]'
try:
ret = getattr(self, op)(path, *args, **kwargs)
return ret
except Exception as e:
# Log all exceptions and then reraise them
if getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES):
logger.debug('-- %s %r', op, e)
else:
logger.exception('-- %s', op)
raise e
finally:
if op != 'read':
logger.debug('<- %s %s', op, repr(ret))
else:
logger.debug('<- %s (length %d) %r', op, len(ret), ret[:16])
def __init__(self, name):
try:
if isinstance(name, string_types):
imageKey = DockerImage.getHashKey(name)
self.data = {}
self.data[DockerImage.imageName] = name
self.data[DockerImage.cli_dict] = {}
self.data[DockerImage.imageHash] = imageKey
self.hash = imageKey
self.name = name
# TODO check/validate schema of dict
elif isinstance(name, dict):
jsonschema.validate(name, DockerImageStructure.ImageSchema)
self.data = name.copy()
self.name = self.data[DockerImage.imageName]
self.hash = DockerImage.getHashKey(self.name)
else:
raise DockerImageError(
'Image should be a string, or dict'
' could not add the image', 'bad init val')
except Exception as err:
logger.exception('Could not initialize docker image %r', name)
raise DockerImageError(
'Could not initialize instance of Docker Image \n' + str(err))
def getCliData(name, client, img, jobModel, job):
try:
if isinstance(client, docker.DockerClient) and isinstance(
img, DockerImage):
cli_dict = getDockerOutput(name, '--list_cli', client)
# contains nested dict
# {<cliname>:{type:<type>}}
if isinstance(cli_dict, six.binary_type):
cli_dict = cli_dict.decode('utf8')
cli_dict = json.loads(cli_dict)
for (key, val) in six.iteritems(cli_dict):
cli_xml = getDockerOutput(name, '%s --xml' % key, client)
if isinstance(cli_xml, six.binary_type):
cli_xml = cli_xml.decode('utf8')
cli_dict[key][DockerImage.xml] = cli_xml
jobModel.updateJob(
job,
log='Got image %s, cli %s metadata\n' % (name, key),
status=JobStatus.RUNNING,
)
img.addCLI(key, cli_dict[key])
return cli_dict
except Exception as err:
logger.exception('Error getting %s cli data from image %s', name, img)
raise DockerImageError('Error getting %s cli data from image %s ' %
(name, img) + str(err))
def gcs_save_record(self, data: dict):
"""
https://cloud.google.com/pubsub/docs/push#receiving_messages
"""
try:
payload = GCSPushNotificationPayload(**data)
GCSNotificationRecord().create(payload.message)
if payload.message.attributes.eventType == 'OBJECT_FINALIZE':
# This is a create notification
store = Assetstore().findOne({
'type':
2, # S3 type
AssetstoreRuleMarker: {
'$exists': True
},
'bucket':
payload.message.attributes.bucketId,
# The only viable GSC Service string
'service':
'https://storage.googleapis.com',
})
if store is not None:
rule = NotificationRouterRule(
**store[AssetstoreRuleMarker])
mountRoot = Folder().findOne(
{'_id': ObjectId(rule.folderId)})
BucketNotification.processNotification(
store, mountRoot, payload.message.attributes.objectId)
except Exception as err:
# exceptions must be swallowed to prevent pub/sub queue backups
# message loss is always easily recoverable by running a manual
# import through the admin console.
logger.exception(f'Failed to process GCS notification {err}')
return "done"
def trigger(eventName, info=None):
"""
Fire an event with the given name. All listeners bound on that name will be
called until they are exhausted or one of the handlers calls the
stopPropagation() method on the event.
:param eventName: The name that identifies the event.
:type eventName: str
:param info: The info argument to pass to the handler function. The type of
this argument is opaque, and can be anything.
:return
"""
global _mapping
e = Event(eventName, info)
for handlerName, handler in _mapping.get(eventName, {}).iteritems():
e.currentHandlerName = handlerName
try:
handler(e)
except:
logger.exception('In handler "{}" for event "{}":'
.format(handlerName, eventName))
if e.propagate is False:
break
return e
def endpointDecorator(self, *path, **params):
_setCommonCORSHeaders()
cherrypy.lib.caching.expires(0)
cherrypy.request.girderRequestUid = str(uuid.uuid4())
setResponseHeader('Girder-Request-Uid', cherrypy.request.girderRequestUid)
try:
val = fun(self, path, params)
# If this is a partial response, we set the status appropriately
if 'Content-Range' in cherrypy.response.headers:
cherrypy.response.status = 206
val = _mongoCursorToList(val)
if callable(val):
# If the endpoint returned anything callable (function,
# lambda, functools.partial), we assume it's a generator
# function for a streaming response.
cherrypy.response.stream = True
_logRestRequest(self, path, params)
return val()
if isinstance(val, cherrypy.lib.file_generator):
# Don't do any post-processing of static files
return val
if isinstance(val, types.GeneratorType):
val = list(val)
except RestException as e:
val = _handleRestException(e)
except AccessException as e:
val = _handleAccessException(e)
except GirderException as e:
val = _handleGirderException(e)
except ValidationException as e:
val = _handleValidationException(e)
except cherrypy.HTTPRedirect:
raise
except Exception:
# These are unexpected failures; send a 500 status
logger.exception('500 Error')
cherrypy.response.status = 500
val = dict(type='internal', uid=cherrypy.request.girderRequestUid)
if config.getConfig()['server']['mode'] == 'production':
# Sanitize errors in production mode
val['message'] = 'An unexpected error occurred on the server.'
else:
# Provide error details in non-production modes
t, value, tb = sys.exc_info()
val['message'] = '%s: %s' % (t.__name__, repr(value))
val['trace'] = traceback.extract_tb(tb)
resp = _createResponse(val)
_logRestRequest(self, path, params)
return resp
def _handleGirderException(e):
# Handle general Girder exceptions
logger.exception('500 Error')
cherrypy.response.status = 500
val = {'message': e.message, 'type': 'girder'}
if e.identifier is not None:
val['identifier'] = e.identifier
return val
def _handleGirderException(e):
# Handle general Girder exceptions
logger.exception('500 Error')
cherrypy.response.status = 500
val = {'message': e.message, 'type': 'girder'}
if e.identifier is not None:
val['identifier'] = e.identifier
return val
def createAnnotation(self, item, params):
try:
return self.model('annotation', 'large_image').createAnnotation(
item, self.getCurrentUser(), self.getBodyJson())
except ValidationException as exc:
logger.exception('Failed to validate annotation')
raise RestException(
'Validation Error: JSON doesn\'t follow schema (%s).' %
(exc.message, ))
def createAnnotation(self, item, params):
try:
return self.model('annotation', 'large_image').createAnnotation(
item, self.getCurrentUser(), self.getBodyJson())
except ValidationException as exc:
logger.exception('Failed to validate annotation')
raise RestException(
'Validation Error: JSON doesn\'t follow schema (%s).' % (
exc.message, ))
def _handleAccessException(e):
# Permission exceptions should throw a 401 or 403, depending
# on whether the user is logged in or not
if getCurrentUser() is None:
cherrypy.response.status = 401
else:
cherrypy.response.status = 403
logger.exception('403 Error')
return {'message': e.message, 'type': 'access'}
def undoFunction():
try:
restResource.removeRoute('POST', restRunPath, cliRunHandler)
if registerNamedRoute:
restResource.removeRoute('POST', restNamedRunPath,
cliRunHandler)
delattr(restResource, cliRunHandlerName)
except Exception:
logger.exception('Failed to remove route')
def _handleAccessException(e):
# Permission exceptions should throw a 401 or 403, depending
# on whether the user is logged in or not
if getCurrentUser() is None:
cherrypy.response.status = 401
else:
cherrypy.response.status = 403
logger.exception('403 Error')
return {'message': e.message, 'type': 'access'}
def createColormap(self, name, public, colormap, labels):
try:
return Colormap().createColormap(self.getCurrentUser(), colormap,
name, labels, public)
except ValidationException as exc:
logger.exception('Failed to validate colormap')
raise RestException(
"Validation Error: JSON doesn\'t follow schema (%r)." %
(exc.args, ))
def send_new_user_email(event):
try:
info = event.info
email = info.get('email')
brandName = Setting().get(SettingKey.BRAND_NAME)
rendered = renderTemplate('welcome.mako')
sendMail(f'Welcome to {brandName}', rendered, [email])
except Exception:
logger.exception("Failed to send new user email")
def validateInfo(doc):
"""
Makes sure the root field is a valid absolute path and is writeable.
"""
if 'prefix' not in doc:
doc['prefix'] = ''
# remove slashes from front and back of the prefix
doc['prefix'] = doc['prefix'].strip('/')
if not doc.get('bucket'):
raise ValidationException('Bucket must not be empty.', 'bucket')
if not doc.get('readOnly'):
if not doc.get('secret'):
raise ValidationException('Secret key must not be empty.',
'secret')
if not doc.get('accessKeyId'):
raise ValidationException('Access key ID must not be empty.',
'accessKeyId')
# construct a set of connection parameters based on the keys and the
# service
if 'service' not in doc:
doc['service'] = ''
if doc['service'] != '':
service = re.match("^((https?)://)?([^:/]+)(:([0-9]+))?$",
doc['service'])
if not service:
raise ValidationException(
'The service must of the form [http[s]://](host domain)'
'[:(port)].', 'service')
doc['botoConnect'] = makeBotoConnectParams(doc['accessKeyId'],
doc['secret'],
doc['service'])
# Make sure we can write into the given bucket using boto
conn = botoConnectS3(doc['botoConnect'])
if doc.get('readOnly'):
try:
conn.get_bucket(bucket_name=doc['bucket'], validate=True)
except Exception:
logger.exception('S3 assetstore validation exception')
raise ValidationException(
'Unable to connect to bucket "%s".' % doc['bucket'],
'bucket')
else:
try:
bucket = conn.get_bucket(bucket_name=doc['bucket'],
validate=True)
testKey = boto.s3.key.Key(bucket=bucket,
name='/'.join(
filter(None,
(doc['prefix'], 'test'))))
testKey.set_contents_from_string('')
except Exception:
logger.exception('S3 assetstore validation exception')
raise ValidationException(
'Unable to write into bucket "%s".' % doc['bucket'],
'bucket')
return doc
def createAnnotation(self, item, params):
try:
return Annotation().createAnnotation(item, self.getCurrentUser(),
self.getBodyJson())
except ValidationException as exc:
logger.exception('Failed to validate annotation')
raise RestException(
"Validation Error: JSON doesn't follow schema (%r)." %
(exc.args, ))
def import_recursive(job):
try:
root = job['kwargs']['root']
token = job['kwargs']['token']
user = User().load(job['userId'], force=True)
children = list(Folder().childFolders(root, 'collection', user=user))
count = len(children)
progress = 0
job = Job().updateJob(job,
log='Started TCGA import\n',
status=JobStatus.RUNNING,
progressCurrent=progress,
progressTotal=count)
logger.info('Starting recursive TCGA import')
for child in children:
progress += 1
try:
msg = 'Importing "%s"' % child.get('name', '')
job = Job().updateJob(job,
log=msg,
progressMessage=msg + '\n',
progressCurrent=progress)
logger.debug(msg)
Cohort().importDocument(child,
recurse=True,
user=user,
token=token,
job=job)
job = Job().load(id=job['_id'], force=True)
# handle any request to stop execution
if (not job or job['status']
in (JobStatus.CANCELED, JobStatus.ERROR)):
logger.info('TCGA import job halted with')
return
except ValidationException:
logger.warning('Failed to import %s' % child.get('name', ''))
logger.info('Starting recursive TCGA import')
job = Job().updateJob(job,
log='Finished TCGA import\n',
status=JobStatus.SUCCESS,
progressCurrent=count,
progressMessage='Finished TCGA import')
except Exception as e:
logger.exception('Importing TCGA failed with %s' % str(e))
job = Job().updateJob(job,
log='Import failed with %s\n' % str(e),
status=JobStatus.ERROR)
def endpointDecorator(self, *path, **params):
_setCommonCORSHeaders()
cherrypy.lib.caching.expires(0)
try:
val = fun(self, path, params)
# If this is a partial response, we set the status appropriately
if 'Content-Range' in cherrypy.response.headers:
cherrypy.response.status = 206
val = _mongoCursorToList(val)
if callable(val):
# If the endpoint returned anything callable (function,
# lambda, functools.partial), we assume it's a generator
# function for a streaming response.
cherrypy.response.stream = True
_logRestRequest(self, path, params)
return val()
if isinstance(val, cherrypy.lib.file_generator):
# Don't do any post-processing of static files
return val
if isinstance(val, types.GeneratorType):
val = list(val)
except RestException as e:
val = _handleRestException(e)
except AccessException as e:
val = _handleAccessException(e)
except GirderException as e:
val = _handleGirderException(e)
except ValidationException as e:
val = _handleValidationException(e)
except cherrypy.HTTPRedirect:
raise
except Exception:
# These are unexpected failures; send a 500 status
logger.exception('500 Error')
cherrypy.response.status = 500
t, value, tb = sys.exc_info()
val = {'message': '%s: %s' % (t.__name__, repr(value)),
'type': 'internal'}
curConfig = config.getConfig()
if curConfig['server']['mode'] != 'production':
# Unless we are in production mode, send a traceback too
val['trace'] = traceback.extract_tb(tb)
resp = _createResponse(val)
_logRestRequest(self, path, params)
return resp
def wrapped(*args, **kwargs):
try:
return fun(*args, **kwargs)
except ResourcePathNotFound:
return paramiko.SFTP_NO_SUCH_FILE
except ValidationException:
return paramiko.SFTP_FAILURE
except AccessException:
return paramiko.SFTP_PERMISSION_DENIED
except Exception:
logger.exception('SFTP server internal error')
return paramiko.SFTP_FAILURE
def wrapped(*args, **kwargs):
try:
return fun(*args, **kwargs)
except ResourcePathNotFound:
return paramiko.SFTP_NO_SUCH_FILE
except ValidationException:
return paramiko.SFTP_FAILURE
except AccessException:
return paramiko.SFTP_PERMISSION_DENIED
except Exception:
logger.exception('SFTP server internal error')
return paramiko.SFTP_FAILURE
def validateInfo(doc):
"""
Makes sure the root field is a valid absolute path and is writeable.
"""
if 'prefix' not in doc:
doc['prefix'] = ''
# remove slashes from front and back of the prefix
doc['prefix'] = doc['prefix'].strip('/')
if not doc.get('bucket'):
raise ValidationException('Bucket must not be empty.', 'bucket')
if not doc.get('readOnly'):
if not doc.get('secret'):
raise ValidationException(
'Secret key must not be empty.', 'secret')
if not doc.get('accessKeyId'):
raise ValidationException(
'Access key ID must not be empty.', 'accessKeyId')
# construct a set of connection parameters based on the keys and the
# service
if 'service' not in doc:
doc['service'] = ''
if doc['service'] != '':
service = re.match("^((https?)://)?([^:/]+)(:([0-9]+))?$",
doc['service'])
if not service:
raise ValidationException(
'The service must of the form [http[s]://](host domain)'
'[:(port)].', 'service')
doc['botoConnect'] = makeBotoConnectParams(
doc['accessKeyId'], doc['secret'], doc['service'])
# Make sure we can write into the given bucket using boto
conn = botoConnectS3(doc['botoConnect'])
if doc.get('readOnly'):
try:
conn.get_bucket(bucket_name=doc['bucket'], validate=True)
except Exception:
logger.exception('S3 assetstore validation exception')
raise ValidationException('Unable to connect to bucket "%s".' %
doc['bucket'], 'bucket')
else:
try:
bucket = conn.get_bucket(bucket_name=doc['bucket'],
validate=True)
testKey = boto.s3.key.Key(
bucket=bucket, name='/'.join(
filter(None, (doc['prefix'], 'test'))))
testKey.set_contents_from_string('')
except Exception:
logger.exception('S3 assetstore validation exception')
raise ValidationException('Unable to write into bucket "%s".' %
doc['bucket'], 'bucket')
return doc
def _s3Client(connectParams):
try:
client = boto3.client('s3', **connectParams)
if 'googleapis' in urllib.parse.urlparse(connectParams.get(
'endpoint_url', '')).netloc.split('.'):
client.meta.events.unregister(
'before-parameter-build.s3.ListObjects',
botocore.handlers.set_list_objects_encoding_type_url)
client._useGoogleAccessId = True
return client
except Exception:
logger.exception('S3 assetstore validation exception')
raise ValidationException('Unable to connect to S3 assetstore')
def botoConnectS3(connectParams):
"""
Connect to the S3 server, throwing an appropriate exception if we fail.
:param connectParams: a dictionary of paramters to use in the connection.
:returns: the boto connection object.
"""
try:
conn = boto.connect_s3(calling_format=BotoCallingFormat(),
**connectParams)
except Exception:
logger.exception('S3 assetstore validation exception')
raise ValidationException('Unable to connect to S3 assetstore')
return conn
def __init__(self, assetstore):
self.assetstore = assetstore
# If we can't create the temp directory, the assetstore still needs to
# be initialized so that it can be deleted or modified. The validation
# prevents invalid new assetstores from being created, so this only
# happens to existing assetstores that no longer can access their temp
# directories.
self.tempDir = os.path.join(assetstore['root'], 'temp')
if not os.path.exists(self.tempDir):
try:
os.makedirs(self.tempDir)
except OSError:
logger.exception('Failed to create filesystem assetstore '
'directories {}'.format(self.tempDir))
def endpointDecorator(self, *args, **kwargs):
# Note that the cyclomatic complexity of this function crosses our
# flake8 configuration threshold. Because it is largely exception
# handling, I think that breaking it into smaller functions actually
# reduces readability and maintainability. To work around this, some
# simple branches have been marked to be skipped in the cyclomatic
# analysis.
_setCommonCORSHeaders()
cherrypy.lib.caching.expires(0)
try:
val = fun(self, args, kwargs)
# If this is a partial response, we set the status appropriately
if 'Content-Range' in cherrypy.response.headers:
cherrypy.response.status = 206
if callable(val):
# If the endpoint returned anything callable (function,
# lambda, functools.partial), we assume it's a generator
# function for a streaming response.
cherrypy.response.stream = True
return val()
if isinstance(val, cherrypy.lib.file_generator):
# Don't do any post-processing of static files
return val
except RestException as e:
val = _handleRestException(e)
except AccessException as e:
val = _handleAccessException(e)
except GirderException as e:
val = _handleGirderException(e)
except ValidationException as e:
val = _handleValidationException(e)
except cherrypy.HTTPRedirect:
raise
except Exception:
# These are unexpected failures; send a 500 status
logger.exception('500 Error')
cherrypy.response.status = 500
t, value, tb = sys.exc_info()
val = {'message': '%s: %s' % (t.__name__, repr(value)),
'type': 'internal'}
curConfig = config.getConfig()
if curConfig['server']['mode'] != 'production':
# Unless we are in production mode, send a traceback too
val['trace'] = traceback.extract_tb(tb)
return _createResponse(val)
def botoConnectS3(connectParams):
"""
Connect to the S3 server, throwing an appropriate exception if we fail.
:param connectParams: a dictionary of parameters to use in the connection.
:returns: the boto connection object.
"""
if 'anon' not in connectParams or not connectParams['anon']:
connectParams = connectParams.copy()
connectParams['calling_format'] = BotoCallingFormat()
try:
return boto.connect_s3(**connectParams)
except Exception:
logger.exception('S3 assetstore validation exception')
raise ValidationException('Unable to connect to S3 assetstore')
def _loadModel(model, module, plugin):
className = camelcase(model)
try:
imported = importlib.import_module(module)
except ImportError:
logger.exception('Could not load model "%s".' % module)
raise
try:
constructor = getattr(imported, className)
except AttributeError:
raise Exception('Incorrect model class name "%s" for model "%s".' % (
className, module))
_modelInstances[plugin][model] = constructor()
def capacityInfo(self):
"""
For filesystem assetstores, we just need to report the free and total
space on the filesystem where the assetstore lives.
"""
try:
usage = psutil.disk_usage(self.assetstore['root'])
return {'free': usage.free, 'total': usage.total}
except OSError:
logger.exception(
'Failed to get disk usage of %s' % self.assetstore['root'])
# If psutil.disk_usage fails or we can't query the assetstore's root
# directory, just report nothing regarding disk capacity
return { # pragma: no cover
'free': None,
'total': None
}
def _loadModel(model, module, plugin):
global _modelInstances
className = camelcase(model)
try:
imported = importlib.import_module(module)
except ImportError:
logger.exception('Could not load model "{}".'.format(module))
raise
try:
constructor = getattr(imported, className)
except AttributeError: # pragma: no cover
raise Exception('Incorrect model class name "{}" for model "{}".'
.format(className, module))
_modelInstances[plugin][model] = constructor()
def endpointDecorator(self, *args, **kwargs):
try:
val = fun(self, args, kwargs)
if isinstance(val, types.FunctionType):
# If the endpoint returned a function, we assume it's a
# generator function for a streaming response.
cherrypy.response.stream = True
return val()
except RestException as e:
# Handle all user-error exceptions from the rest layer
cherrypy.response.status = e.code
val = {'message': e.message, 'type': 'rest'}
if e.extra is not None:
val['extra'] = e.extra
except AccessException as e:
# Permission exceptions should throw a 401 or 403, depending
# on whether the user is logged in or not
if self.getCurrentUser() is None:
cherrypy.response.status = 401
else:
cherrypy.response.status = 403
logger.exception('403 Error')
val = {'message': e.message, 'type': 'access'}
except ValidationException as e:
cherrypy.response.status = 400
val = {'message': e.message, 'type': 'validation'}
if e.field is not None:
val['field'] = e.field
except cherrypy.HTTPRedirect:
raise
except:
# These are unexpected failures; send a 500 status
logger.exception('500 Error')
cherrypy.response.status = 500
t, value, tb = sys.exc_info()
val = {'message': '%s: %s' % (t.__name__, str(value)),
'type': 'internal'}
curConfig = config.getConfig()
if curConfig['server']['mode'] != 'production':
# Unless we are in production mode, send a traceback too
val['trace'] = traceback.extract_tb(tb)
return _createResponse(val)
def __init__(self, assetstore):
super(FilesystemAssetstoreAdapter, self).__init__(assetstore)
# If we can't create the temp directory, the assetstore still needs to
# be initialized so that it can be deleted or modified. The validation
# prevents invalid new assetstores from being created, so this only
# happens to existing assetstores that no longer can access their temp
# directories.
self.tempDir = os.path.join(self.assetstore['root'], 'temp')
try:
mkdir(self.tempDir)
except OSError:
self.unavailable = True
logger.exception('Failed to create filesystem assetstore '
'directories %s' % self.tempDir)
if not os.access(self.assetstore['root'], os.W_OK):
self.unavailable = True
logger.error('Could not write to assetstore root: %s',
self.assetstore['root'])
def __init__(self, assetstore):
"""
:param assetstore: The assetstore to act on.
"""
super(GridFsAssetstoreAdapter, self).__init__(assetstore)
recent = False
try:
# Guard in case the connectionArgs is unhashable
key = (self.assetstore.get('mongohost'),
self.assetstore.get('replicaset'),
self.assetstore.get('shard'))
if key in _recentConnections:
recent = (time.time() - _recentConnections[key]['created'] <
RECENT_CONNECTION_CACHE_TIME)
except TypeError:
key = None
try:
# MongoClient automatically reuses connections from a pool, but we
# want to avoid redoing ensureChunkIndices each time we get such a
# connection.
client = getDbConnection(self.assetstore.get('mongohost'),
self.assetstore.get('replicaset'),
quiet=recent)
self.chunkColl = MongoProxy(client[self.assetstore['db']].chunk)
if not recent:
_ensureChunkIndices(self.chunkColl)
if self.assetstore.get('shard') == 'auto':
_setupSharding(self.chunkColl)
if key is not None:
if len(_recentConnections) >= RECENT_CONNECTION_CACHE_MAX_SIZE:
_recentConnections.clear()
_recentConnections[key] = {
'created': time.time()
}
except pymongo.errors.ConnectionFailure:
logger.error('Failed to connect to GridFS assetstore %s',
self.assetstore['db'])
self.chunkColl = 'Failed to connect'
self.unavailable = True
except pymongo.errors.ConfigurationError:
logger.exception('Failed to configure GridFS assetstore %s',
self.assetstore['db'])
self.chunkColl = 'Failed to configure'
self.unavailable = True
def run(self):
"""
Loops over all queued events. If the queue is empty, this thread gets
put to sleep until someone calls trigger() on it with a new event to
dispatch.
"""
print(TerminalColor.info('Started asynchronous event manager thread.'))
while not self.terminate:
eventName, info, callback = self.eventQueue.get(block=True)
try:
event = trigger(eventName, info, async=True)
if isinstance(callback, types.FunctionType):
callback(event)
except Exception:
logger.exception('In handler for event "%s":' % eventName)
pass # Must continue the event loop even if handler failed
print(TerminalColor.info('Stopped asynchronous event manager thread.'))
def validateInfo(doc):
"""
Makes sure the root field is a valid absolute path and is writeable.
"""
if 'prefix' not in doc:
doc['prefix'] = ''
# remove slashes from front and back of the prefix
doc['prefix'] = doc['prefix'].strip('/')
if not doc.get('bucket'):
raise ValidationException('Bucket must not be empty.', 'bucket')
# construct a set of connection parameters based on the keys and the service
if 'service' not in doc:
doc['service'] = ''
if doc['service'] != '':
if not re.match('^((https?)://)?([^:/]+)(:([0-9]+))?$', doc['service']):
raise ValidationException(
'The service must of the form [http[s]://](host domain)[:(port)].', 'service')
params = makeBotoConnectParams(
doc['accessKeyId'], doc['secret'], doc['service'], doc.get('region'),
doc.get('inferCredentials'))
client = S3AssetstoreAdapter._s3Client(params)
if doc.get('readOnly'):
try:
client.head_bucket(Bucket=doc['bucket'])
except Exception:
logger.exception('S3 assetstore validation exception')
raise ValidationException(
'Unable to connect to bucket "%s".' % doc['bucket'], 'bucket')
else:
# Make sure we can write into the given bucket using boto
try:
key = '/'.join(filter(None, (doc['prefix'], 'girder_test')))
client.put_object(Bucket=doc['bucket'], Key=key, Body=b'')
client.delete_object(Bucket=doc['bucket'], Key=key)
except Exception:
logger.exception('S3 assetstore validation exception')
raise ValidationException(
'Unable to write into bucket "%s".' % doc['bucket'], 'bucket')
return doc
def __init__(self, assetstore):
"""
:param assetstore: The assetstore to act on.
"""
self.assetstore = assetstore
try:
self.chunkColl = getDbConnection(
assetstore.get('mongohost', None),
assetstore.get('replicaset', None))[assetstore['db']].chunk
except pymongo.errors.ConnectionFailure:
logger.error('Failed to connect to GridFS assetstore %s',
assetstore['db'])
self.chunkColl = 'Failed to connect'
self.unavailable = True
return
except pymongo.errors.ConfigurationError:
logger.exception('Failed to configure GridFS assetstore %s',
assetstore['db'])
self.chunkColl = 'Failed to configure'
self.unavailable = True
return
def validateInfo(doc):
"""
Makes sure the root field is a valid absolute path and is writeable.
It also conveniently update the root field replacing the initial
component by the user home directory running the server if it matches
``~`` or ``~user``.
"""
doc['root'] = os.path.expanduser(doc['root'])
if not os.path.isabs(doc['root']):
raise ValidationException('You must provide an absolute path '
'for the root directory.', 'root')
try:
mkdir(doc['root'])
except OSError:
msg = 'Could not make directory "%s".' % doc['root']
logger.exception(msg)
raise ValidationException(msg)
if not os.access(doc['root'], os.W_OK):
raise ValidationException(
'Unable to write into directory "%s".' % doc['root'])
if not doc.get('perms'):
doc['perms'] = DEFAULT_PERMS
else:
try:
perms = doc['perms']
if not isinstance(perms, int):
perms = int(doc['perms'], 8)
# Make sure that mode is still rw for user
if not perms & stat.S_IRUSR or not perms & stat.S_IWUSR:
raise ValidationException(
'File permissions must allow "rw" for user.')
doc['perms'] = perms
except ValueError:
raise ValidationException(
'File permissions must be an octal integer.')
def validateInfo(doc):
"""
Makes sure the root field is a valid absolute path and is writeable.
It also conveniently update the root field replacing the initial
component by the user home directory running the server if it matches
``~`` or ``~user``.
"""
doc['root'] = os.path.expanduser(doc['root'])
if not os.path.isabs(doc['root']):
raise ValidationException('You must provide an absolute path '
'for the root directory.', 'root')
try:
mkdir(doc['root'])
except OSError:
msg = 'Could not make directory "%s".' % doc['root']
logger.exception(msg)
raise ValidationException(msg)
if not os.access(doc['root'], os.W_OK):
raise ValidationException(
'Unable to write into directory "%s".' % doc['root'])
def deleteFile(self, file):
"""
Deletes the file from disk if it is the only File in this assetstore
with the given sha512. Imported files are not actually deleted.
"""
if file.get('imported') or 'path' not in file:
return
q = {
'sha512': file['sha512'],
'assetstoreId': self.assetstore['_id']
}
path = os.path.join(self.assetstore['root'], file['path'])
if os.path.isfile(path):
with filelock.FileLock(path + '.deleteLock'):
matching = self.model('file').find(q, limit=2, fields=[])
matchingUpload = self.model('upload').findOne(q)
if matching.count(True) == 1 and matchingUpload is None:
try:
os.unlink(path)
except Exception:
logger.exception('Failed to delete file %s' % path)
