def __init__(self, assetstore): """ :param assetstore: The assetstore to act on. """ self.assetstore = assetstore try: self.chunkColl = getDbConnection( assetstore.get('mongohost', None), assetstore.get('replicaset', None))[assetstore['db']]['chunk'] except pymongo.errors.ConnectionFailure: logger.error('Failed to connect to GridFS assetstore %s', assetstore['db']) self.chunkColl = 'Failed to connect' self.unavailable = True return except pymongo.errors.ConfigurationError: logger.exception('Failed to configure GridFS assetstore %s', assetstore['db']) self.chunkColl = 'Failed to configure' self.unavailable = True return self.chunkColl.ensure_index([ ('uuid', pymongo.ASCENDING), ('n', pymongo.ASCENDING) ], unique=True)
def validateInfo(doc): """ Makes sure the root field is a valid absolute path and is writeable. """ if 'prefix' not in doc: doc['prefix'] = '' while len(doc['prefix']) and doc['prefix'][0] == '/': doc['prefix'] = doc['prefix'][1:] while len(doc['prefix']) and doc['prefix'][-1] == '/': doc['prefix'] = doc['prefix'][:-1] if not doc.get('bucket'): raise ValidationException('Bucket must not be empty.', 'bucket') if not doc.get('secret'): raise ValidationException('Secret key must not be empty.', 'secretKey') if not doc.get('accessKeyId'): raise ValidationException('Access key ID must not be empty.', 'accessKeyId') # Make sure we can write into the given bucket using boto try: conn = boto.connect_s3(aws_access_key_id=doc['accessKeyId'], aws_secret_access_key=doc['secret']) bucket = conn.lookup(bucket_name=doc['bucket'], validate=False) testKey = boto.s3.key.Key(bucket=bucket, name=os.path.join(doc['prefix'], 'test')) testKey.set_contents_from_string('') except: logger.exception('S3 assetstore validation exception') raise ValidationException( 'Unable to write into bucket "{}".'.format(doc['bucket']), 'bucket') return doc
def _getPath(self, path): """ Given a fuse path, return the associated resource. :param path: path within the fuse. :returns: a Girder resource dictionary. """ # If asked about a file in top level directory or the top directory, # return that it doesn't exist. Other methods should handle '', # '/user', and 'collection' before calling this method. if '/' not in path.rstrip('/')[1:]: raise fuse.FuseOSError(errno.ENOENT) try: # We can't filter the resource, since that removes files' # assetstore information and users' size information. resource = path_util.lookUpPath(path.rstrip('/'), filter=False, force=True) except (path_util.NotFoundException, AccessException): raise fuse.FuseOSError(errno.ENOENT) except ValidationException: raise fuse.FuseOSError(errno.EROFS) except Exception: logger.exception('ServerFuse server internal error') raise fuse.FuseOSError(errno.EROFS) return resource # {model, document}
def __call__(self, op, path, *args, **kwargs): """ Generically allow logging and error handling for any operation. :param op: operation to perform. :param path: path within the fuse (e.g., '', '/user', '/user/<name>', etc.). """ logger.debug('-> %s %s %s', op, path, repr(args)) ret = '[exception]' try: ret = getattr(self, op)(path, *args, **kwargs) return ret except Exception as e: # Log all exceptions and then reraise them if getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): logger.debug('-- %s %r', op, e) else: logger.exception('-- %s', op) raise e finally: if op != 'read': logger.debug('<- %s %s', op, repr(ret)) else: logger.debug('<- %s (length %d) %r', op, len(ret), ret[:16])
def _ldapAuth(event): login, password = event.info['login'], event.info['password'] servers = Setting().get(PluginSettings.LDAP_SERVERS) for server in servers: try: # ldap requires a uri complete with protocol. # Append one if the user did not specify. conn = ldap.initialize(server['uri']) conn.set_option(ldap.OPT_TIMEOUT, _CONNECT_TIMEOUT) conn.set_option(ldap.OPT_NETWORK_TIMEOUT, _CONNECT_TIMEOUT) conn.bind_s(server['bindName'], server['password'], ldap.AUTH_SIMPLE) searchStr = '%s=%s' % (server['searchField'], login) results = conn.search_s(server['baseDn'], ldap.SCOPE_ONELEVEL, searchStr, _LDAP_ATTRS) if results: entry, attrs = results[0] dn = attrs['distinguishedName'][0].decode('utf8') try: conn.bind_s(dn, password, ldap.AUTH_SIMPLE) except ldap.LDAPError: # Try other LDAP servers or fall back to core auth continue finally: conn.unbind_s() user = _getLdapUser(attrs, server) if user: event.stopPropagation().preventDefault().addResponse(user) except ldap.LDAPError: logger.exception('LDAP connection exception (%s).' % server['uri']) continue
def _ldapAuth(event): login, password = event.info['login'], event.info['password'] servers = Setting().get(PluginSettings.LDAP_SERVERS) for server in servers: try: # ldap requires a uri complete with protocol. # Append one if the user did not specify. conn = ldap.initialize(server['uri']) conn.set_option(ldap.OPT_TIMEOUT, _CONNECT_TIMEOUT) conn.set_option(ldap.OPT_NETWORK_TIMEOUT, _CONNECT_TIMEOUT) conn.bind_s(server['bindName'], server['password'], ldap.AUTH_SIMPLE) searchStr = '%s=%s' % (server['searchField'], login) # Add the searchStr to the attributes, keep local scope. lattr = _LDAP_ATTRS + (server['searchField'],) results = conn.search_s(server['baseDn'], ldap.SCOPE_SUBTREE, searchStr, lattr) if results: entry, attrs = results[0] dn = attrs['distinguishedName'][0].decode('utf8') try: conn.bind_s(dn, password, ldap.AUTH_SIMPLE) except ldap.LDAPError: # Try other LDAP servers or fall back to core auth continue finally: conn.unbind_s() user = _getLdapUser(attrs, server) if user: event.stopPropagation().preventDefault().addResponse(user) except ldap.LDAPError: logger.exception('LDAP connection exception (%s).' % server['uri']) continue
def cache_tile_frames_job(job): from girder_jobs.constants import JobStatus from girder_jobs.models.job import Job from girder_large_image.models.image_item import ImageItem from girder import logger kwargs = job['kwargs'] item = ImageItem().load(kwargs.pop('itemId'), force=True) job = Job().updateJob(job, log='Started caching tile frames\n', status=JobStatus.RUNNING) try: for entry in kwargs.get('tileFramesList'): job = Job().load(job['_id'], force=True) if job['status'] == JobStatus.CANCELED: return job = Job().updateJob(job, log='Caching %r\n' % entry) ImageItem().tileFrames(item, checkAndCreate=True, **entry) job = Job().updateJob(job, log='Finished caching tile frames\n', status=JobStatus.SUCCESS) except Exception as exc: logger.exception('Failed caching tile frames') job = Job().updateJob(job, log='Failed caching tile frames (%s)\n' % exc, status=JobStatus.ERROR)
def CLIListEntrypoint(cli_list_spec_file=None): if cli_list_spec_file is None: cli_list_spec_file = os.path.join(os.getcwd(), 'slicer_cli_list.json') # Parse CLI List spec with open(cli_list_spec_file, 'rt') as f: cli_list_spec = json.load(f) # create command-line argument parser cmdparser = argparse.ArgumentParser( formatter_class=_MultilineHelpFormatter) # add --cli_list cmdparser.add_argument( '--list_cli', action=_make_print_cli_list_spec_action(cli_list_spec_file), help='Prints the json file containing the list of CLIs present') # add cl-rel-path argument cmdparser.add_argument('cli', help='CLI to run', metavar='<cli>', choices=cli_list_spec.keys()) args = cmdparser.parse_args(sys.argv[1:2]) args.cli = os.path.normpath(args.cli) if cli_list_spec[args.cli]['type'] == 'python': script_file = os.path.join(args.cli, os.path.basename(args.cli) + '.py') # python <cli-rel-path>/<cli-name>.py [<args>] output_code = subprocess.call([sys.executable, script_file] + sys.argv[2:]) elif cli_list_spec[args.cli]['type'] == 'cxx': script_file = os.path.join('.', args.cli, os.path.basename(args.cli)) if os.path.isfile(script_file): # ./<cli-rel-path>/<cli-name> [<args>] output_code = subprocess.call([script_file] + sys.argv[2:]) else: # assumes parent dir of CLI executable is in ${PATH} output_code = subprocess.call([os.path.basename(args.cli)] + sys.argv[2:]) else: logger.exception('CLIs of type %s are not supported', cli_list_spec[args.cli]['type']) raise Exception('CLIs of type %s are not supported', cli_list_spec[args.cli]['type']) return output_code
def createThumbnailsJobTask(item, spec): """ For an individual item, check or create all of the appropriate thumbnails. :param item: the image item. :param spec: a list of thumbnail specifications. :returns: a dictionary with the total status of the thumbnail job. """ status = {'checked': 0, 'created': 0, 'failed': 0} for entry in spec: try: if entry.get('imageKey'): result = ImageItem().getAssociatedImage(item, checkAndCreate=True, **entry) else: result = ImageItem().getThumbnail(item, checkAndCreate=True, **entry) status['checked' if result is True else 'created'] += 1 except TileGeneralException as exc: status['failed'] += 1 status['lastFailed'] = str(item['_id']) logger.info('Failed to get thumbnail for item %s: %r' % (item['_id'], exc)) except AttributeError: raise except Exception: status['failed'] += 1 status['lastFailed'] = str(item['_id']) logger.exception( 'Unexpected exception when trying to create a thumbnail for item %s' % item['_id']) return status
def endpointDecorator(self, *path, **params): _setCommonCORSHeaders() cherrypy.lib.caching.expires(0) cherrypy.request.girderRequestUid = str(uuid.uuid4()) setResponseHeader('Girder-Request-Uid', cherrypy.request.girderRequestUid) try: val = fun(self, path, params) # If this is a partial response, we set the status appropriately if 'Content-Range' in cherrypy.response.headers: cherrypy.response.status = 206 val = _mongoCursorToList(val) if callable(val): # If the endpoint returned anything callable (function, # lambda, functools.partial), we assume it's a generator # function for a streaming response. cherrypy.response.stream = True _logRestRequest(self, path, params) return val() if isinstance(val, cherrypy.lib.file_generator): # Don't do any post-processing of static files return val if isinstance(val, types.GeneratorType): val = list(val) except RestException as e: val = _handleRestException(e) except AccessException as e: val = _handleAccessException(e) except GirderException as e: val = _handleGirderException(e) except ValidationException as e: val = _handleValidationException(e) except cherrypy.HTTPRedirect: raise except Exception: # These are unexpected failures; send a 500 status logger.exception('500 Error') cherrypy.response.status = 500 val = dict(type='internal', uid=cherrypy.request.girderRequestUid) if config.getConfig()['server']['mode'] == 'production': # Sanitize errors in production mode val['message'] = 'An unexpected error occurred on the server.' else: # Provide error details in non-production modes t, value, tb = sys.exc_info() val['message'] = '%s: %s' % (t.__name__, repr(value)) val['trace'] = traceback.extract_tb(tb) resp = _createResponse(val) _logRestRequest(self, path, params) return resp
def _addOptionalInputParamsToContainerArgs(opt_input_params, containerArgs, hargs): for param in opt_input_params: if param.longflag: curFlag = param.longflag elif param.flag: curFlag = param.flag else: continue if _is_on_girder(param) and param.identifier() in hargs: curValue = "$input{%s}" % param.identifier() elif param.identifier() in hargs['params']: try: curValue = _getParamCommandLineValue( param, hargs['params'][param.identifier()]) except Exception: logger.exception( 'Error: Parameter value is not in json.dumps format\n' ' Parameter name = %r\n Parameter type = %r\n' ' Value passed = %r', param.identifier(), param.typ, hargs['params'][param.identifier()]) raise else: continue containerArgs.append(curFlag) containerArgs.append(curValue)
def _addAssociatedImage(self, largeImagePath, directoryNum): """ Check if the specfied TIFF directory contains a non-tiled image with a sensible image description that can be used as an ID. If so, and if the image isn't too large, add this image as an associated image. :param largeImagePath: path to the TIFF file. :param directoryNum: libtiff directory number of the image. """ try: associated = TiledTiffDirectory(largeImagePath, directoryNum, False) id = associated._tiffInfo.get('imagedescription').strip().split( None, 1)[0].lower() if not isinstance(id, six.text_type): id = id.decode('utf8') # Only use this as an associated image if the parsed id is # a reasonable length, alphanumeric characters, and the # image isn't too large. if (id.isalnum() and len(id) > 3 and len(id) <= 20 and associated._pixelInfo['width'] <= 8192 and associated._pixelInfo['height'] <= 8192): self._associatedImages[id] = associated._tiffFile.read_image() except (TiffException, AttributeError): # If we can't validate or read an associated image or it has no # useful imagedescription, fail quietly without adding an # associated image. pass except Exception: # If we fail for other reasons, don't raise an exception, but log # what happened. logger.exception( 'Could not use non-tiled TIFF image as an associated image.')
def updateAnnotation(self, annotation, params): # Set the response time limit to a very long value setResponseTimeLimit(86400) user = self.getCurrentUser() item = Item().load(annotation.get('itemId'), force=True) if item is not None: Item().requireAccess(item, user=user, level=AccessType.WRITE) # If we have a content length, then we have replacement JSON. If # elements are not included, don't replace them returnElements = True if cherrypy.request.body.length: oldElements = annotation.get('annotation', {}).get('elements') annotation['annotation'] = self.getBodyJson() if 'elements' not in annotation['annotation'] and oldElements: annotation['annotation']['elements'] = oldElements returnElements = False if params.get('itemId'): newitem = Item().load(params['itemId'], force=True) Item().requireAccess(newitem, user=user, level=AccessType.WRITE) annotation['itemId'] = newitem['_id'] try: annotation = Annotation().updateAnnotation(annotation, updateUser=user) except ValidationException as exc: logger.exception('Failed to validate annotation') raise RestException( "Validation Error: JSON doesn't follow schema (%r)." % (exc.args, )) if not returnElements and 'elements' in annotation['annotation']: del annotation['annotation']['elements'] return annotation
def getDockerOutput(imgName, command, client): """ Data from each docker image is collected by executing the equivalent of a docker run <imgName> <command/args> and collecting the output to standard output :param imgName: The name of the docker image :param command: The commands/ arguments to be passed to the docker image :param client: The docker python client """ cont = None try: cont = client.containers.create(image=imgName, command=command) cont.start() ret_code = cont.wait() if isinstance(ret_code, dict): ret_code = ret_code['StatusCode'] logs = cont.logs(stdout=True, stderr=False, stream=False) cont.remove() except Exception as err: if cont: try: cont.remove() except Exception: pass logger.exception('Attempt to docker run %s %s failed', imgName, command) raise DockerImageError( 'Attempt to docker run %s %s failed ' % (imgName, command) + str(err), imgName) if ret_code != 0: raise DockerImageError( 'Attempt to docker run %s %s failed' % (imgName, command), imgName) return logs
def _parseParamValue(param, value, user, token): if isinstance(value, six.binary_type): value = value.decode('utf8') param_id = param.identifier() if is_on_girder(param): girder_type = SLICER_TYPE_TO_GIRDER_MODEL_MAP[param.typ] curModel = ModelImporter.model(girder_type) loaded = curModel.load(value, level=AccessType.READ, user=user) if not loaded: raise RestException('Invalid %s id (%s).' % (curModel.name, str(value))) return loaded try: if param.isVector(): return '%s' % ', '.join(map(str, json.loads(value))) elif param.typ in OPENAPI_DIRECT_TYPES or param.typ == 'string-enumeration': return str(value) else: # json return str(json.loads(value)) except json.JSONDecodeError: msg = 'Error: Parameter value is not in json.dumps format\n' \ ' Parameter name = %r\n Parameter type = %r\n' \ ' Value passed = %r' % (param_id, param.typ, value) logger.exception(msg) raise RestException(msg)
def createThumb(self, file, challenge, params): self.requireParams('size', params) size = int(params['size']) user = self.getCurrentUser() if challenge.get('thumbnailSourceId') != file['_id']: challenge['thumbnails'] = [] challenge['thumbnailSourceId'] = file['_id'] i = 0 for thumbnail in challenge['thumbnails']: if thumbnail['size'] == size: return File().filter( File().load(thumbnail['fileId'], force=True), user) elif size < thumbnail['size']: break i += 1 try: newThumb = createThumbnail( width=size, height=size, crop=True, fileId=file['_id'], attachToType='item', attachToId=file['itemId']) except IOError: logger.exception('Thumbnail creation IOError') raise RestException('Could not create thumbnail from the file.') challenge['thumbnails'].insert(i, { 'size': size, 'fileId': newThumb['_id'] }) Challenge().save(challenge) return File().filter(newThumb, user)
def getCliData(name, client, jobModel, job): try: cli_dict = getDockerOutput(name, '--list_cli', client) # contains nested dict # {<cliname>:{type:<type>}} if isinstance(cli_dict, six.binary_type): cli_dict = cli_dict.decode('utf8') cli_dict = json.loads(cli_dict) for key, info in six.iteritems(cli_dict): desc_type = info.get('desc-type', 'xml') cli_desc = getDockerOutput(name, '%s --%s' % (key, desc_type), client) if isinstance(cli_desc, six.binary_type): cli_desc = cli_desc.decode('utf8') cli_dict[key][desc_type] = cli_desc jobModel.updateJob( job, log='Got image %s, cli %s metadata\n' % (name, key), status=JobStatus.RUNNING, ) return cli_dict except Exception as err: logger.exception('Error getting %s cli data from image', name) raise DockerImageError('Error getting %s cli data from image ' % (name) + str(err))
def createThumb(self, file, challenge, params): self.requireParams('size', params) size = int(params['size']) user = self.getCurrentUser() if challenge.get('thumbnailSourceId') != file['_id']: challenge['thumbnails'] = [] challenge['thumbnailSourceId'] = file['_id'] i = 0 for thumbnail in challenge['thumbnails']: if thumbnail['size'] == size: return self.model('file').filter( self.model('file').load(thumbnail['fileId'], force=True), user) elif size < thumbnail['size']: break i += 1 try: newThumb = createThumbnail( width=size, height=size, crop=True, fileId=file['_id'], attachToType='item', attachToId=file['itemId']) except IOError: logger.exception('Thumbnail creation IOError') raise RestException('Could not create thumbnail from the file.') challenge['thumbnails'].insert(i, { 'size': size, 'fileId': newThumb['_id'] }) self.model('challenge', 'challenge').save(challenge) return self.model('file').filter(newThumb, user)
def _getPath(self, path): """ Given a fuse path, return the associated resource. :param path: path within the fuse. :returns: a Girder resource dictionary. """ # If asked about a file in top level directory or the top directory, # return that it doesn't exist. Other methods should handle '', # '/user', and 'collection' before calling this method. if '/' not in path.rstrip('/')[1:]: raise fuse.FuseOSError(errno.ENOENT) try: # We can't filter the resource, since that removes files' # assetstore information and users' size information. resource = path_util.lookUpPath( path.rstrip('/'), filter=False, force=True) except (path_util.NotFoundException, AccessException): raise fuse.FuseOSError(errno.ENOENT) except ValidationException: raise fuse.FuseOSError(errno.EROFS) except Exception: logger.exception('ServerFuse server internal error') raise fuse.FuseOSError(errno.EROFS) return resource # {model, document}
def updateAnnotation(self, annotation, params): user = self.getCurrentUser() item = self.model('item').load(annotation.get('itemId'), force=True) if item is not None: self.model('item').requireAccess(item, user=user, level=AccessType.WRITE) # If we have a content length, then we have replacement JSON. if cherrypy.request.body.length: annotation['annotation'] = self.getBodyJson() if params.get('itemId'): newitem = self.model('item').load(params['itemId'], force=True) self.model('item').requireAccess(newitem, user=user, level=AccessType.WRITE) annotation['itemId'] = newitem['_id'] try: self.model('annotation', 'large_image').updateAnnotation(annotation, updateUser=user) except ValidationException as exc: logger.exception('Failed to validate annotation') raise RestException( 'Validation Error: JSON doesn\'t follow schema (%s).' % (exc.message, )) return annotation
def __init__(self, name): try: if isinstance(name, string_types): imageKey = DockerImage.getHashKey(name) self.data = {} self.data[DockerImage.imageName] = name self.data[DockerImage.cli_dict] = {} self.data[DockerImage.imageHash] = imageKey self.hash = imageKey self.name = name # TODO check/validate schema of dict elif isinstance(name, dict): jsonschema.validate(name, DockerImageStructure.ImageSchema) self.data = name.copy() self.name = self.data[DockerImage.imageName] self.hash = DockerImage.getHashKey(self.name) else: raise DockerImageError( 'Image should be a string, or dict' ' could not add the image', 'bad init val') except Exception as err: logger.exception('Could not initialize docker image %r', name) raise DockerImageError( 'Could not initialize instance of Docker Image \n' + str(err))
def getCliData(name, client, img, jobModel, job): try: if isinstance(client, docker.DockerClient) and isinstance( img, DockerImage): cli_dict = getDockerOutput(name, '--list_cli', client) # contains nested dict # {<cliname>:{type:<type>}} if isinstance(cli_dict, six.binary_type): cli_dict = cli_dict.decode('utf8') cli_dict = json.loads(cli_dict) for (key, val) in six.iteritems(cli_dict): cli_xml = getDockerOutput(name, '%s --xml' % key, client) if isinstance(cli_xml, six.binary_type): cli_xml = cli_xml.decode('utf8') cli_dict[key][DockerImage.xml] = cli_xml jobModel.updateJob( job, log='Got image %s, cli %s metadata\n' % (name, key), status=JobStatus.RUNNING, ) img.addCLI(key, cli_dict[key]) return cli_dict except Exception as err: logger.exception('Error getting %s cli data from image %s', name, img) raise DockerImageError('Error getting %s cli data from image %s ' % (name, img) + str(err))
def gcs_save_record(self, data: dict): """ https://cloud.google.com/pubsub/docs/push#receiving_messages """ try: payload = GCSPushNotificationPayload(**data) GCSNotificationRecord().create(payload.message) if payload.message.attributes.eventType == 'OBJECT_FINALIZE': # This is a create notification store = Assetstore().findOne({ 'type': 2, # S3 type AssetstoreRuleMarker: { '$exists': True }, 'bucket': payload.message.attributes.bucketId, # The only viable GSC Service string 'service': 'https://storage.googleapis.com', }) if store is not None: rule = NotificationRouterRule( **store[AssetstoreRuleMarker]) mountRoot = Folder().findOne( {'_id': ObjectId(rule.folderId)}) BucketNotification.processNotification( store, mountRoot, payload.message.attributes.objectId) except Exception as err: # exceptions must be swallowed to prevent pub/sub queue backups # message loss is always easily recoverable by running a manual # import through the admin console. logger.exception(f'Failed to process GCS notification {err}') return "done"
def trigger(eventName, info=None): """ Fire an event with the given name. All listeners bound on that name will be called until they are exhausted or one of the handlers calls the stopPropagation() method on the event. :param eventName: The name that identifies the event. :type eventName: str :param info: The info argument to pass to the handler function. The type of this argument is opaque, and can be anything. :return """ global _mapping e = Event(eventName, info) for handlerName, handler in _mapping.get(eventName, {}).iteritems(): e.currentHandlerName = handlerName try: handler(e) except: logger.exception('In handler "{}" for event "{}":' .format(handlerName, eventName)) if e.propagate is False: break return e
def _handleGirderException(e): # Handle general Girder exceptions logger.exception('500 Error') cherrypy.response.status = 500 val = {'message': e.message, 'type': 'girder'} if e.identifier is not None: val['identifier'] = e.identifier return val
def createAnnotation(self, item, params): try: return self.model('annotation', 'large_image').createAnnotation( item, self.getCurrentUser(), self.getBodyJson()) except ValidationException as exc: logger.exception('Failed to validate annotation') raise RestException( 'Validation Error: JSON doesn\'t follow schema (%s).' % (exc.message, ))
def createAnnotation(self, item, params): try: return self.model('annotation', 'large_image').createAnnotation( item, self.getCurrentUser(), self.getBodyJson()) except ValidationException as exc: logger.exception('Failed to validate annotation') raise RestException( 'Validation Error: JSON doesn\'t follow schema (%s).' % ( exc.message, ))
def _handleAccessException(e): # Permission exceptions should throw a 401 or 403, depending # on whether the user is logged in or not if getCurrentUser() is None: cherrypy.response.status = 401 else: cherrypy.response.status = 403 logger.exception('403 Error') return {'message': e.message, 'type': 'access'}
def undoFunction(): try: restResource.removeRoute('POST', restRunPath, cliRunHandler) if registerNamedRoute: restResource.removeRoute('POST', restNamedRunPath, cliRunHandler) delattr(restResource, cliRunHandlerName) except Exception: logger.exception('Failed to remove route')
def createColormap(self, name, public, colormap, labels): try: return Colormap().createColormap(self.getCurrentUser(), colormap, name, labels, public) except ValidationException as exc: logger.exception('Failed to validate colormap') raise RestException( "Validation Error: JSON doesn\'t follow schema (%r)." % (exc.args, ))
def send_new_user_email(event): try: info = event.info email = info.get('email') brandName = Setting().get(SettingKey.BRAND_NAME) rendered = renderTemplate('welcome.mako') sendMail(f'Welcome to {brandName}', rendered, [email]) except Exception: logger.exception("Failed to send new user email")
def validateInfo(doc): """ Makes sure the root field is a valid absolute path and is writeable. """ if 'prefix' not in doc: doc['prefix'] = '' # remove slashes from front and back of the prefix doc['prefix'] = doc['prefix'].strip('/') if not doc.get('bucket'): raise ValidationException('Bucket must not be empty.', 'bucket') if not doc.get('readOnly'): if not doc.get('secret'): raise ValidationException('Secret key must not be empty.', 'secret') if not doc.get('accessKeyId'): raise ValidationException('Access key ID must not be empty.', 'accessKeyId') # construct a set of connection parameters based on the keys and the # service if 'service' not in doc: doc['service'] = '' if doc['service'] != '': service = re.match("^((https?)://)?([^:/]+)(:([0-9]+))?$", doc['service']) if not service: raise ValidationException( 'The service must of the form [http[s]://](host domain)' '[:(port)].', 'service') doc['botoConnect'] = makeBotoConnectParams(doc['accessKeyId'], doc['secret'], doc['service']) # Make sure we can write into the given bucket using boto conn = botoConnectS3(doc['botoConnect']) if doc.get('readOnly'): try: conn.get_bucket(bucket_name=doc['bucket'], validate=True) except Exception: logger.exception('S3 assetstore validation exception') raise ValidationException( 'Unable to connect to bucket "%s".' % doc['bucket'], 'bucket') else: try: bucket = conn.get_bucket(bucket_name=doc['bucket'], validate=True) testKey = boto.s3.key.Key(bucket=bucket, name='/'.join( filter(None, (doc['prefix'], 'test')))) testKey.set_contents_from_string('') except Exception: logger.exception('S3 assetstore validation exception') raise ValidationException( 'Unable to write into bucket "%s".' % doc['bucket'], 'bucket') return doc
def createAnnotation(self, item, params): try: return Annotation().createAnnotation(item, self.getCurrentUser(), self.getBodyJson()) except ValidationException as exc: logger.exception('Failed to validate annotation') raise RestException( "Validation Error: JSON doesn't follow schema (%r)." % (exc.args, ))
def import_recursive(job): try: root = job['kwargs']['root'] token = job['kwargs']['token'] user = User().load(job['userId'], force=True) children = list(Folder().childFolders(root, 'collection', user=user)) count = len(children) progress = 0 job = Job().updateJob(job, log='Started TCGA import\n', status=JobStatus.RUNNING, progressCurrent=progress, progressTotal=count) logger.info('Starting recursive TCGA import') for child in children: progress += 1 try: msg = 'Importing "%s"' % child.get('name', '') job = Job().updateJob(job, log=msg, progressMessage=msg + '\n', progressCurrent=progress) logger.debug(msg) Cohort().importDocument(child, recurse=True, user=user, token=token, job=job) job = Job().load(id=job['_id'], force=True) # handle any request to stop execution if (not job or job['status'] in (JobStatus.CANCELED, JobStatus.ERROR)): logger.info('TCGA import job halted with') return except ValidationException: logger.warning('Failed to import %s' % child.get('name', '')) logger.info('Starting recursive TCGA import') job = Job().updateJob(job, log='Finished TCGA import\n', status=JobStatus.SUCCESS, progressCurrent=count, progressMessage='Finished TCGA import') except Exception as e: logger.exception('Importing TCGA failed with %s' % str(e)) job = Job().updateJob(job, log='Import failed with %s\n' % str(e), status=JobStatus.ERROR)
def endpointDecorator(self, *path, **params): _setCommonCORSHeaders() cherrypy.lib.caching.expires(0) try: val = fun(self, path, params) # If this is a partial response, we set the status appropriately if 'Content-Range' in cherrypy.response.headers: cherrypy.response.status = 206 val = _mongoCursorToList(val) if callable(val): # If the endpoint returned anything callable (function, # lambda, functools.partial), we assume it's a generator # function for a streaming response. cherrypy.response.stream = True _logRestRequest(self, path, params) return val() if isinstance(val, cherrypy.lib.file_generator): # Don't do any post-processing of static files return val if isinstance(val, types.GeneratorType): val = list(val) except RestException as e: val = _handleRestException(e) except AccessException as e: val = _handleAccessException(e) except GirderException as e: val = _handleGirderException(e) except ValidationException as e: val = _handleValidationException(e) except cherrypy.HTTPRedirect: raise except Exception: # These are unexpected failures; send a 500 status logger.exception('500 Error') cherrypy.response.status = 500 t, value, tb = sys.exc_info() val = {'message': '%s: %s' % (t.__name__, repr(value)), 'type': 'internal'} curConfig = config.getConfig() if curConfig['server']['mode'] != 'production': # Unless we are in production mode, send a traceback too val['trace'] = traceback.extract_tb(tb) resp = _createResponse(val) _logRestRequest(self, path, params) return resp
def wrapped(*args, **kwargs): try: return fun(*args, **kwargs) except ResourcePathNotFound: return paramiko.SFTP_NO_SUCH_FILE except ValidationException: return paramiko.SFTP_FAILURE except AccessException: return paramiko.SFTP_PERMISSION_DENIED except Exception: logger.exception('SFTP server internal error') return paramiko.SFTP_FAILURE
def validateInfo(doc): """ Makes sure the root field is a valid absolute path and is writeable. """ if 'prefix' not in doc: doc['prefix'] = '' # remove slashes from front and back of the prefix doc['prefix'] = doc['prefix'].strip('/') if not doc.get('bucket'): raise ValidationException('Bucket must not be empty.', 'bucket') if not doc.get('readOnly'): if not doc.get('secret'): raise ValidationException( 'Secret key must not be empty.', 'secret') if not doc.get('accessKeyId'): raise ValidationException( 'Access key ID must not be empty.', 'accessKeyId') # construct a set of connection parameters based on the keys and the # service if 'service' not in doc: doc['service'] = '' if doc['service'] != '': service = re.match("^((https?)://)?([^:/]+)(:([0-9]+))?$", doc['service']) if not service: raise ValidationException( 'The service must of the form [http[s]://](host domain)' '[:(port)].', 'service') doc['botoConnect'] = makeBotoConnectParams( doc['accessKeyId'], doc['secret'], doc['service']) # Make sure we can write into the given bucket using boto conn = botoConnectS3(doc['botoConnect']) if doc.get('readOnly'): try: conn.get_bucket(bucket_name=doc['bucket'], validate=True) except Exception: logger.exception('S3 assetstore validation exception') raise ValidationException('Unable to connect to bucket "%s".' % doc['bucket'], 'bucket') else: try: bucket = conn.get_bucket(bucket_name=doc['bucket'], validate=True) testKey = boto.s3.key.Key( bucket=bucket, name='/'.join( filter(None, (doc['prefix'], 'test')))) testKey.set_contents_from_string('') except Exception: logger.exception('S3 assetstore validation exception') raise ValidationException('Unable to write into bucket "%s".' % doc['bucket'], 'bucket') return doc
def _s3Client(connectParams): try: client = boto3.client('s3', **connectParams) if 'googleapis' in urllib.parse.urlparse(connectParams.get( 'endpoint_url', '')).netloc.split('.'): client.meta.events.unregister( 'before-parameter-build.s3.ListObjects', botocore.handlers.set_list_objects_encoding_type_url) client._useGoogleAccessId = True return client except Exception: logger.exception('S3 assetstore validation exception') raise ValidationException('Unable to connect to S3 assetstore')
def botoConnectS3(connectParams): """ Connect to the S3 server, throwing an appropriate exception if we fail. :param connectParams: a dictionary of paramters to use in the connection. :returns: the boto connection object. """ try: conn = boto.connect_s3(calling_format=BotoCallingFormat(), **connectParams) except Exception: logger.exception('S3 assetstore validation exception') raise ValidationException('Unable to connect to S3 assetstore') return conn
def __init__(self, assetstore): self.assetstore = assetstore # If we can't create the temp directory, the assetstore still needs to # be initialized so that it can be deleted or modified. The validation # prevents invalid new assetstores from being created, so this only # happens to existing assetstores that no longer can access their temp # directories. self.tempDir = os.path.join(assetstore['root'], 'temp') if not os.path.exists(self.tempDir): try: os.makedirs(self.tempDir) except OSError: logger.exception('Failed to create filesystem assetstore ' 'directories {}'.format(self.tempDir))
def endpointDecorator(self, *args, **kwargs): # Note that the cyclomatic complexity of this function crosses our # flake8 configuration threshold. Because it is largely exception # handling, I think that breaking it into smaller functions actually # reduces readability and maintainability. To work around this, some # simple branches have been marked to be skipped in the cyclomatic # analysis. _setCommonCORSHeaders() cherrypy.lib.caching.expires(0) try: val = fun(self, args, kwargs) # If this is a partial response, we set the status appropriately if 'Content-Range' in cherrypy.response.headers: cherrypy.response.status = 206 if callable(val): # If the endpoint returned anything callable (function, # lambda, functools.partial), we assume it's a generator # function for a streaming response. cherrypy.response.stream = True return val() if isinstance(val, cherrypy.lib.file_generator): # Don't do any post-processing of static files return val except RestException as e: val = _handleRestException(e) except AccessException as e: val = _handleAccessException(e) except GirderException as e: val = _handleGirderException(e) except ValidationException as e: val = _handleValidationException(e) except cherrypy.HTTPRedirect: raise except Exception: # These are unexpected failures; send a 500 status logger.exception('500 Error') cherrypy.response.status = 500 t, value, tb = sys.exc_info() val = {'message': '%s: %s' % (t.__name__, repr(value)), 'type': 'internal'} curConfig = config.getConfig() if curConfig['server']['mode'] != 'production': # Unless we are in production mode, send a traceback too val['trace'] = traceback.extract_tb(tb) return _createResponse(val)
def botoConnectS3(connectParams): """ Connect to the S3 server, throwing an appropriate exception if we fail. :param connectParams: a dictionary of parameters to use in the connection. :returns: the boto connection object. """ if 'anon' not in connectParams or not connectParams['anon']: connectParams = connectParams.copy() connectParams['calling_format'] = BotoCallingFormat() try: return boto.connect_s3(**connectParams) except Exception: logger.exception('S3 assetstore validation exception') raise ValidationException('Unable to connect to S3 assetstore')
def _loadModel(model, module, plugin): className = camelcase(model) try: imported = importlib.import_module(module) except ImportError: logger.exception('Could not load model "%s".' % module) raise try: constructor = getattr(imported, className) except AttributeError: raise Exception('Incorrect model class name "%s" for model "%s".' % ( className, module)) _modelInstances[plugin][model] = constructor()
def capacityInfo(self): """ For filesystem assetstores, we just need to report the free and total space on the filesystem where the assetstore lives. """ try: usage = psutil.disk_usage(self.assetstore['root']) return {'free': usage.free, 'total': usage.total} except OSError: logger.exception( 'Failed to get disk usage of %s' % self.assetstore['root']) # If psutil.disk_usage fails or we can't query the assetstore's root # directory, just report nothing regarding disk capacity return { # pragma: no cover 'free': None, 'total': None }
def _loadModel(model, module, plugin): global _modelInstances className = camelcase(model) try: imported = importlib.import_module(module) except ImportError: logger.exception('Could not load model "{}".'.format(module)) raise try: constructor = getattr(imported, className) except AttributeError: # pragma: no cover raise Exception('Incorrect model class name "{}" for model "{}".' .format(className, module)) _modelInstances[plugin][model] = constructor()
def endpointDecorator(self, *args, **kwargs): try: val = fun(self, args, kwargs) if isinstance(val, types.FunctionType): # If the endpoint returned a function, we assume it's a # generator function for a streaming response. cherrypy.response.stream = True return val() except RestException as e: # Handle all user-error exceptions from the rest layer cherrypy.response.status = e.code val = {'message': e.message, 'type': 'rest'} if e.extra is not None: val['extra'] = e.extra except AccessException as e: # Permission exceptions should throw a 401 or 403, depending # on whether the user is logged in or not if self.getCurrentUser() is None: cherrypy.response.status = 401 else: cherrypy.response.status = 403 logger.exception('403 Error') val = {'message': e.message, 'type': 'access'} except ValidationException as e: cherrypy.response.status = 400 val = {'message': e.message, 'type': 'validation'} if e.field is not None: val['field'] = e.field except cherrypy.HTTPRedirect: raise except: # These are unexpected failures; send a 500 status logger.exception('500 Error') cherrypy.response.status = 500 t, value, tb = sys.exc_info() val = {'message': '%s: %s' % (t.__name__, str(value)), 'type': 'internal'} curConfig = config.getConfig() if curConfig['server']['mode'] != 'production': # Unless we are in production mode, send a traceback too val['trace'] = traceback.extract_tb(tb) return _createResponse(val)
def __init__(self, assetstore): super(FilesystemAssetstoreAdapter, self).__init__(assetstore) # If we can't create the temp directory, the assetstore still needs to # be initialized so that it can be deleted or modified. The validation # prevents invalid new assetstores from being created, so this only # happens to existing assetstores that no longer can access their temp # directories. self.tempDir = os.path.join(self.assetstore['root'], 'temp') try: mkdir(self.tempDir) except OSError: self.unavailable = True logger.exception('Failed to create filesystem assetstore ' 'directories %s' % self.tempDir) if not os.access(self.assetstore['root'], os.W_OK): self.unavailable = True logger.error('Could not write to assetstore root: %s', self.assetstore['root'])
def __init__(self, assetstore): """ :param assetstore: The assetstore to act on. """ super(GridFsAssetstoreAdapter, self).__init__(assetstore) recent = False try: # Guard in case the connectionArgs is unhashable key = (self.assetstore.get('mongohost'), self.assetstore.get('replicaset'), self.assetstore.get('shard')) if key in _recentConnections: recent = (time.time() - _recentConnections[key]['created'] < RECENT_CONNECTION_CACHE_TIME) except TypeError: key = None try: # MongoClient automatically reuses connections from a pool, but we # want to avoid redoing ensureChunkIndices each time we get such a # connection. client = getDbConnection(self.assetstore.get('mongohost'), self.assetstore.get('replicaset'), quiet=recent) self.chunkColl = MongoProxy(client[self.assetstore['db']].chunk) if not recent: _ensureChunkIndices(self.chunkColl) if self.assetstore.get('shard') == 'auto': _setupSharding(self.chunkColl) if key is not None: if len(_recentConnections) >= RECENT_CONNECTION_CACHE_MAX_SIZE: _recentConnections.clear() _recentConnections[key] = { 'created': time.time() } except pymongo.errors.ConnectionFailure: logger.error('Failed to connect to GridFS assetstore %s', self.assetstore['db']) self.chunkColl = 'Failed to connect' self.unavailable = True except pymongo.errors.ConfigurationError: logger.exception('Failed to configure GridFS assetstore %s', self.assetstore['db']) self.chunkColl = 'Failed to configure' self.unavailable = True
def run(self): """ Loops over all queued events. If the queue is empty, this thread gets put to sleep until someone calls trigger() on it with a new event to dispatch. """ print(TerminalColor.info('Started asynchronous event manager thread.')) while not self.terminate: eventName, info, callback = self.eventQueue.get(block=True) try: event = trigger(eventName, info, async=True) if isinstance(callback, types.FunctionType): callback(event) except Exception: logger.exception('In handler for event "%s":' % eventName) pass # Must continue the event loop even if handler failed print(TerminalColor.info('Stopped asynchronous event manager thread.'))
def validateInfo(doc): """ Makes sure the root field is a valid absolute path and is writeable. """ if 'prefix' not in doc: doc['prefix'] = '' # remove slashes from front and back of the prefix doc['prefix'] = doc['prefix'].strip('/') if not doc.get('bucket'): raise ValidationException('Bucket must not be empty.', 'bucket') # construct a set of connection parameters based on the keys and the service if 'service' not in doc: doc['service'] = '' if doc['service'] != '': if not re.match('^((https?)://)?([^:/]+)(:([0-9]+))?$', doc['service']): raise ValidationException( 'The service must of the form [http[s]://](host domain)[:(port)].', 'service') params = makeBotoConnectParams( doc['accessKeyId'], doc['secret'], doc['service'], doc.get('region'), doc.get('inferCredentials')) client = S3AssetstoreAdapter._s3Client(params) if doc.get('readOnly'): try: client.head_bucket(Bucket=doc['bucket']) except Exception: logger.exception('S3 assetstore validation exception') raise ValidationException( 'Unable to connect to bucket "%s".' % doc['bucket'], 'bucket') else: # Make sure we can write into the given bucket using boto try: key = '/'.join(filter(None, (doc['prefix'], 'girder_test'))) client.put_object(Bucket=doc['bucket'], Key=key, Body=b'') client.delete_object(Bucket=doc['bucket'], Key=key) except Exception: logger.exception('S3 assetstore validation exception') raise ValidationException( 'Unable to write into bucket "%s".' % doc['bucket'], 'bucket') return doc
def __init__(self, assetstore): """ :param assetstore: The assetstore to act on. """ self.assetstore = assetstore try: self.chunkColl = getDbConnection( assetstore.get('mongohost', None), assetstore.get('replicaset', None))[assetstore['db']].chunk except pymongo.errors.ConnectionFailure: logger.error('Failed to connect to GridFS assetstore %s', assetstore['db']) self.chunkColl = 'Failed to connect' self.unavailable = True return except pymongo.errors.ConfigurationError: logger.exception('Failed to configure GridFS assetstore %s', assetstore['db']) self.chunkColl = 'Failed to configure' self.unavailable = True return
def validateInfo(doc): """ Makes sure the root field is a valid absolute path and is writeable. It also conveniently update the root field replacing the initial component by the user home directory running the server if it matches ``~`` or ``~user``. """ doc['root'] = os.path.expanduser(doc['root']) if not os.path.isabs(doc['root']): raise ValidationException('You must provide an absolute path ' 'for the root directory.', 'root') try: mkdir(doc['root']) except OSError: msg = 'Could not make directory "%s".' % doc['root'] logger.exception(msg) raise ValidationException(msg) if not os.access(doc['root'], os.W_OK): raise ValidationException( 'Unable to write into directory "%s".' % doc['root']) if not doc.get('perms'): doc['perms'] = DEFAULT_PERMS else: try: perms = doc['perms'] if not isinstance(perms, int): perms = int(doc['perms'], 8) # Make sure that mode is still rw for user if not perms & stat.S_IRUSR or not perms & stat.S_IWUSR: raise ValidationException( 'File permissions must allow "rw" for user.') doc['perms'] = perms except ValueError: raise ValidationException( 'File permissions must be an octal integer.')
def validateInfo(doc): """ Makes sure the root field is a valid absolute path and is writeable. It also conveniently update the root field replacing the initial component by the user home directory running the server if it matches ``~`` or ``~user``. """ doc['root'] = os.path.expanduser(doc['root']) if not os.path.isabs(doc['root']): raise ValidationException('You must provide an absolute path ' 'for the root directory.', 'root') try: mkdir(doc['root']) except OSError: msg = 'Could not make directory "%s".' % doc['root'] logger.exception(msg) raise ValidationException(msg) if not os.access(doc['root'], os.W_OK): raise ValidationException( 'Unable to write into directory "%s".' % doc['root'])
def deleteFile(self, file): """ Deletes the file from disk if it is the only File in this assetstore with the given sha512. Imported files are not actually deleted. """ if file.get('imported') or 'path' not in file: return q = { 'sha512': file['sha512'], 'assetstoreId': self.assetstore['_id'] } path = os.path.join(self.assetstore['root'], file['path']) if os.path.isfile(path): with filelock.FileLock(path + '.deleteLock'): matching = self.model('file').find(q, limit=2, fields=[]) matchingUpload = self.model('upload').findOne(q) if matching.count(True) == 1 and matchingUpload is None: try: os.unlink(path) except Exception: logger.exception('Failed to delete file %s' % path)