def _imageContentSearch(self, params): limit = params['limit'] if 'limit' in params else '100' # Use FLANN index if env variable set if 'IMAGE_SPACE_FLANN_INDEX' in os.environ: if 'histogram' not in params: features = ImageFeatures() f = features.getImageFeatures({'url': params['url']}) params['histogram'] = json.dumps(f['histogram']) logger.info( 'Using FLANN INDEX at ' + os.environ['IMAGE_SPACE_FLANN_INDEX']) return requests.get( os.environ['IMAGE_SPACE_FLANN_INDEX'] + '?query=' + params['histogram'] + '&k=' + str(limit)).json() # Use Columbia index logger.info( 'Using COLUMBIA INDEX at ' + os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] + '?url=' + params['url'] + '&num=' + str(limit)) return [{'id': d} for d in requests.get( os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] + '?url=' + params['url'] + '&num=' + str(limit), verify=False ).json()['images'][0]['similar_images']['cached_image_urls']]
def download(self, item, event): mountObject = item.get(self.mongoMountAttribute) if mountObject is None: return logger.info('Bypassing default route to item ' + str(event.info['id'])) query = event.info['params'] # copy the query dict so we don't change the original query = query.copy() # parse the query parsed = self.parseQuery(query) limit = parsed['limit'] skip = parsed['skip'] sort = parsed['sort'] sortdir = parsed['sortdir'] query = parsed['query'] # connect to the database if no connection yet exists collection = self.getCollection(event.info['id'], mountObject) # get the cursor cursor = collection.find(query) # modify the cursor cursor.limit(limit) cursor.skip(skip) cursor.sort(sort, sortdir) # return a streaming function to the response handler event.addResponse(dbStreamer(cursor)) event.preventDefault()
def checkForLargeImageFiles(event): file = event.info possible = False mimeType = file.get('mimeType') if mimeType in ('image/tiff', 'image/x-tiff', 'image/x-ptif'): possible = True exts = [ext.split()[0] for ext in file.get('exts')] if set(exts[-2:]).intersection({ 'svs', 'ptif', 'tif', 'tiff', 'ndpi', 'mrxs', 'nc', 'ntf', 'nitf', 'scn' }): possible = True if not file.get('itemId') or not possible: return if not Setting().get(constants.PluginSettings.LARGE_IMAGE_AUTO_SET): return item = Item().load(file['itemId'], force=True, exc=False) if not item or item.get('largeImage'): return try: ImageItem().createImageItem(item, file, createJob=False) except Exception: # We couldn't automatically set this as a large image logger.info('Saved file %s cannot be automatically used as a ' 'largeImage' % str(file['_id']))
def createThumbnailsJobTask(item, spec): """ For an individual item, check or create all of the appropriate thumbnails. :param item: the image item. :param spec: a list of thumbnail specifications. :returns: a dictionary with the total status of the thumbnail job. """ status = {'checked': 0, 'created': 0, 'failed': 0} for entry in spec: try: if entry.get('imageKey'): result = ImageItem().getAssociatedImage(item, checkAndCreate=True, **entry) else: result = ImageItem().getThumbnail(item, checkAndCreate=True, **entry) status['checked' if result is True else 'created'] += 1 except TileGeneralException as exc: status['failed'] += 1 status['lastFailed'] = str(item['_id']) logger.info('Failed to get thumbnail for item %s: %r' % (item['_id'], exc)) except AttributeError: raise except Exception: status['failed'] += 1 status['lastFailed'] = str(item['_id']) logger.exception( 'Unexpected exception when trying to create a thumbnail for item %s' % item['_id']) return status
def getUploadAssetstore(self, event): """ Handle the model.upload.assetstore event. This event passes a dictionary consisting of a model type and resource document. If the base document has an assetstore policy, then set an assetstore key of this dictionary to an assetstore document that should be used or prevent the default action if no appropriate assetstores are allowed. :param event: event record. """ model, resource = self._getBaseResource(event.info['model'], event.info['resource']) if resource is None: return policy = resource[QUOTA_FIELD] assetstore = self._checkAssetstore( policy.get('preferredAssetstore', None)) if assetstore is False: assetstore = self._checkAssetstore( policy.get('fallbackAssetstore', None)) if assetstore is not False: logger.info('preferredAssetstore not available for %s %s, ' 'using fallbackAssetstore', model, resource['_id']) if assetstore is False: raise GirderException('Required assetstore is unavailable') if assetstore: event.addResponse(assetstore)
def __init__(self, item, **kwargs): super(TiffFileTileSource, self).__init__(item, **kwargs) largeImagePath = self._getLargeImagePath() lastException = None self._tiffDirectories = list() for directoryNum in itertools.count(): try: tiffDirectory = TiledTiffDirectory(largeImagePath, directoryNum) except TiffException as lastException: break else: self._tiffDirectories.append(tiffDirectory) if not self._tiffDirectories: logger.info('File %s didn\'t meet requirements for tile source: ' '%s' % (largeImagePath, lastException)) raise TileSourceException('File must have at least 1 level') # Multiresolution TIFFs are stored with full-resolution layer in # directory 0 self._tiffDirectories.reverse() self.tileWidth = self._tiffDirectories[-1].tileWidth self.tileHeight = self._tiffDirectories[-1].tileHeight self.levels = len(self._tiffDirectories) self.sizeX = self._tiffDirectories[-1].imageWidth self.sizeY = self._tiffDirectories[-1].imageHeight
def _copyAnnotationsFromOtherItem(self, srcItemId, destItem): # Copy annotations from the original item to this one query = {'_active': {'$ne': False}, 'itemId': srcItemId} annotations = Annotation().find(query) total = annotations.count() if not total: return destItemId = destItem['_id'] folder = Folder().load(destItem['folderId'], force=True) count = 0 for annotation in annotations: logger.info('Copying annotation %d of %d from %s to %s', count + 1, total, srcItemId, destItemId) # Make sure we have the elements annotation = Annotation().load(annotation['_id'], force=True) # This could happen, for instance, if the annotation were deleted # while we are copying other annotations. if annotation is None: continue annotation['itemId'] = destItemId del annotation['_id'] # Remove existing permissionsi, then give it the same permissions # as the item's folder. annotation.pop('access', None) self.copyAccessPolicies(destItem, annotation, save=False) self.setPublic(annotation, folder.get('public'), save=False) Annotation().save(annotation) count += 1 logger.info('Copied %d annotations from %s to %s ', count, srcItemId, destItemId)
def validateSettings(event, plugin_name=None): """ Validate plugin-specific settings and prevent disabling this plugin if there are any files in database assetstores. :param plugin_name: the name of our plugin. :param event: the validation event """ key, val = event.info['key'], event.info['value'] # If we are validating the list of enabled plugins, and there are any # database assetstores with files, do not allow the plugin to be disabled. if (key == SettingKey.PLUGINS_ENABLED and plugin_name and plugin_name not in val): store = next((store for store in Assetstore().list() if store['type'] == AssetstoreType.DATABASE and store['hasFiles']), None) if store: val.append(plugin_name) log.info('Won\'t disable %s because there are files in the %s assetstore' % ( plugin_name, store['name'])) if (key == SettingKey.PLUGINS_ENABLED and plugin_name): if plugin_name not in val: _removeUserAssetstore() else: _createUserAssetstore()
def checkForLargeImageFiles(event): file = event.info possible = False mimeType = file.get('mimeType') if mimeType in ('image/tiff', 'image/x-tiff', 'image/x-ptif'): possible = True exts = file.get('exts') if exts and exts[-1] in ('svs', 'ptif', 'tif', 'tiff', 'ndpi'): possible = True if not file.get('itemId') or not possible: return if not ModelImporter.model('setting').get( constants.PluginSettings.LARGE_IMAGE_AUTO_SET): return item = ModelImporter.model('item').load( file['itemId'], force=True, exc=False) if not item or item.get('largeImage'): return imageItemModel = ModelImporter.model('image_item', 'large_image') try: imageItemModel.createImageItem(item, file, createJob=False) except Exception: # We couldn't automatically set this as a large image logger.info('Saved file %s cannot be automatically used as a ' 'largeImage' % str(file['_id']))
def updateElements(self, annotation): """ Given an annotation, extract the elements from it and update the database of them. :param annotation: the annotation to save elements for. Modified. """ startTime = time.time() elements = annotation['annotation'].get('elements', []) if not len(elements): return now = datetime.datetime.utcnow() chunkSize = 100000 for chunk in range(0, len(elements), chunkSize): chunkStartTime = time.time() entries = [{ 'annotationId': annotation['_id'], '_version': annotation['_version'], 'created': now, 'bbox': self._boundingBox(element), 'element': element } for element in elements[chunk:chunk + chunkSize]] prepTime = time.time() - chunkStartTime res = self.collection.insert_many(entries) for pos, entry in enumerate(entries): if 'id' not in entry['element']: entry['element']['id'] = str(res.inserted_ids[pos]) # If the whole insert is slow, log information about it. if time.time() - startTime > 10: logger.info('insert %d elements in %4.2fs (prep time %4.2fs), done %d/%d' % ( len(entries), time.time() - chunkStartTime, prepTime, chunk + len(entries), len(elements))) if time.time() - startTime > 10: logger.info('inserted %d elements in %4.2fs' % ( len(elements), time.time() - startTime))
def getChildMetadata(self, id, params): # The `autoDescribeRoute` decorator processes the incoming request and # populates the function arguments. Path parameters are added as # individual arguments, while query parameters are packed into the # `params` dictionary. user = self.getCurrentUser() modelType = params['type'] model = ModelImporter.model(modelType) doc = model.load(id=id, user=user, level=AccessType.READ) if not doc: raise RestException('Resource not found.') results = {} if doc.get('meta'): results[str(doc['_id'])] = doc['meta'] logger.info('Getting child metadata') for folder in allChildFolders(parentType=modelType, parent=doc, user=user, limit=0, offset=0): if folder.get('meta'): results[str(folder['_id'])] = folder['meta'] for item in allChildItems(parentType=modelType, parent=doc, user=user, limit=0, offset=0): if item.get('meta'): results[str(item['_id'])] = item['meta'] # By default, responses to girder endpoints are json encoded when # returned to the client. In this case, it is a dictionary mapping # `id` -> `metadata`. return results
def updateElementChunk(self, elements, chunk, chunkSize, annotation, now): """ Update the database for a chunk of elements. See the updateElements method for details. """ lastTime = time.time() chunkStartTime = time.time() entries = [{ 'annotationId': annotation['_id'], '_version': annotation['_version'], 'created': now, 'bbox': self._boundingBox(element), 'element': element } for element in elements[chunk:chunk + chunkSize]] prepTime = time.time() - chunkStartTime if (len(entries) == 1 and len(entries[0]['element'].get( 'points', entries[0]['element'].get('values', []))) > MAX_ELEMENT_DOCUMENT): self.saveElementAsFile(annotation, entries) res = self.collection.insert_many(entries, ordered=False) for pos, entry in enumerate(entries): if 'id' not in entry['element']: entry['element']['id'] = str(res.inserted_ids[pos]) # If the insert is slow, log information about it. if time.time() - lastTime > 10: logger.info( 'insert %d elements in %4.2fs (prep time %4.2fs), chunk %d/%d' % (len(entries), time.time() - chunkStartTime, prepTime, chunk + len(entries), len(elements))) lastTime = time.time()
def checkForLargeImageFiles(event): file = event.info possible = False mimeType = file.get('mimeType') if mimeType in ('image/tiff', 'image/x-tiff', 'image/x-ptif'): possible = True exts = file.get('exts') if exts and exts[-1] in ('svs', 'ptif', 'tif', 'tiff', 'ndpi'): possible = True if not file.get('itemId') or not possible: return if not ModelImporter.model('setting').get( constants.PluginSettings.LARGE_IMAGE_AUTO_SET): return item = ModelImporter.model('item').load(file['itemId'], force=True, exc=False) if not item or item.get('largeImage'): return imageItemModel = ModelImporter.model('image_item', 'large_image') try: imageItemModel.createImageItem(item, file, createJob=False) except Exception: # We couldn't automatically set this as a large image logger.info('Saved file %s cannot be automatically used as a ' 'largeImage' % str(file['_id']))
def load(info): notebook = Notebook() info['apiRoot'].ythub = ytHub() info['apiRoot'].notebook = notebook info['apiRoot'].frontend = Frontend() info['apiRoot'].folder.route('GET', (':id', 'listing'), listFolder) info['apiRoot'].item.route('GET', (':id', 'listing'), listItem) info['apiRoot'].item.route('PUT', (':id', 'check'), checkItem) info['apiRoot'].folder.route('GET', (':id', 'rootpath'), folderRootpath) info['apiRoot'].folder.route('PUT', (':id', 'check'), checkFolder) info['apiRoot'].collection.route('PUT', (':id', 'check'), checkCollection) curConfig = config.getConfig() if curConfig['server']['mode'] == 'testing': cull_period = 1 else: cull_period = int(curConfig['server'].get('heartbeat', -1)) if cull_period > 0: def _heartbeat(): events.trigger('heartbeat') logger.info('Starting Heartbeat every %i s' % cull_period) heartbeat = cherrypy.process.plugins.Monitor( cherrypy.engine, _heartbeat, frequency=cull_period, name="Heartbeat") heartbeat.subscribe() events.bind('heartbeat', 'ythub', notebook.cullNotebooks) events.bind('model.user.save.created', 'ythub', addDefaultFolders)
def __init__(self, dbUri=None, **params): self.dbUri = dbUri db_connection = self.getDbConnection() self.database = db_connection.get_default_database() self.collectionName = params.get('collection', 'data') self.coll = self.database[self.collectionName] self.random = params.get('random', '_random') self.KeyTable = params['keytable'] if self.random and 'rand1' not in self.KeyTable: self.KeyTable['rand1'] = self.random self.RevTable = {v: k for k, v in self.KeyTable.items()} self.queryBase = params.get('refname', 'data') self.fieldTable = globals()[self.queryBase + '_FieldTable'] # Ensure that we have a random value for sorting if self.random: cursor = self.coll.find({self.random: {'$exists': False}}) if cursor.count(): logger.info( 'Adding random values to %s field of %s collection (%d)', self.random, self.collectionName, cursor.count()) for row in cursor: self.coll.update({'_id': row['_id']}, {'$set': {self.random: random.random()}}) logger.info( 'Added random values to %s collection', self.collectionName) self.coll.create_index([(self.random, pymongo.ASCENDING)])
def validateSettings(event, plugin_name=None): """ Validate plugin-specific settings and prevent disabling this plugin if there are any files in database assetstores. :param plugin_name: the name of our plugin. :param event: the validation event """ key, val = event.info['key'], event.info['value'] # If we are validating the list of enabled plugins, and there are any # database assetstores with files, do not allow the plugin to be disabled. if (key == SettingKey.PLUGINS_ENABLED and plugin_name and plugin_name not in val): store = next(( store for store in Assetstore().list() if store['type'] == AssetstoreType.DATABASE and store['hasFiles']), None) if store: val.append(plugin_name) log.info( 'Won\'t disable %s because there are files in the %s assetstore' % (plugin_name, store['name'])) if (key == SettingKey.PLUGINS_ENABLED and plugin_name): if plugin_name not in val: _removeUserAssetstore() else: _createUserAssetstore()
def find(self, params={}, limit=50, offset=0, sort=None, fields=None, **kwargs): """ Get data from the mongo database. Return each row in turn as a python object with the default keys or the entire dataset as a list with metadata. :param params: a dictionary of query restrictions. See the FieldTable. For values that aren't of type 'text', we also support (field)_min and (field)_max parameters, which are inclusive and exclusive respectively. :param limit: default limit for the data. :param offset: default offset for the data. :param sort: a list of tuples of the form (key, direction). :param fields: a list of fields to return, or None for all fields. :returns: a dictionary of results. """ query, sort, fields = self.processParams(params, sort, fields) logger.info('Query %r', ((query, offset, limit, sort, fields), )) cursor = self.coll.find(spec=query, skip=offset, limit=limit, sort=sort, timeout=False, fields=fields) total = cursor.count() epoch = datetime.datetime.utcfromtimestamp(0) dt = datetime.datetime result = {'count': total, 'data': [{ self.RevTable.get(k, k): v if not isinstance(v, dt) else int( (v - epoch).total_seconds() * 1000) for k, v in row.items() if k != '_id'} for row in cursor ]} return result
def createContainer(self, user, dataSet): """ Create a new container. :param user: The user creating the container. :type user: dict or None """ dmSession = Session() session = dmSession.createSession(user, dataSet) container = { '_id': objectid.ObjectId(), 'ownerId': user['_id'], 'status': 'Starting', 'sessionId': session['_id'] } self.setUserAccess(container, user = user, level = AccessType.ADMIN) container = self.save(container) self.startContainer(user, container) logger.info("Container " + str(container['_id']) + " started") return container
def purge_leaf_folder(path): folder = path_utils.lookUpPath(path, user=admin)['document'] if Item().find({'folderId': folder['_id']}).count() > 0 or \ list(Folder().childFolders(folder, 'folder', user=admin)): return logger.info("Removing empty folder %s" % path) Folder().remove(folder) purge_leaf_folder(os.path.dirname(path))
def run(self): while True: try: logger.info('Running DM file GC') self.collect() except Exception: logger.error('File collection failure', exc_info=1) time.sleep(self.settings.get(constants.PluginSettings.GC_RUN_INTERVAL))
def find(self, params={}, limit=50, offset=0, sort=None, fields=None, allowUnsorted=True, **kwargs): """ Get data from the mongo database. Return each row in turn as a python object with the default keys or the entire dataset as a list with metadata. :param params: a dictionary of query restrictions. See the TaxiFieldTable. For values that aren't of type 'text', we also support (field)_min and (field)_max parameters, which are inclusive and exclusive respectively. :param limit: default limit for the data. :param offset: default offset for the data. :param sort: a list of tuples of the form (key, direction). :param fields: a list of fields to return, or None for all fields. :param allowUnsorted: if true, and the entire data set will be returned (rather than being restricted by limit), then return the data unsorted. :returns: a dictionary of results. """ query, sort, mfields = self.processParams(params, sort, fields) logger.info('Query %r', ((query, offset, limit, sort, mfields), )) cursor = None if not offset and sort is not None and allowUnsorted: cursor = self.trips.find(spec=query, skip=offset, limit=limit, sort=None, timeout=False, fields=mfields, manipulate=False, slave_okay=True, compile_re=False) total = cursor.count() if limit and total >= limit: cursor = None if not cursor: cursor = self.trips.find(spec=query, skip=offset, limit=limit, sort=sort, timeout=False, fields=mfields, manipulate=False, slave_okay=True, compile_re=False) total = cursor.count() if fields: columns = {fields[col]: col for col in xrange(len(fields))} mcol = [self.KeyTable.get(fields[col], fields[col]) for col in xrange(len(fields))] result = { 'count': total, 'format': 'list', 'fields': fields, 'columns': columns, 'data': [[row[k] for k in mcol] for row in cursor] } else: result = {'count': total, 'data': [{ self.RevTable.get(k, k): v for k, v in row.items() if k != '_id'} for row in cursor ]} return result
def removeContainer(self, user, container): if container['ownerId'] != user['_id']: raise AccessException("This container is not yours") try: self._stopContainer(container) except: # some of these things need to be properly synchronized logger.info("Could not stop container") self.remove(container)
def _sendmail(event): msg = event.info['message'] smtpHost = ModelImporter.model('setting').get(SettingKey.SMTP_HOST, 'localhost') logger.info('Sending email to %s through %s', msg['To'], smtpHost) s = smtplib.SMTP(smtpHost) s.sendmail(msg['From'], event.info['recipients'], msg.as_string()) s.quit() logger.info('Sent email to %s', msg['To'])
def _runGCCommand(*args): p = subprocess.run(list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE) if p.returncode != 0: msg = 'Command %s failed with exit code %s: \n%s\n%s' % ( args, p.returncode, p.stdout, p.stderr) logger.warn(msg) raise Exception(msg) logger.info('Output from command %s: %s, %s' % (args, p.stdout, p.stderr)) return p
def _imageContentSearch(self, params): limit = params['limit'] if 'limit' in params else '100' if 'histogram' not in params: features = ImageFeatures() f = features.getImageFeatures({'url': params['url']}) params['histogram'] = json.dumps(f['histogram']) logger.info('Using FLANN INDEX at ' + os.environ['IMAGE_SPACE_FLANN_INDEX']) return requests.get(os.environ['IMAGE_SPACE_FLANN_INDEX'] + '?query=' + params['histogram'] + '&k=' + str(limit)).json()
def testOneFile(tempLog): tempLog = configureLogging({'log_max_info_level': 'CRITICAL'}, oneFile=True) logger.info(INFO_MSG) infoSize = os.path.getsize(tempLog['info_log_file']) errorSize = os.path.getsize(tempLog['error_log_file']) assert infoSize == errorSize logger.error(ERROR_MSG) newInfoSize = os.path.getsize(tempLog['info_log_file']) newErrorSize = os.path.getsize(tempLog['error_log_file']) assert newInfoSize == newErrorSize assert newInfoSize > infoSize
def __init__(self): # If the config was already loaded, make sure we reload using our app's # added files. loadConfig() # Use our logger values, not girder's logConfig = girder.utility.config.getConfig().get('logging', {}) if 'log_root' in logConfig: logConfig['log_root'] = os.path.expanduser(logConfig['log_root']) for hdlr in logger.handlers[:]: logger.removeHandler(hdlr) girder._setupLogger() logger.info('Minerva Taxi starting')
def testOneFile(self): self.configureLogging({'log_max_info_level': 'CRITICAL'}, oneFile=True) logger.info(self.infoMessage) infoSize = os.path.getsize(self.infoFile) errorSize = os.path.getsize(self.errorFile) self.assertEqual(infoSize, errorSize) logger.error(self.errorMessage) newInfoSize = os.path.getsize(self.infoFile) newErrorSize = os.path.getsize(self.errorFile) self.assertEqual(newInfoSize, newErrorSize) self.assertGreater(newInfoSize, infoSize)
def load(self, info): events.bind('no_password_login_attempt', 'openid', _checkOpenIdUser) info['apiRoot'].openid = rest.OpenId() SettingDefault.defaults[constants.PluginSettings.PROVIDERS] = [] if 'GIRDER_REQUESTS_VERIFY' in os.environ: path = os.environ['GIRDER_REQUESTS_VERIFY'] if not os.path.isfile(path): raise Exception('Requests cert not found: %s' % path) # We use a custom fetcher class and set it as the default to support customization # of the "verify" parameter of the requests session logger.info('OpenID: using verify value=%s' % path) setDefaultFetcher(_CustomCAFetcher(path), wrap_exceptions=False)
def getYAMLConfigFile(self, folder, name): addConfig = None user = self.getCurrentUser() last = False while folder: item = Item().findOne({'folderId': folder['_id'], 'name': name}) if item: for file in Item().childFiles(item): if file['size'] > 10 * 1024**2: logger.info('Not loading %s -- too large' % file['name']) continue with File().open(file) as fptr: config = yaml.safe_load(fptr) # combine and adjust config values based on current user if isinstance( config, dict) and 'access' in config or 'group' in config: config = adjustConfigForUser(config, user) if addConfig and isinstance(config, dict): config = _mergeDictionaries(config, addConfig) if not isinstance(config, dict) or config.get( '__inherit__') is not True: return config config.pop('__inherit__') addConfig = config if last: break if folder['parentCollection'] != 'folder': if folder['name'] != '.config': folder = Folder().findOne({ 'parentId': folder['parentId'], 'parentCollection': folder['parentCollection'], 'name': '.config' }) else: last = 'setting' if not folder or last == 'setting': folderId = Setting().get( constants.PluginSettings.LARGE_IMAGE_CONFIG_FOLDER) if not folderId: break folder = Folder().load(folderId, force=True) last = True else: folder = Folder().load(folder['parentId'], user=user, level=AccessType.READ) return addConfig
def _imageContentSearch(self, params): limit = params['limit'] if 'limit' in params else '100' logger.info( 'Using COLUMBIA INDEX at ' + os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] + '?url=' + params['url'] + '&num=' + str(limit)) return [{'id': d} for d in requests.get( os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] + '?url=' + params['url'] + '&num=' + str(limit), verify=False ).json()['images'][0]['similar_images']['cached_image_urls']]
def createThumbnailsJob(job): """ Create thumbnails for all of the large image items. :param spec: an array, each entry of which is the parameter dictionary for the model getThumbnail function. """ jobModel = ModelImporter.model('job', 'jobs') job = jobModel.updateJob( job, log='Started creating large image thumbnails\n', status=JobStatus.RUNNING) checkedOrCreated = 0 try: spec = job['kwargs']['spec'] logInterval = float(job['kwargs'].get('logInterval', 10)) itemModel = ModelImporter.model('item') imageItemModel = ModelImporter.model('image_item', 'large_image') for entry in spec: job = jobModel.updateJob( job, log='Creating thumbnails for %r\n' % entry) lastLogTime = time.time() items = itemModel.find({'largeImage.fileId': {'$exists': True}}) for item in items: imageItemModel.getThumbnail(item, **entry) checkedOrCreated += 1 # Periodically, log the state of the job and check if it was # deleted or canceled. if time.time() - lastLogTime > logInterval: job = jobModel.updateJob( job, log='Checked or created %d thumbnail file%s\n' % ( checkedOrCreated, 's' if checkedOrCreated != 1 else '')) lastLogTime = time.time() # Check if the job was deleted or canceled; if so, quit job = jobModel.load(id=job['_id'], force=True) if not job or job['status'] == JobStatus.CANCELED: logger.info('Large image thumbnails job %s' % ( 'deleted' if not job else 'canceled')) return except Exception: logger.exception('Error with large image create thumbnails job') job = jobModel.updateJob( job, log='Error creating large image thumbnails\n', status=JobStatus.ERROR) return msg = 'Finished creating large image thumbnails (%d checked or created)' % ( checkedOrCreated) logger.info(msg) job = jobModel.updateJob(job, log=msg + '\n', status=JobStatus.SUCCESS)
def testInfoMaxLevel(tempLog): tempLog = configureLogging({'log_max_info_level': 'CRITICAL'}) infoSize1 = os.path.getsize(tempLog['info_log_file']) errorSize1 = os.path.getsize(tempLog['error_log_file']) logger.info(INFO_MSG) infoSize2 = os.path.getsize(tempLog['info_log_file']) errorSize2 = os.path.getsize(tempLog['error_log_file']) assert infoSize2 > infoSize1 assert errorSize2 == errorSize1 logger.error(ERROR_MSG) infoSize3 = os.path.getsize(tempLog['info_log_file']) errorSize3 = os.path.getsize(tempLog['error_log_file']) assert infoSize3 > infoSize2 assert errorSize3 > errorSize2
def _imageContentSearch(self, params): limit = params['limit'] if 'limit' in params else '100' if 'histogram' not in params: features = ImageFeatures() f = features.getImageFeatures({'url': params['url']}) params['histogram'] = json.dumps(f['histogram']) logger.info( 'Using FLANN INDEX at ' + os.environ['IMAGE_SPACE_FLANN_INDEX']) return requests.get( os.environ['IMAGE_SPACE_FLANN_INDEX'] + '?query=' + params['histogram'] + '&k=' + str(limit)).json()
def testInfoMaxLevel(self): self.configureLogging({'log_max_info_level': 'CRITICAL'}) infoSize1 = os.path.getsize(self.infoFile) errorSize1 = os.path.getsize(self.errorFile) logger.info(self.infoMessage) infoSize2 = os.path.getsize(self.infoFile) errorSize2 = os.path.getsize(self.errorFile) self.assertGreater(infoSize2, infoSize1) self.assertEqual(errorSize2, errorSize1) logger.error(self.errorMessage) infoSize3 = os.path.getsize(self.infoFile) errorSize3 = os.path.getsize(self.errorFile) self.assertGreater(infoSize3, infoSize2) self.assertGreater(errorSize3, errorSize2)
def _imageContentSearch(self, params): setting = ColumbiaSetting() limit = params['limit'] if 'limit' in params else '100' logger.info( 'Using COLUMBIA INDEX at ' + setting.get('IMAGE_SPACE_COLUMBIA_INDEX') + '?url=' + params['url'] + '&num=' + str(limit)) return [{'id': d} for d in requests.get( setting.get('IMAGE_SPACE_COLUMBIA_INDEX') + '?url=' + params['url'] + '&num=' + str(limit), verify=False ).json()['images'][0]['similar_images']['cached_image_urls']]
def _submitEmail(msg, recipients): from girder.models.setting import Setting setting = Setting() smtp = _SMTPConnection(host=setting.get(SettingKey.SMTP_HOST), port=setting.get(SettingKey.SMTP_PORT), encryption=setting.get(SettingKey.SMTP_ENCRYPTION), username=setting.get(SettingKey.SMTP_USERNAME), password=setting.get(SettingKey.SMTP_PASSWORD)) logger.info('Sending email to %s through %s', ', '.join(recipients), smtp.host) with smtp: smtp.send(msg['From'], recipients, msg.as_string())
def _sendmail(event): msg = event.info['message'] recipients = event.info['recipients'] setting = ModelImporter.model('setting') smtp = _SMTPConnection( host=setting.get(SettingKey.SMTP_HOST, 'localhost'), port=setting.get(SettingKey.SMTP_PORT, None), encryption=setting.get(SettingKey.SMTP_ENCRYPTION, 'none'), username=setting.get(SettingKey.SMTP_USERNAME, None), password=setting.get(SettingKey.SMTP_PASSWORD, None) ) logger.info('Sending email to %s through %s', ', '.join(recipients), smtp.host) with smtp: smtp.send(msg['From'], recipients, msg.as_string())
def _sendmail(event): from girder.models.setting import Setting msg = event.info['message'] recipients = event.info['recipients'] setting = Setting() smtp = _SMTPConnection( host=setting.get(SettingKey.SMTP_HOST), port=setting.get(SettingKey.SMTP_PORT), encryption=setting.get(SettingKey.SMTP_ENCRYPTION), username=setting.get(SettingKey.SMTP_USERNAME), password=setting.get(SettingKey.SMTP_PASSWORD) ) logger.info('Sending email to %s through %s', ', '.join(recipients), smtp.host) with smtp: smtp.send(msg['From'], recipients, msg.as_string())
def _imageSearch(self, params): limit = params['limit'] if 'limit' in params else '10' if 'histogram' in params: if 'IMAGE_SPACE_FLANN_INDEX' in os.environ: logger.info('Using FLANN INDEX at ' + os.environ['IMAGE_SPACE_FLANN_INDEX']) return requests.get( os.environ['IMAGE_SPACE_FLANN_INDEX'] + '?query=' + params['histogram'] + '&k=' + str(limit)).json() logger.info('Using COLUMBIA INDEX at '+os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] + '?url=' + params['url'] + '&num=' + str(limit)) return [{'id' : d} for d in requests.get( os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] + '?url=' + params['url'] + '&num=' + str(limit), verify=False).json()['images'][0]['similar_images']['image_urls']] query = params['query'] if 'query' in params else '*' base = os.environ['IMAGE_SPACE_SOLR'] + '/select?wt=json&indent=true' try: result = requests.get(base + '&q=' + query + '&rows=' + str(limit), verify=False).json() except ValueError: return [] return result['response']['docs']
def sendEmail(to, subject, text): if isinstance(to, str): to = (to,) msg = MIMEText(text, 'html') msg['Subject'] = subject or '[no subject]' msg['To'] = ', '.join(to) msg['From'] = Setting().get(SettingKey.EMAIL_FROM_ADDRESS) recipients = list(set(to)) smtp = mail_utils._SMTPConnection( host=Setting().get(SettingKey.SMTP_HOST, 'localhost'), port=Setting().get(SettingKey.SMTP_PORT, None), encryption=Setting().get(SettingKey.SMTP_ENCRYPTION, 'none'), username=Setting().get(SettingKey.SMTP_USERNAME, None), password=Setting().get(SettingKey.SMTP_PASSWORD, None) ) logger.info('Sending email to %s through %s', ', '.join(recipients), smtp.host) with smtp: smtp.send(msg['From'], recipients, msg.as_string())
def recalculateSize(self, item): """ Recalculate the item size based on the files that are in it. If this is different than the recorded size, propagate the changes. :param item: The item to recalculate the size of. :returns: the recalculated size in bytes """ size = 0 for file in self.childFiles(item): # We could add a recalculateSize to the file model, in which case # this would be: # size += self.model('file').recalculateSize(file) size += file.get('size', 0) delta = size-item.get('size', 0) if delta: logger.info('Item %s was wrong size: was %d, is %d' % ( item['_id'], item['size'], size)) item['size'] = size self.update({'_id': item['_id']}, update={'$set': {'size': size}}) self.propagateSizeChange(item, delta) return size
def addDeps(plugin): if plugin not in allPlugins: message = 'Required plugin %s does not exist.' % plugin if ignoreMissing: logprint.error(message) return else: raise ValidationException(message) deps = set() for key in keys: deps |= allPlugins[plugin][key] dag[plugin] = deps for dep in deps: if dep in visited: continue visited.add(dep) if dep not in plugins: logger.info('Adding plugin %s because %s requires it' % (dep, plugin)) addDeps(dep)
def __new__(metacls, name, bases, namespace, **kwargs): # noqa - N804 # Get metaclass parameters by finding and removing them from the class # namespace (necessary for Python 2), or preferentially as metaclass # arguments (only in Python 3). cacheName = namespace.get('cacheName', None) cacheName = kwargs.get('cacheName', cacheName) maxSize = CacheProperties.get(cacheName, {}).get('cacheMaxSize', None) maxSize = namespace.pop('cacheMaxSize', maxSize) maxSize = kwargs.get('cacheMaxSize', maxSize) if maxSize is None: raise TypeError('Usage of the LruCacheMetaclass requires a ' '"cacheMaxSize" attribute on the class %s.' % name) timeout = CacheProperties.get(cacheName, {}).get('cacheTimeout', None) timeout = namespace.pop('cacheTimeout', timeout) timeout = kwargs.get('cacheTimeout', timeout) cls = super(LruCacheMetaclass, metacls).__new__( metacls, name, bases, namespace) if not cacheName: cacheName = cls if LruCacheMetaclass.namedCaches.get(cacheName) is None: cache, cacheLock = CacheFactory().getCache(maxSize) LruCacheMetaclass.namedCaches[cacheName] = cache logger.info('Created LRU Cache for %r with %d maximum size' % ( cacheName, maxSize)) else: cache = LruCacheMetaclass.namedCaches[cacheName] # Don't store the cache in cls.__dict__, because we don't want it to be # part of the attribute lookup hierarchy # TODO: consider putting it in cls.__dict__, to inspect statistics # cls is hashable though, so use it to lookup the cache, in case an # identically-named class gets redefined LruCacheMetaclass.classCaches[cls] = cache return cls
def _wait_for_server(url, timeout=30, wait_time=0.5): """Wait for a server to show up within a newly launched instance.""" tic = time.time() # Fudge factor of IPython notebook bootup. time.sleep(0.5) while time.time() - tic < timeout: try: urlopen(url, timeout=1) except HTTPError as err: logger.info( 'Booting server at [%s], getting HTTP status [%s]', url, err.code) time.sleep(wait_time) except URLError as err: logger.info( 'Booting server at [%s], getting URLError due to [%s]', url, err.reason) except (ssl.SSLError, ssl.CertificateError): time.sleep(wait_time) else: break
def _processImages(folder, itemFilePairs): """ Create and schedule a Girder job for processing images. :param folder: Folder to process images for. :param fileIds: File IDs to process, these are converted into Girder data elemnts. """ jobModel = ModelImporter.model('job', 'jobs') # TODO Use a more granular token. # Ideally this would be scoped to only allow: # - Job Updates # - Data management of folder # - Retrieval of SMQTK settings token = ModelImporter.model('token').createToken(user=getCurrentUser(), days=1, scope=(TokenScope.USER_AUTH, SMQTK_SETTING_READ)) dataElementUris = [(itemId, 'girder://*****:*****@%s/file/%s' % (token['_id'], getWorkerApiUrl(), fileId)) for (itemId, fileId) in itemFilePairs] job = jobModel.createJob(title='Processing Images', type='GPU', handler='worker_handler', user=getCurrentUser(), args=(str(folder['_id']), dataElementUris), otherFields={'celeryTaskName': 'smqtk_worker.tasks.process_images', 'celeryQueue': 'process-images'}) job['token'] = token logger.info('assigning token %s' % token['_id']) jobModel.save(job) jobModel.scheduleJob(job)
def connectToMongoCollection(self, params): self.requireParams(('host', 'port', 'database', 'collection'), params) host = params['host'] port = params['port'] try: port = int(port) except ValueError: raise RestException('Invalid port number') try: client = pymongo.MongoClient(host=host, port=port) except pymongo.errors.ConnectionFailure: raise RestException('Could not connect to mounted database') dbName = params['database'] if not dbName in client.database_names(): raise RestException('Database does not exist') db = client[dbName] logger.info('Connected to mongo collection at %s:%i/%s/%s' % (host, port, dbName, params['collection'])) return db[params['collection']]
def getTableList(cls, uri, internalTables=False, dbparams={}, **kwargs): """ Get a list of known databases, each of which has a list of known tables from the database. This is of the form [{'database': (database), 'tables': [{'schema': (schema), 'table': (table 1)}, ...]}] :param uri: uri to connect to the database. :param internaltables: True to return tables about the database itself. :param dbparams: optional parameters to send to the connection. :returns: A list of known tables. """ dbEngine = sqlalchemy.create_engine(cls.adjustDBUri(uri), **dbparams) insp = sqlalchemy.engine.reflection.Inspector.from_engine(dbEngine) schemas = insp.get_schema_names() defaultSchema = insp.default_schema_name tables = [{'name': table, 'table': table} for table in dbEngine.table_names()] tables.extend([{'name': view, 'table': view} for view in insp.get_view_names()]) databaseName = base.databaseFromUri(uri) results = [{'database': databaseName, 'tables': tables}] if len(schemas) <= MAX_SCHEMAS_IN_TABLE_LIST: for schema in schemas: if not internalTables and schema.lower() == 'information_schema': continue if schema != defaultSchema: tables = [{'name': '%s.%s' % (schema, table), 'table': table, 'schema': schema} for table in dbEngine.table_names(schema=schema)] tables.extend([{'name': '%s.%s' % (schema, view), 'table': view, 'schema': schema} for view in insp.get_view_names(schema=schema)]) results[0]['tables'].extend(tables) else: log.info('Not enumerating all schemas for table list (%d schemas)', len(schemas)) return results
def cullNotebooks(self): resp = requests.get( self.model('setting').get(PluginSettings.TMPNB_URL)) content = resp.content if isinstance(content, six.binary_type): content = content.decode('utf8') try: resp.raise_for_status() except requests.HTTPError: raise RestException( 'Got %s code from tmpnb, response="%s"/' % ( resp.status_code, content ), code=502) try: activity = json.loads(content) except ValueError: raise RestException('Non-JSON response: %s' % content, code=502) admin = next(_ for _ in self.model('user').getAdmins()) token = self.model('token').createToken(user=admin, days=1) # Iterate over all notebooks, not the prettiest way... cull_period = self.model('setting').get( PluginSettings.CULLING_PERIOD, '4') cull_time = datetime.datetime.utcnow() - \ datetime.timedelta(hours=float(cull_period)) for nb in self.find({}): try: last_activity = dateutil.parser.parse( activity[nb['containerId']], ignoretz=True) if last_activity < cull_time: logger.info('Deleting nb %s' % nb['_id']) self.deleteNotebook(nb, token) except KeyError: # proxy is not aware of such container, kill it... logger.info('Deleting nb %s' % nb['_id']) self.deleteNotebook(nb, token)
def testFileRotation(self): self.configureLogging({ 'log_access': ['screen', 'info'], 'log_quiet': True, 'log_max_size': '1 kb', 'log_backup_count': 2, 'log_level': 'DEBUG', }) logger.info(self.infoMessage) logger.error(self.errorMessage) infoSize = os.path.getsize(self.infoFile) errorSize = os.path.getsize(self.errorFile) self.assertFalse(os.path.exists(self.infoFile + '.1')) self.assertFalse(os.path.exists(self.errorFile + '.1')) logger.info(self.infoMessage) logger.error(self.errorMessage) newInfoSize = os.path.getsize(self.infoFile) newErrorSize = os.path.getsize(self.errorFile) deltaInfo = newInfoSize - infoSize deltaError = newErrorSize - errorSize self.assertGreater(deltaInfo, len(self.infoMessage)) self.assertGreater(deltaError, len(self.errorMessage)) while newInfoSize < 1024 * 1.5: logger.info(self.infoMessage) newInfoSize += deltaInfo while newErrorSize < 1024 * 1.5: logger.error(self.errorMessage) newErrorSize += deltaError self.assertTrue(os.path.exists(self.infoFile + '.1')) self.assertTrue(os.path.exists(self.errorFile + '.1')) self.assertFalse(os.path.exists(self.infoFile + '.2')) self.assertFalse(os.path.exists(self.errorFile + '.2')) while newInfoSize < 1024 * 3.5: logger.info(self.infoMessage) newInfoSize += deltaInfo while newErrorSize < 1024 * 3.5: logger.error(self.errorMessage) newErrorSize += deltaError self.assertTrue(os.path.exists(self.infoFile + '.1')) self.assertTrue(os.path.exists(self.errorFile + '.1')) self.assertTrue(os.path.exists(self.infoFile + '.2')) self.assertTrue(os.path.exists(self.errorFile + '.2')) self.assertFalse(os.path.exists(self.infoFile + '.3')) self.assertFalse(os.path.exists(self.errorFile + '.3'))
def testFileRotation(tempLog): tempLog = configureLogging({ 'log_access': ['screen', 'info'], 'log_quiet': True, 'log_max_size': '1 kb', 'log_backup_count': 2, 'log_level': 'DEBUG', }) logger.info(INFO_MSG) logger.error(ERROR_MSG) infoSize = os.path.getsize(tempLog['info_log_file']) errorSize = os.path.getsize(tempLog['error_log_file']) assert os.path.exists(tempLog['info_log_file'] + '.1') is False assert os.path.exists(tempLog['error_log_file'] + '.1') is False logger.info(INFO_MSG) logger.error(ERROR_MSG) newInfoSize = os.path.getsize(tempLog['info_log_file']) newErrorSize = os.path.getsize(tempLog['error_log_file']) deltaInfo = newInfoSize - infoSize deltaError = newErrorSize - errorSize assert deltaInfo > len(INFO_MSG) assert deltaError > len(ERROR_MSG) while newInfoSize < 1024 * 1.5: logger.info(INFO_MSG) newInfoSize += deltaInfo while newErrorSize < 1024 * 1.5: logger.error(ERROR_MSG) newErrorSize += deltaError assert os.path.exists(tempLog['info_log_file'] + '.1') is True assert os.path.exists(tempLog['error_log_file'] + '.1') is True assert os.path.exists(tempLog['info_log_file'] + '.2') is False assert os.path.exists(tempLog['error_log_file'] + '.2') is False while newInfoSize < 1024 * 3.5: logger.info(INFO_MSG) newInfoSize += deltaInfo while newErrorSize < 1024 * 3.5: logger.error(ERROR_MSG) newErrorSize += deltaError assert os.path.exists(tempLog['info_log_file'] + '.1') is True assert os.path.exists(tempLog['error_log_file'] + '.1') is True assert os.path.exists(tempLog['info_log_file'] + '.2') is True assert os.path.exists(tempLog['error_log_file'] + '.2') is True assert os.path.exists(tempLog['info_log_file'] + '.3') is False assert os.path.exists(tempLog['error_log_file'] + '.3') is False
def refine(self, params): sid = str(params['item']['_id']) pos_uuids = params['pos_uuids'] neg_uuids = params['neg_uuids'] if params['neg_uuids'] is not None else [] if len(pos_uuids) == 0: raise RestException('No positive UUIDs given.') with self.controller: if not self.controller.has_session_uuid(sid): raise RestException('Session ID %s not found.' % sid, 404) iqrs = self.controller.get_session(sid) iqrs.lock.acquire() # lock BEFORE releasing controller try: descriptor_index = self._descriptorIndexFromSessionId(sid) neighbor_index = self._nearestNeighborIndex(sid, descriptor_index) if descriptor_index is None or neighbor_index is None: logger.error('Unable to compute descriptor or neighbor index from sid %s.' % sid) raise RestException('Unable to compute descriptor or neighbor index from sid %s.' % sid, 500) # Get appropriate descriptor elements from index for # setting new adjudication state. try: pos_descrs = set(descriptor_index.get_many_descriptors(pos_uuids)) neg_descrs = set(descriptor_index.get_many_descriptors(neg_uuids)) except KeyError as ex: logger.warn(traceback.format_exc()) raise RestException('Descriptor UUID %s not found in index.' % ex, 404) # if a new classifier should be made upon the next # classification request. diff_pos = pos_descrs.symmetric_difference(iqrs.positive_descriptors) diff_neg = neg_descrs.symmetric_difference(iqrs.negative_descriptors) if diff_pos or diff_neg: logger.debug("[%s] session Classifier dirty", sid) self.session_classifier_dirty[sid] = True logger.info("[%s] Setting adjudications", sid) iqrs.positive_descriptors = pos_descrs iqrs.negative_descriptors = neg_descrs logger.info("[%s] Updating working index", sid) iqrs.update_working_index(neighbor_index) logger.info("[%s] Refining", sid) iqrs.refine() finally: iqrs.lock.release() return sid
def uploadChunk(self, upload, chunk): """ Stores the uploaded chunk in fixed-sized pieces in the chunks collection of this assetstore's database. """ # If we know the chunk size is too large or small, fail early. self.checkUploadSize(upload, self.getChunkSize(chunk)) if isinstance(chunk, six.text_type): chunk = chunk.encode('utf8') if isinstance(chunk, six.binary_type): chunk = BytesIO(chunk) # Restore the internal state of the streaming SHA-512 checksum checksum = sha512_state.restoreHex(upload['sha512state']) # This bit of code will only do anything if there is a discrepancy # between the received count of the upload record and the length of # the file stored as chunks in the database. This code simply updates # the sha512 state with the difference before reading the bytes sent # from the user. if self.requestOffset(upload) > upload['received']: cursor = self.chunkColl.find({ 'uuid': upload['chunkUuid'], 'n': {'$gte': upload['received'] // CHUNK_SIZE} }, fields=['data']).sort('n', pymongo.ASCENDING) for result in cursor: checksum.update(result['data']) cursor = self.chunkColl.find({ 'uuid': upload['chunkUuid'] }, fields=['n']).sort('n', pymongo.DESCENDING).limit(1) if cursor.count(True) == 0: n = 0 else: n = cursor[0]['n'] + 1 size = 0 startingN = n while not upload['received']+size > upload['size']: data = chunk.read(CHUNK_SIZE) if not data: break # If a timeout occurs while we are trying to load data, we might # have succeeded, in which case we will get a DuplicateKeyError # when it automatically retries. Therefore, log this error but # don't stop. try: self.chunkColl.insert({ 'n': n, 'uuid': upload['chunkUuid'], 'data': bson.binary.Binary(data) }) except pymongo.errors.DuplicateKeyError: logger.info('Received a DuplicateKeyError while uploading, ' 'probably because we reconnected to the database ' '(chunk uuid %s part %d)', upload['chunkUuid'], n) n += 1 size += len(data) checksum.update(data) chunk.close() try: self.checkUploadSize(upload, size) except ValidationException: # The user tried to upload too much or too little. Delete # everything we added self.chunkColl.remove({'uuid': upload['chunkUuid'], 'n': {'$gte': startingN}}, multi=True) raise # Persist the internal state of the checksum upload['sha512state'] = sha512_state.serializeHex(checksum) upload['received'] += size return upload
def performSelect(self, fields, queryProps={}, filters=[], client=None): """ Select data from the database. The results are passed back as a dictionary with the following values: limit: the limit used in the query offset: the offset used in the query sort: the list of sort parameters used in the query. fields: a list of the fields that are being returned in the order that they are returned. data: a list with one entry per row of results. Each entry is a list with one entry per column. :param fields: the results from getFieldInfo. :param queryProps: general query properties, including limit, offset, and sort. :param filters: a list of filters to apply. :param client: if a client is specified, a previous query made by this client can be cancelled. :return: the results of the query. See above. """ result = super(MongoConnector, self).performSelect( fields, queryProps, filters) if queryProps.get('group'): raise DatabaseConnectorException( 'Group unsupported by this database.') filterQueryClauses = [] for filt in filters: filterQueryClauses = self._addFilter(filterQueryClauses, filt) opts = {} for k, v in six.iteritems(queryProps): target = None if k == 'fields' and v and v != []: target = 'projection' v = {field: True for field in v} if '_id' not in v: v['_id'] = False elif k == 'offset': target = 'skip' elif k in ['limit', 'no_cursor_timeout', 'cursor_type', 'sort', 'allow_partial_results', 'oplog_replay', 'modifiers']: target = k if target is not None: opts[target] = v if len(filterQueryClauses) > 0: opts['filter'] = {'$and': filterQueryClauses} result['format'] = 'dict' if queryProps.get('limit') == 0: result['data'] = [] else: if queryProps.get('limit') < 0: opts['limit'] = 0 coll = self.connect() log.info('Query: %s', bson.json_util.dumps( opts, check_circular=False, separators=(',', ':'), sort_keys=False, default=str, indent=None)) cursor = coll.find(**opts) result['datacount'] = cursor.count(True) result['data'] = cursor self.disconnect() return result