def _imageContentSearch(self, params):
limit = params['limit'] if 'limit' in params else '100'
# Use FLANN index if env variable set
if 'IMAGE_SPACE_FLANN_INDEX' in os.environ:
if 'histogram' not in params:
features = ImageFeatures()
f = features.getImageFeatures({'url': params['url']})
params['histogram'] = json.dumps(f['histogram'])
logger.info(
'Using FLANN INDEX at ' +
os.environ['IMAGE_SPACE_FLANN_INDEX'])
return requests.get(
os.environ['IMAGE_SPACE_FLANN_INDEX'] +
'?query=' + params['histogram'] +
'&k=' + str(limit)).json()
# Use Columbia index
logger.info(
'Using COLUMBIA INDEX at ' +
os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] +
'?url=' + params['url'] +
'&num=' + str(limit))
return [{'id': d} for d in requests.get(
os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] +
'?url=' + params['url'] +
'&num=' + str(limit),
verify=False
).json()['images'][0]['similar_images']['cached_image_urls']]
def download(self, item, event):
mountObject = item.get(self.mongoMountAttribute)
if mountObject is None:
return
logger.info('Bypassing default route to item ' + str(event.info['id']))
query = event.info['params']
# copy the query dict so we don't change the original
query = query.copy()
# parse the query
parsed = self.parseQuery(query)
limit = parsed['limit']
skip = parsed['skip']
sort = parsed['sort']
sortdir = parsed['sortdir']
query = parsed['query']
# connect to the database if no connection yet exists
collection = self.getCollection(event.info['id'], mountObject)
# get the cursor
cursor = collection.find(query)
# modify the cursor
cursor.limit(limit)
cursor.skip(skip)
cursor.sort(sort, sortdir)
# return a streaming function to the response handler
event.addResponse(dbStreamer(cursor))
event.preventDefault()
def checkForLargeImageFiles(event):
file = event.info
possible = False
mimeType = file.get('mimeType')
if mimeType in ('image/tiff', 'image/x-tiff', 'image/x-ptif'):
possible = True
exts = [ext.split()[0] for ext in file.get('exts')]
if set(exts[-2:]).intersection({
'svs', 'ptif', 'tif', 'tiff', 'ndpi', 'mrxs', 'nc', 'ntf', 'nitf',
'scn'
}):
possible = True
if not file.get('itemId') or not possible:
return
if not Setting().get(constants.PluginSettings.LARGE_IMAGE_AUTO_SET):
return
item = Item().load(file['itemId'], force=True, exc=False)
if not item or item.get('largeImage'):
return
try:
ImageItem().createImageItem(item, file, createJob=False)
except Exception:
# We couldn't automatically set this as a large image
logger.info('Saved file %s cannot be automatically used as a '
'largeImage' % str(file['_id']))
def createThumbnailsJobTask(item, spec):
"""
For an individual item, check or create all of the appropriate thumbnails.
:param item: the image item.
:param spec: a list of thumbnail specifications.
:returns: a dictionary with the total status of the thumbnail job.
"""
status = {'checked': 0, 'created': 0, 'failed': 0}
for entry in spec:
try:
if entry.get('imageKey'):
result = ImageItem().getAssociatedImage(item,
checkAndCreate=True,
**entry)
else:
result = ImageItem().getThumbnail(item,
checkAndCreate=True,
**entry)
status['checked' if result is True else 'created'] += 1
except TileGeneralException as exc:
status['failed'] += 1
status['lastFailed'] = str(item['_id'])
logger.info('Failed to get thumbnail for item %s: %r' %
(item['_id'], exc))
except AttributeError:
raise
except Exception:
status['failed'] += 1
status['lastFailed'] = str(item['_id'])
logger.exception(
'Unexpected exception when trying to create a thumbnail for item %s'
% item['_id'])
return status
def getUploadAssetstore(self, event):
"""
Handle the model.upload.assetstore event. This event passes a
dictionary consisting of a model type and resource document. If the
base document has an assetstore policy, then set an assetstore key of
this dictionary to an assetstore document that should be used or
prevent the default action if no appropriate assetstores are allowed.
:param event: event record.
"""
model, resource = self._getBaseResource(event.info['model'],
event.info['resource'])
if resource is None:
return
policy = resource[QUOTA_FIELD]
assetstore = self._checkAssetstore(
policy.get('preferredAssetstore', None))
if assetstore is False:
assetstore = self._checkAssetstore(
policy.get('fallbackAssetstore', None))
if assetstore is not False:
logger.info('preferredAssetstore not available for %s %s, '
'using fallbackAssetstore', model, resource['_id'])
if assetstore is False:
raise GirderException('Required assetstore is unavailable')
if assetstore:
event.addResponse(assetstore)
def __init__(self, item, **kwargs):
super(TiffFileTileSource, self).__init__(item, **kwargs)
largeImagePath = self._getLargeImagePath()
lastException = None
self._tiffDirectories = list()
for directoryNum in itertools.count():
try:
tiffDirectory = TiledTiffDirectory(largeImagePath,
directoryNum)
except TiffException as lastException:
break
else:
self._tiffDirectories.append(tiffDirectory)
if not self._tiffDirectories:
logger.info('File %s didn\'t meet requirements for tile source: '
'%s' % (largeImagePath, lastException))
raise TileSourceException('File must have at least 1 level')
# Multiresolution TIFFs are stored with full-resolution layer in
# directory 0
self._tiffDirectories.reverse()
self.tileWidth = self._tiffDirectories[-1].tileWidth
self.tileHeight = self._tiffDirectories[-1].tileHeight
self.levels = len(self._tiffDirectories)
self.sizeX = self._tiffDirectories[-1].imageWidth
self.sizeY = self._tiffDirectories[-1].imageHeight
def _copyAnnotationsFromOtherItem(self, srcItemId, destItem):
# Copy annotations from the original item to this one
query = {'_active': {'$ne': False}, 'itemId': srcItemId}
annotations = Annotation().find(query)
total = annotations.count()
if not total:
return
destItemId = destItem['_id']
folder = Folder().load(destItem['folderId'], force=True)
count = 0
for annotation in annotations:
logger.info('Copying annotation %d of %d from %s to %s',
count + 1, total, srcItemId, destItemId)
# Make sure we have the elements
annotation = Annotation().load(annotation['_id'], force=True)
# This could happen, for instance, if the annotation were deleted
# while we are copying other annotations.
if annotation is None:
continue
annotation['itemId'] = destItemId
del annotation['_id']
# Remove existing permissionsi, then give it the same permissions
# as the item's folder.
annotation.pop('access', None)
self.copyAccessPolicies(destItem, annotation, save=False)
self.setPublic(annotation, folder.get('public'), save=False)
Annotation().save(annotation)
count += 1
logger.info('Copied %d annotations from %s to %s ',
count, srcItemId, destItemId)
def validateSettings(event, plugin_name=None):
"""
Validate plugin-specific settings and prevent disabling this plugin if
there are any files in database assetstores.
:param plugin_name: the name of our plugin.
:param event: the validation event
"""
key, val = event.info['key'], event.info['value']
# If we are validating the list of enabled plugins, and there are any
# database assetstores with files, do not allow the plugin to be disabled.
if (key == SettingKey.PLUGINS_ENABLED and plugin_name and
plugin_name not in val):
store = next((store for store in Assetstore().list()
if store['type'] == AssetstoreType.DATABASE and
store['hasFiles']), None)
if store:
val.append(plugin_name)
log.info('Won\'t disable %s because there are files in the %s assetstore' % (
plugin_name, store['name']))
if (key == SettingKey.PLUGINS_ENABLED and plugin_name):
if plugin_name not in val:
_removeUserAssetstore()
else:
_createUserAssetstore()
def checkForLargeImageFiles(event):
file = event.info
possible = False
mimeType = file.get('mimeType')
if mimeType in ('image/tiff', 'image/x-tiff', 'image/x-ptif'):
possible = True
exts = file.get('exts')
if exts and exts[-1] in ('svs', 'ptif', 'tif', 'tiff', 'ndpi'):
possible = True
if not file.get('itemId') or not possible:
return
if not ModelImporter.model('setting').get(
constants.PluginSettings.LARGE_IMAGE_AUTO_SET):
return
item = ModelImporter.model('item').load(
file['itemId'], force=True, exc=False)
if not item or item.get('largeImage'):
return
imageItemModel = ModelImporter.model('image_item', 'large_image')
try:
imageItemModel.createImageItem(item, file, createJob=False)
except Exception:
# We couldn't automatically set this as a large image
logger.info('Saved file %s cannot be automatically used as a '
'largeImage' % str(file['_id']))
def updateElements(self, annotation):
"""
Given an annotation, extract the elements from it and update the
database of them.
:param annotation: the annotation to save elements for. Modified.
"""
startTime = time.time()
elements = annotation['annotation'].get('elements', [])
if not len(elements):
return
now = datetime.datetime.utcnow()
chunkSize = 100000
for chunk in range(0, len(elements), chunkSize):
chunkStartTime = time.time()
entries = [{
'annotationId': annotation['_id'],
'_version': annotation['_version'],
'created': now,
'bbox': self._boundingBox(element),
'element': element
} for element in elements[chunk:chunk + chunkSize]]
prepTime = time.time() - chunkStartTime
res = self.collection.insert_many(entries)
for pos, entry in enumerate(entries):
if 'id' not in entry['element']:
entry['element']['id'] = str(res.inserted_ids[pos])
# If the whole insert is slow, log information about it.
if time.time() - startTime > 10:
logger.info('insert %d elements in %4.2fs (prep time %4.2fs), done %d/%d' % (
len(entries), time.time() - chunkStartTime, prepTime,
chunk + len(entries), len(elements)))
if time.time() - startTime > 10:
logger.info('inserted %d elements in %4.2fs' % (
len(elements), time.time() - startTime))
def getChildMetadata(self, id, params):
# The `autoDescribeRoute` decorator processes the incoming request and
# populates the function arguments. Path parameters are added as
# individual arguments, while query parameters are packed into the
# `params` dictionary.
user = self.getCurrentUser()
modelType = params['type']
model = ModelImporter.model(modelType)
doc = model.load(id=id, user=user, level=AccessType.READ)
if not doc:
raise RestException('Resource not found.')
results = {}
if doc.get('meta'):
results[str(doc['_id'])] = doc['meta']
logger.info('Getting child metadata')
for folder in allChildFolders(parentType=modelType, parent=doc,
user=user, limit=0, offset=0):
if folder.get('meta'):
results[str(folder['_id'])] = folder['meta']
for item in allChildItems(parentType=modelType, parent=doc,
user=user, limit=0, offset=0):
if item.get('meta'):
results[str(item['_id'])] = item['meta']
# By default, responses to girder endpoints are json encoded when
# returned to the client. In this case, it is a dictionary mapping
# `id` -> `metadata`.
return results
def updateElementChunk(self, elements, chunk, chunkSize, annotation, now):
"""
Update the database for a chunk of elements. See the updateElements
method for details.
"""
lastTime = time.time()
chunkStartTime = time.time()
entries = [{
'annotationId': annotation['_id'],
'_version': annotation['_version'],
'created': now,
'bbox': self._boundingBox(element),
'element': element
} for element in elements[chunk:chunk + chunkSize]]
prepTime = time.time() - chunkStartTime
if (len(entries) == 1 and len(entries[0]['element'].get(
'points', entries[0]['element'].get('values', []))) >
MAX_ELEMENT_DOCUMENT):
self.saveElementAsFile(annotation, entries)
res = self.collection.insert_many(entries, ordered=False)
for pos, entry in enumerate(entries):
if 'id' not in entry['element']:
entry['element']['id'] = str(res.inserted_ids[pos])
# If the insert is slow, log information about it.
if time.time() - lastTime > 10:
logger.info(
'insert %d elements in %4.2fs (prep time %4.2fs), chunk %d/%d'
% (len(entries), time.time() - chunkStartTime, prepTime,
chunk + len(entries), len(elements)))
lastTime = time.time()
def checkForLargeImageFiles(event):
file = event.info
possible = False
mimeType = file.get('mimeType')
if mimeType in ('image/tiff', 'image/x-tiff', 'image/x-ptif'):
possible = True
exts = file.get('exts')
if exts and exts[-1] in ('svs', 'ptif', 'tif', 'tiff', 'ndpi'):
possible = True
if not file.get('itemId') or not possible:
return
if not ModelImporter.model('setting').get(
constants.PluginSettings.LARGE_IMAGE_AUTO_SET):
return
item = ModelImporter.model('item').load(file['itemId'],
force=True,
exc=False)
if not item or item.get('largeImage'):
return
imageItemModel = ModelImporter.model('image_item', 'large_image')
try:
imageItemModel.createImageItem(item, file, createJob=False)
except Exception:
# We couldn't automatically set this as a large image
logger.info('Saved file %s cannot be automatically used as a '
'largeImage' % str(file['_id']))
def load(info):
notebook = Notebook()
info['apiRoot'].ythub = ytHub()
info['apiRoot'].notebook = notebook
info['apiRoot'].frontend = Frontend()
info['apiRoot'].folder.route('GET', (':id', 'listing'), listFolder)
info['apiRoot'].item.route('GET', (':id', 'listing'), listItem)
info['apiRoot'].item.route('PUT', (':id', 'check'), checkItem)
info['apiRoot'].folder.route('GET', (':id', 'rootpath'), folderRootpath)
info['apiRoot'].folder.route('PUT', (':id', 'check'), checkFolder)
info['apiRoot'].collection.route('PUT', (':id', 'check'), checkCollection)
curConfig = config.getConfig()
if curConfig['server']['mode'] == 'testing':
cull_period = 1
else:
cull_period = int(curConfig['server'].get('heartbeat', -1))
if cull_period > 0:
def _heartbeat():
events.trigger('heartbeat')
logger.info('Starting Heartbeat every %i s' % cull_period)
heartbeat = cherrypy.process.plugins.Monitor(
cherrypy.engine, _heartbeat, frequency=cull_period,
name="Heartbeat")
heartbeat.subscribe()
events.bind('heartbeat', 'ythub', notebook.cullNotebooks)
events.bind('model.user.save.created', 'ythub', addDefaultFolders)
def __init__(self, dbUri=None, **params):
self.dbUri = dbUri
db_connection = self.getDbConnection()
self.database = db_connection.get_default_database()
self.collectionName = params.get('collection', 'data')
self.coll = self.database[self.collectionName]
self.random = params.get('random', '_random')
self.KeyTable = params['keytable']
if self.random and 'rand1' not in self.KeyTable:
self.KeyTable['rand1'] = self.random
self.RevTable = {v: k for k, v in self.KeyTable.items()}
self.queryBase = params.get('refname', 'data')
self.fieldTable = globals()[self.queryBase + '_FieldTable']
# Ensure that we have a random value for sorting
if self.random:
cursor = self.coll.find({self.random: {'$exists': False}})
if cursor.count():
logger.info(
'Adding random values to %s field of %s collection (%d)',
self.random, self.collectionName, cursor.count())
for row in cursor:
self.coll.update({'_id': row['_id']},
{'$set': {self.random: random.random()}})
logger.info(
'Added random values to %s collection',
self.collectionName)
self.coll.create_index([(self.random, pymongo.ASCENDING)])
def validateSettings(event, plugin_name=None):
"""
Validate plugin-specific settings and prevent disabling this plugin if
there are any files in database assetstores.
:param plugin_name: the name of our plugin.
:param event: the validation event
"""
key, val = event.info['key'], event.info['value']
# If we are validating the list of enabled plugins, and there are any
# database assetstores with files, do not allow the plugin to be disabled.
if (key == SettingKey.PLUGINS_ENABLED and plugin_name
and plugin_name not in val):
store = next((
store for store in Assetstore().list()
if store['type'] == AssetstoreType.DATABASE and store['hasFiles']),
None)
if store:
val.append(plugin_name)
log.info(
'Won\'t disable %s because there are files in the %s assetstore'
% (plugin_name, store['name']))
if (key == SettingKey.PLUGINS_ENABLED and plugin_name):
if plugin_name not in val:
_removeUserAssetstore()
else:
_createUserAssetstore()
def find(self, params={}, limit=50, offset=0, sort=None, fields=None,
**kwargs):
"""
Get data from the mongo database. Return each row in turn as a python
object with the default keys or the entire dataset as a list with
metadata.
:param params: a dictionary of query restrictions. See the
FieldTable. For values that aren't of type 'text',
we also support (field)_min and (field)_max parameters,
which are inclusive and exclusive respectively.
:param limit: default limit for the data.
:param offset: default offset for the data.
:param sort: a list of tuples of the form (key, direction).
:param fields: a list of fields to return, or None for all fields.
:returns: a dictionary of results.
"""
query, sort, fields = self.processParams(params, sort, fields)
logger.info('Query %r', ((query, offset, limit, sort, fields), ))
cursor = self.coll.find(spec=query, skip=offset, limit=limit,
sort=sort, timeout=False, fields=fields)
total = cursor.count()
epoch = datetime.datetime.utcfromtimestamp(0)
dt = datetime.datetime
result = {'count': total, 'data': [{
self.RevTable.get(k, k):
v if not isinstance(v, dt) else int(
(v - epoch).total_seconds() * 1000)
for k, v in row.items() if k != '_id'}
for row in cursor
]}
return result
def createContainer(self, user, dataSet):
"""
Create a new container.
:param user: The user creating the container.
:type user: dict or None
"""
dmSession = Session()
session = dmSession.createSession(user, dataSet)
container = {
'_id': objectid.ObjectId(),
'ownerId': user['_id'],
'status': 'Starting',
'sessionId': session['_id']
}
self.setUserAccess(container, user = user, level = AccessType.ADMIN)
container = self.save(container)
self.startContainer(user, container)
logger.info("Container " + str(container['_id']) + " started")
return container
def purge_leaf_folder(path):
folder = path_utils.lookUpPath(path, user=admin)['document']
if Item().find({'folderId': folder['_id']}).count() > 0 or \
list(Folder().childFolders(folder, 'folder', user=admin)):
return
logger.info("Removing empty folder %s" % path)
Folder().remove(folder)
purge_leaf_folder(os.path.dirname(path))
def run(self):
while True:
try:
logger.info('Running DM file GC')
self.collect()
except Exception:
logger.error('File collection failure', exc_info=1)
time.sleep(self.settings.get(constants.PluginSettings.GC_RUN_INTERVAL))
def find(self, params={}, limit=50, offset=0, sort=None, fields=None,
allowUnsorted=True, **kwargs):
"""
Get data from the mongo database. Return each row in turn as a python
object with the default keys or the entire dataset as a list with
metadata.
:param params: a dictionary of query restrictions. See the
TaxiFieldTable. For values that aren't of type 'text',
we also support (field)_min and (field)_max parameters,
which are inclusive and exclusive respectively.
:param limit: default limit for the data.
:param offset: default offset for the data.
:param sort: a list of tuples of the form (key, direction).
:param fields: a list of fields to return, or None for all fields.
:param allowUnsorted: if true, and the entire data set will be returned
(rather than being restricted by limit), then
return the data unsorted.
:returns: a dictionary of results.
"""
query, sort, mfields = self.processParams(params, sort, fields)
logger.info('Query %r', ((query, offset, limit, sort, mfields), ))
cursor = None
if not offset and sort is not None and allowUnsorted:
cursor = self.trips.find(spec=query, skip=offset, limit=limit,
sort=None, timeout=False, fields=mfields,
manipulate=False, slave_okay=True,
compile_re=False)
total = cursor.count()
if limit and total >= limit:
cursor = None
if not cursor:
cursor = self.trips.find(spec=query, skip=offset, limit=limit,
sort=sort, timeout=False, fields=mfields,
manipulate=False, slave_okay=True,
compile_re=False)
total = cursor.count()
if fields:
columns = {fields[col]: col for col in xrange(len(fields))}
mcol = [self.KeyTable.get(fields[col], fields[col])
for col in xrange(len(fields))]
result = {
'count': total,
'format': 'list',
'fields': fields,
'columns': columns,
'data': [[row[k] for k in mcol] for row in cursor]
}
else:
result = {'count': total, 'data': [{
self.RevTable.get(k, k): v for k, v in row.items()
if k != '_id'}
for row in cursor
]}
return result
def removeContainer(self, user, container):
if container['ownerId'] != user['_id']:
raise AccessException("This container is not yours")
try:
self._stopContainer(container)
except:
# some of these things need to be properly synchronized
logger.info("Could not stop container")
self.remove(container)
def _sendmail(event):
msg = event.info['message']
smtpHost = ModelImporter.model('setting').get(SettingKey.SMTP_HOST,
'localhost')
logger.info('Sending email to %s through %s', msg['To'], smtpHost)
s = smtplib.SMTP(smtpHost)
s.sendmail(msg['From'], event.info['recipients'], msg.as_string())
s.quit()
logger.info('Sent email to %s', msg['To'])
def _runGCCommand(*args):
p = subprocess.run(list(args),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if p.returncode != 0:
msg = 'Command %s failed with exit code %s: \n%s\n%s' % (
args, p.returncode, p.stdout, p.stderr)
logger.warn(msg)
raise Exception(msg)
logger.info('Output from command %s: %s, %s' % (args, p.stdout, p.stderr))
return p
def _imageContentSearch(self, params):
limit = params['limit'] if 'limit' in params else '100'
if 'histogram' not in params:
features = ImageFeatures()
f = features.getImageFeatures({'url': params['url']})
params['histogram'] = json.dumps(f['histogram'])
logger.info('Using FLANN INDEX at ' +
os.environ['IMAGE_SPACE_FLANN_INDEX'])
return requests.get(os.environ['IMAGE_SPACE_FLANN_INDEX'] + '?query=' +
params['histogram'] + '&k=' + str(limit)).json()
def testOneFile(tempLog):
tempLog = configureLogging({'log_max_info_level': 'CRITICAL'}, oneFile=True)
logger.info(INFO_MSG)
infoSize = os.path.getsize(tempLog['info_log_file'])
errorSize = os.path.getsize(tempLog['error_log_file'])
assert infoSize == errorSize
logger.error(ERROR_MSG)
newInfoSize = os.path.getsize(tempLog['info_log_file'])
newErrorSize = os.path.getsize(tempLog['error_log_file'])
assert newInfoSize == newErrorSize
assert newInfoSize > infoSize
def __init__(self):
# If the config was already loaded, make sure we reload using our app's
# added files.
loadConfig()
# Use our logger values, not girder's
logConfig = girder.utility.config.getConfig().get('logging', {})
if 'log_root' in logConfig:
logConfig['log_root'] = os.path.expanduser(logConfig['log_root'])
for hdlr in logger.handlers[:]:
logger.removeHandler(hdlr)
girder._setupLogger()
logger.info('Minerva Taxi starting')
def testOneFile(self):
self.configureLogging({'log_max_info_level': 'CRITICAL'}, oneFile=True)
logger.info(self.infoMessage)
infoSize = os.path.getsize(self.infoFile)
errorSize = os.path.getsize(self.errorFile)
self.assertEqual(infoSize, errorSize)
logger.error(self.errorMessage)
newInfoSize = os.path.getsize(self.infoFile)
newErrorSize = os.path.getsize(self.errorFile)
self.assertEqual(newInfoSize, newErrorSize)
self.assertGreater(newInfoSize, infoSize)
def testOneFile(tempLog):
tempLog = configureLogging({'log_max_info_level': 'CRITICAL'}, oneFile=True)
logger.info(INFO_MSG)
infoSize = os.path.getsize(tempLog['info_log_file'])
errorSize = os.path.getsize(tempLog['error_log_file'])
assert infoSize == errorSize
logger.error(ERROR_MSG)
newInfoSize = os.path.getsize(tempLog['info_log_file'])
newErrorSize = os.path.getsize(tempLog['error_log_file'])
assert newInfoSize == newErrorSize
assert newInfoSize > infoSize
def testOneFile(self):
self.configureLogging({'log_max_info_level': 'CRITICAL'}, oneFile=True)
logger.info(self.infoMessage)
infoSize = os.path.getsize(self.infoFile)
errorSize = os.path.getsize(self.errorFile)
self.assertEqual(infoSize, errorSize)
logger.error(self.errorMessage)
newInfoSize = os.path.getsize(self.infoFile)
newErrorSize = os.path.getsize(self.errorFile)
self.assertEqual(newInfoSize, newErrorSize)
self.assertGreater(newInfoSize, infoSize)
def load(self, info):
events.bind('no_password_login_attempt', 'openid', _checkOpenIdUser)
info['apiRoot'].openid = rest.OpenId()
SettingDefault.defaults[constants.PluginSettings.PROVIDERS] = []
if 'GIRDER_REQUESTS_VERIFY' in os.environ:
path = os.environ['GIRDER_REQUESTS_VERIFY']
if not os.path.isfile(path):
raise Exception('Requests cert not found: %s' % path)
# We use a custom fetcher class and set it as the default to support customization
# of the "verify" parameter of the requests session
logger.info('OpenID: using verify value=%s' % path)
setDefaultFetcher(_CustomCAFetcher(path), wrap_exceptions=False)
def getYAMLConfigFile(self, folder, name):
addConfig = None
user = self.getCurrentUser()
last = False
while folder:
item = Item().findOne({'folderId': folder['_id'], 'name': name})
if item:
for file in Item().childFiles(item):
if file['size'] > 10 * 1024**2:
logger.info('Not loading %s -- too large' % file['name'])
continue
with File().open(file) as fptr:
config = yaml.safe_load(fptr)
# combine and adjust config values based on current user
if isinstance(
config,
dict) and 'access' in config or 'group' in config:
config = adjustConfigForUser(config, user)
if addConfig and isinstance(config, dict):
config = _mergeDictionaries(config, addConfig)
if not isinstance(config, dict) or config.get(
'__inherit__') is not True:
return config
config.pop('__inherit__')
addConfig = config
if last:
break
if folder['parentCollection'] != 'folder':
if folder['name'] != '.config':
folder = Folder().findOne({
'parentId':
folder['parentId'],
'parentCollection':
folder['parentCollection'],
'name':
'.config'
})
else:
last = 'setting'
if not folder or last == 'setting':
folderId = Setting().get(
constants.PluginSettings.LARGE_IMAGE_CONFIG_FOLDER)
if not folderId:
break
folder = Folder().load(folderId, force=True)
last = True
else:
folder = Folder().load(folder['parentId'],
user=user,
level=AccessType.READ)
return addConfig
def _imageContentSearch(self, params):
limit = params['limit'] if 'limit' in params else '100'
logger.info(
'Using COLUMBIA INDEX at ' +
os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] +
'?url=' + params['url'] +
'&num=' + str(limit))
return [{'id': d} for d in requests.get(
os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] +
'?url=' + params['url'] +
'&num=' + str(limit),
verify=False
).json()['images'][0]['similar_images']['cached_image_urls']]
def createThumbnailsJob(job):
"""
Create thumbnails for all of the large image items.
:param spec: an array, each entry of which is the parameter dictionary for
the model getThumbnail function.
"""
jobModel = ModelImporter.model('job', 'jobs')
job = jobModel.updateJob(
job, log='Started creating large image thumbnails\n',
status=JobStatus.RUNNING)
checkedOrCreated = 0
try:
spec = job['kwargs']['spec']
logInterval = float(job['kwargs'].get('logInterval', 10))
itemModel = ModelImporter.model('item')
imageItemModel = ModelImporter.model('image_item', 'large_image')
for entry in spec:
job = jobModel.updateJob(
job, log='Creating thumbnails for %r\n' % entry)
lastLogTime = time.time()
items = itemModel.find({'largeImage.fileId': {'$exists': True}})
for item in items:
imageItemModel.getThumbnail(item, **entry)
checkedOrCreated += 1
# Periodically, log the state of the job and check if it was
# deleted or canceled.
if time.time() - lastLogTime > logInterval:
job = jobModel.updateJob(
job, log='Checked or created %d thumbnail file%s\n' % (
checkedOrCreated,
's' if checkedOrCreated != 1 else ''))
lastLogTime = time.time()
# Check if the job was deleted or canceled; if so, quit
job = jobModel.load(id=job['_id'], force=True)
if not job or job['status'] == JobStatus.CANCELED:
logger.info('Large image thumbnails job %s' % (
'deleted' if not job else 'canceled'))
return
except Exception:
logger.exception('Error with large image create thumbnails job')
job = jobModel.updateJob(
job, log='Error creating large image thumbnails\n',
status=JobStatus.ERROR)
return
msg = 'Finished creating large image thumbnails (%d checked or created)' % (
checkedOrCreated)
logger.info(msg)
job = jobModel.updateJob(job, log=msg + '\n', status=JobStatus.SUCCESS)
def testInfoMaxLevel(tempLog):
tempLog = configureLogging({'log_max_info_level': 'CRITICAL'})
infoSize1 = os.path.getsize(tempLog['info_log_file'])
errorSize1 = os.path.getsize(tempLog['error_log_file'])
logger.info(INFO_MSG)
infoSize2 = os.path.getsize(tempLog['info_log_file'])
errorSize2 = os.path.getsize(tempLog['error_log_file'])
assert infoSize2 > infoSize1
assert errorSize2 == errorSize1
logger.error(ERROR_MSG)
infoSize3 = os.path.getsize(tempLog['info_log_file'])
errorSize3 = os.path.getsize(tempLog['error_log_file'])
assert infoSize3 > infoSize2
assert errorSize3 > errorSize2
def _imageContentSearch(self, params):
limit = params['limit'] if 'limit' in params else '100'
if 'histogram' not in params:
features = ImageFeatures()
f = features.getImageFeatures({'url': params['url']})
params['histogram'] = json.dumps(f['histogram'])
logger.info(
'Using FLANN INDEX at ' +
os.environ['IMAGE_SPACE_FLANN_INDEX'])
return requests.get(
os.environ['IMAGE_SPACE_FLANN_INDEX'] +
'?query=' + params['histogram'] +
'&k=' + str(limit)).json()
def testInfoMaxLevel(self):
self.configureLogging({'log_max_info_level': 'CRITICAL'})
infoSize1 = os.path.getsize(self.infoFile)
errorSize1 = os.path.getsize(self.errorFile)
logger.info(self.infoMessage)
infoSize2 = os.path.getsize(self.infoFile)
errorSize2 = os.path.getsize(self.errorFile)
self.assertGreater(infoSize2, infoSize1)
self.assertEqual(errorSize2, errorSize1)
logger.error(self.errorMessage)
infoSize3 = os.path.getsize(self.infoFile)
errorSize3 = os.path.getsize(self.errorFile)
self.assertGreater(infoSize3, infoSize2)
self.assertGreater(errorSize3, errorSize2)
def _imageContentSearch(self, params):
setting = ColumbiaSetting()
limit = params['limit'] if 'limit' in params else '100'
logger.info(
'Using COLUMBIA INDEX at ' +
setting.get('IMAGE_SPACE_COLUMBIA_INDEX') +
'?url=' + params['url'] +
'&num=' + str(limit))
return [{'id': d} for d in requests.get(
setting.get('IMAGE_SPACE_COLUMBIA_INDEX') +
'?url=' + params['url'] +
'&num=' + str(limit),
verify=False
).json()['images'][0]['similar_images']['cached_image_urls']]
def _submitEmail(msg, recipients):
from girder.models.setting import Setting
setting = Setting()
smtp = _SMTPConnection(host=setting.get(SettingKey.SMTP_HOST),
port=setting.get(SettingKey.SMTP_PORT),
encryption=setting.get(SettingKey.SMTP_ENCRYPTION),
username=setting.get(SettingKey.SMTP_USERNAME),
password=setting.get(SettingKey.SMTP_PASSWORD))
logger.info('Sending email to %s through %s', ', '.join(recipients),
smtp.host)
with smtp:
smtp.send(msg['From'], recipients, msg.as_string())
def testInfoMaxLevel(tempLog):
tempLog = configureLogging({'log_max_info_level': 'CRITICAL'})
infoSize1 = os.path.getsize(tempLog['info_log_file'])
errorSize1 = os.path.getsize(tempLog['error_log_file'])
logger.info(INFO_MSG)
infoSize2 = os.path.getsize(tempLog['info_log_file'])
errorSize2 = os.path.getsize(tempLog['error_log_file'])
assert infoSize2 > infoSize1
assert errorSize2 == errorSize1
logger.error(ERROR_MSG)
infoSize3 = os.path.getsize(tempLog['info_log_file'])
errorSize3 = os.path.getsize(tempLog['error_log_file'])
assert infoSize3 > infoSize2
assert errorSize3 > errorSize2
def _sendmail(event):
msg = event.info['message']
recipients = event.info['recipients']
setting = ModelImporter.model('setting')
smtp = _SMTPConnection(
host=setting.get(SettingKey.SMTP_HOST, 'localhost'),
port=setting.get(SettingKey.SMTP_PORT, None),
encryption=setting.get(SettingKey.SMTP_ENCRYPTION, 'none'),
username=setting.get(SettingKey.SMTP_USERNAME, None),
password=setting.get(SettingKey.SMTP_PASSWORD, None)
)
logger.info('Sending email to %s through %s',
', '.join(recipients), smtp.host)
with smtp:
smtp.send(msg['From'], recipients, msg.as_string())
def _sendmail(event):
from girder.models.setting import Setting
msg = event.info['message']
recipients = event.info['recipients']
setting = Setting()
smtp = _SMTPConnection(
host=setting.get(SettingKey.SMTP_HOST),
port=setting.get(SettingKey.SMTP_PORT),
encryption=setting.get(SettingKey.SMTP_ENCRYPTION),
username=setting.get(SettingKey.SMTP_USERNAME),
password=setting.get(SettingKey.SMTP_PASSWORD)
)
logger.info('Sending email to %s through %s', ', '.join(recipients), smtp.host)
with smtp:
smtp.send(msg['From'], recipients, msg.as_string())
def _imageSearch(self, params):
limit = params['limit'] if 'limit' in params else '10'
if 'histogram' in params:
if 'IMAGE_SPACE_FLANN_INDEX' in os.environ:
logger.info('Using FLANN INDEX at ' + os.environ['IMAGE_SPACE_FLANN_INDEX'])
return requests.get(
os.environ['IMAGE_SPACE_FLANN_INDEX'] +
'?query=' + params['histogram'] + '&k=' + str(limit)).json()
logger.info('Using COLUMBIA INDEX at '+os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] + '?url=' + params['url'] + '&num=' + str(limit))
return [{'id' : d} for d in requests.get(
os.environ['IMAGE_SPACE_COLUMBIA_INDEX'] +
'?url=' + params['url'] + '&num=' + str(limit), verify=False).json()['images'][0]['similar_images']['image_urls']]
query = params['query'] if 'query' in params else '*'
base = os.environ['IMAGE_SPACE_SOLR'] + '/select?wt=json&indent=true'
try:
result = requests.get(base + '&q=' + query + '&rows=' + str(limit), verify=False).json()
except ValueError:
return []
return result['response']['docs']
def sendEmail(to, subject, text):
if isinstance(to, str):
to = (to,)
msg = MIMEText(text, 'html')
msg['Subject'] = subject or '[no subject]'
msg['To'] = ', '.join(to)
msg['From'] = Setting().get(SettingKey.EMAIL_FROM_ADDRESS)
recipients = list(set(to))
smtp = mail_utils._SMTPConnection(
host=Setting().get(SettingKey.SMTP_HOST, 'localhost'),
port=Setting().get(SettingKey.SMTP_PORT, None),
encryption=Setting().get(SettingKey.SMTP_ENCRYPTION, 'none'),
username=Setting().get(SettingKey.SMTP_USERNAME, None),
password=Setting().get(SettingKey.SMTP_PASSWORD, None)
)
logger.info('Sending email to %s through %s', ', '.join(recipients), smtp.host)
with smtp:
smtp.send(msg['From'], recipients, msg.as_string())
def recalculateSize(self, item):
"""
Recalculate the item size based on the files that are in it. If this
is different than the recorded size, propagate the changes.
:param item: The item to recalculate the size of.
:returns: the recalculated size in bytes
"""
size = 0
for file in self.childFiles(item):
# We could add a recalculateSize to the file model, in which case
# this would be:
# size += self.model('file').recalculateSize(file)
size += file.get('size', 0)
delta = size-item.get('size', 0)
if delta:
logger.info('Item %s was wrong size: was %d, is %d' % (
item['_id'], item['size'], size))
item['size'] = size
self.update({'_id': item['_id']}, update={'$set': {'size': size}})
self.propagateSizeChange(item, delta)
return size
def addDeps(plugin):
if plugin not in allPlugins:
message = 'Required plugin %s does not exist.' % plugin
if ignoreMissing:
logprint.error(message)
return
else:
raise ValidationException(message)
deps = set()
for key in keys:
deps |= allPlugins[plugin][key]
dag[plugin] = deps
for dep in deps:
if dep in visited:
continue
visited.add(dep)
if dep not in plugins:
logger.info('Adding plugin %s because %s requires it' % (dep, plugin))
addDeps(dep)
def __new__(metacls, name, bases, namespace, **kwargs): # noqa - N804
# Get metaclass parameters by finding and removing them from the class
# namespace (necessary for Python 2), or preferentially as metaclass
# arguments (only in Python 3).
cacheName = namespace.get('cacheName', None)
cacheName = kwargs.get('cacheName', cacheName)
maxSize = CacheProperties.get(cacheName, {}).get('cacheMaxSize', None)
maxSize = namespace.pop('cacheMaxSize', maxSize)
maxSize = kwargs.get('cacheMaxSize', maxSize)
if maxSize is None:
raise TypeError('Usage of the LruCacheMetaclass requires a '
'"cacheMaxSize" attribute on the class %s.' % name)
timeout = CacheProperties.get(cacheName, {}).get('cacheTimeout', None)
timeout = namespace.pop('cacheTimeout', timeout)
timeout = kwargs.get('cacheTimeout', timeout)
cls = super(LruCacheMetaclass, metacls).__new__(
metacls, name, bases, namespace)
if not cacheName:
cacheName = cls
if LruCacheMetaclass.namedCaches.get(cacheName) is None:
cache, cacheLock = CacheFactory().getCache(maxSize)
LruCacheMetaclass.namedCaches[cacheName] = cache
logger.info('Created LRU Cache for %r with %d maximum size' % (
cacheName, maxSize))
else:
cache = LruCacheMetaclass.namedCaches[cacheName]
# Don't store the cache in cls.__dict__, because we don't want it to be
# part of the attribute lookup hierarchy
# TODO: consider putting it in cls.__dict__, to inspect statistics
# cls is hashable though, so use it to lookup the cache, in case an
# identically-named class gets redefined
LruCacheMetaclass.classCaches[cls] = cache
return cls
def _wait_for_server(url, timeout=30, wait_time=0.5):
"""Wait for a server to show up within a newly launched instance."""
tic = time.time()
# Fudge factor of IPython notebook bootup.
time.sleep(0.5)
while time.time() - tic < timeout:
try:
urlopen(url, timeout=1)
except HTTPError as err:
logger.info(
'Booting server at [%s], getting HTTP status [%s]',
url, err.code)
time.sleep(wait_time)
except URLError as err:
logger.info(
'Booting server at [%s], getting URLError due to [%s]',
url, err.reason)
except (ssl.SSLError, ssl.CertificateError):
time.sleep(wait_time)
else:
break
def _processImages(folder, itemFilePairs):
"""
Create and schedule a Girder job for processing images.
:param folder: Folder to process images for.
:param fileIds: File IDs to process, these are converted into Girder data elemnts.
"""
jobModel = ModelImporter.model('job', 'jobs')
# TODO Use a more granular token.
# Ideally this would be scoped to only allow:
# - Job Updates
# - Data management of folder
# - Retrieval of SMQTK settings
token = ModelImporter.model('token').createToken(user=getCurrentUser(),
days=1,
scope=(TokenScope.USER_AUTH,
SMQTK_SETTING_READ))
dataElementUris = [(itemId, 'girder://*****:*****@%s/file/%s' % (token['_id'],
getWorkerApiUrl(),
fileId))
for (itemId, fileId) in itemFilePairs]
job = jobModel.createJob(title='Processing Images',
type='GPU',
handler='worker_handler',
user=getCurrentUser(),
args=(str(folder['_id']), dataElementUris),
otherFields={'celeryTaskName': 'smqtk_worker.tasks.process_images',
'celeryQueue': 'process-images'})
job['token'] = token
logger.info('assigning token %s' % token['_id'])
jobModel.save(job)
jobModel.scheduleJob(job)
def connectToMongoCollection(self, params):
self.requireParams(('host', 'port', 'database', 'collection'), params)
host = params['host']
port = params['port']
try:
port = int(port)
except ValueError:
raise RestException('Invalid port number')
try:
client = pymongo.MongoClient(host=host, port=port)
except pymongo.errors.ConnectionFailure:
raise RestException('Could not connect to mounted database')
dbName = params['database']
if not dbName in client.database_names():
raise RestException('Database does not exist')
db = client[dbName]
logger.info('Connected to mongo collection at %s:%i/%s/%s' % (host, port, dbName, params['collection']))
return db[params['collection']]
def getTableList(cls, uri, internalTables=False, dbparams={}, **kwargs):
"""
Get a list of known databases, each of which has a list of known tables
from the database. This is of the form [{'database': (database),
'tables': [{'schema': (schema), 'table': (table 1)}, ...]}]
:param uri: uri to connect to the database.
:param internaltables: True to return tables about the database itself.
:param dbparams: optional parameters to send to the connection.
:returns: A list of known tables.
"""
dbEngine = sqlalchemy.create_engine(cls.adjustDBUri(uri), **dbparams)
insp = sqlalchemy.engine.reflection.Inspector.from_engine(dbEngine)
schemas = insp.get_schema_names()
defaultSchema = insp.default_schema_name
tables = [{'name': table, 'table': table}
for table in dbEngine.table_names()]
tables.extend([{'name': view, 'table': view}
for view in insp.get_view_names()])
databaseName = base.databaseFromUri(uri)
results = [{'database': databaseName, 'tables': tables}]
if len(schemas) <= MAX_SCHEMAS_IN_TABLE_LIST:
for schema in schemas:
if not internalTables and schema.lower() == 'information_schema':
continue
if schema != defaultSchema:
tables = [{'name': '%s.%s' % (schema, table),
'table': table, 'schema': schema}
for table in dbEngine.table_names(schema=schema)]
tables.extend([{'name': '%s.%s' % (schema, view),
'table': view, 'schema': schema}
for view in insp.get_view_names(schema=schema)])
results[0]['tables'].extend(tables)
else:
log.info('Not enumerating all schemas for table list (%d schemas)', len(schemas))
return results
def cullNotebooks(self):
resp = requests.get(
self.model('setting').get(PluginSettings.TMPNB_URL))
content = resp.content
if isinstance(content, six.binary_type):
content = content.decode('utf8')
try:
resp.raise_for_status()
except requests.HTTPError:
raise RestException(
'Got %s code from tmpnb, response="%s"/' % (
resp.status_code, content
), code=502)
try:
activity = json.loads(content)
except ValueError:
raise RestException('Non-JSON response: %s' % content, code=502)
admin = next(_ for _ in self.model('user').getAdmins())
token = self.model('token').createToken(user=admin, days=1)
# Iterate over all notebooks, not the prettiest way...
cull_period = self.model('setting').get(
PluginSettings.CULLING_PERIOD, '4')
cull_time = datetime.datetime.utcnow() - \
datetime.timedelta(hours=float(cull_period))
for nb in self.find({}):
try:
last_activity = dateutil.parser.parse(
activity[nb['containerId']], ignoretz=True)
if last_activity < cull_time:
logger.info('Deleting nb %s' % nb['_id'])
self.deleteNotebook(nb, token)
except KeyError:
# proxy is not aware of such container, kill it...
logger.info('Deleting nb %s' % nb['_id'])
self.deleteNotebook(nb, token)
def testFileRotation(self):
self.configureLogging({
'log_access': ['screen', 'info'],
'log_quiet': True,
'log_max_size': '1 kb',
'log_backup_count': 2,
'log_level': 'DEBUG',
})
logger.info(self.infoMessage)
logger.error(self.errorMessage)
infoSize = os.path.getsize(self.infoFile)
errorSize = os.path.getsize(self.errorFile)
self.assertFalse(os.path.exists(self.infoFile + '.1'))
self.assertFalse(os.path.exists(self.errorFile + '.1'))
logger.info(self.infoMessage)
logger.error(self.errorMessage)
newInfoSize = os.path.getsize(self.infoFile)
newErrorSize = os.path.getsize(self.errorFile)
deltaInfo = newInfoSize - infoSize
deltaError = newErrorSize - errorSize
self.assertGreater(deltaInfo, len(self.infoMessage))
self.assertGreater(deltaError, len(self.errorMessage))
while newInfoSize < 1024 * 1.5:
logger.info(self.infoMessage)
newInfoSize += deltaInfo
while newErrorSize < 1024 * 1.5:
logger.error(self.errorMessage)
newErrorSize += deltaError
self.assertTrue(os.path.exists(self.infoFile + '.1'))
self.assertTrue(os.path.exists(self.errorFile + '.1'))
self.assertFalse(os.path.exists(self.infoFile + '.2'))
self.assertFalse(os.path.exists(self.errorFile + '.2'))
while newInfoSize < 1024 * 3.5:
logger.info(self.infoMessage)
newInfoSize += deltaInfo
while newErrorSize < 1024 * 3.5:
logger.error(self.errorMessage)
newErrorSize += deltaError
self.assertTrue(os.path.exists(self.infoFile + '.1'))
self.assertTrue(os.path.exists(self.errorFile + '.1'))
self.assertTrue(os.path.exists(self.infoFile + '.2'))
self.assertTrue(os.path.exists(self.errorFile + '.2'))
self.assertFalse(os.path.exists(self.infoFile + '.3'))
self.assertFalse(os.path.exists(self.errorFile + '.3'))
def testFileRotation(tempLog):
tempLog = configureLogging({
'log_access': ['screen', 'info'],
'log_quiet': True,
'log_max_size': '1 kb',
'log_backup_count': 2,
'log_level': 'DEBUG',
})
logger.info(INFO_MSG)
logger.error(ERROR_MSG)
infoSize = os.path.getsize(tempLog['info_log_file'])
errorSize = os.path.getsize(tempLog['error_log_file'])
assert os.path.exists(tempLog['info_log_file'] + '.1') is False
assert os.path.exists(tempLog['error_log_file'] + '.1') is False
logger.info(INFO_MSG)
logger.error(ERROR_MSG)
newInfoSize = os.path.getsize(tempLog['info_log_file'])
newErrorSize = os.path.getsize(tempLog['error_log_file'])
deltaInfo = newInfoSize - infoSize
deltaError = newErrorSize - errorSize
assert deltaInfo > len(INFO_MSG)
assert deltaError > len(ERROR_MSG)
while newInfoSize < 1024 * 1.5:
logger.info(INFO_MSG)
newInfoSize += deltaInfo
while newErrorSize < 1024 * 1.5:
logger.error(ERROR_MSG)
newErrorSize += deltaError
assert os.path.exists(tempLog['info_log_file'] + '.1') is True
assert os.path.exists(tempLog['error_log_file'] + '.1') is True
assert os.path.exists(tempLog['info_log_file'] + '.2') is False
assert os.path.exists(tempLog['error_log_file'] + '.2') is False
while newInfoSize < 1024 * 3.5:
logger.info(INFO_MSG)
newInfoSize += deltaInfo
while newErrorSize < 1024 * 3.5:
logger.error(ERROR_MSG)
newErrorSize += deltaError
assert os.path.exists(tempLog['info_log_file'] + '.1') is True
assert os.path.exists(tempLog['error_log_file'] + '.1') is True
assert os.path.exists(tempLog['info_log_file'] + '.2') is True
assert os.path.exists(tempLog['error_log_file'] + '.2') is True
assert os.path.exists(tempLog['info_log_file'] + '.3') is False
assert os.path.exists(tempLog['error_log_file'] + '.3') is False
def refine(self, params):
sid = str(params['item']['_id'])
pos_uuids = params['pos_uuids']
neg_uuids = params['neg_uuids'] if params['neg_uuids'] is not None else []
if len(pos_uuids) == 0:
raise RestException('No positive UUIDs given.')
with self.controller:
if not self.controller.has_session_uuid(sid):
raise RestException('Session ID %s not found.' % sid, 404)
iqrs = self.controller.get_session(sid)
iqrs.lock.acquire() # lock BEFORE releasing controller
try:
descriptor_index = self._descriptorIndexFromSessionId(sid)
neighbor_index = self._nearestNeighborIndex(sid, descriptor_index)
if descriptor_index is None or neighbor_index is None:
logger.error('Unable to compute descriptor or neighbor index from sid %s.' % sid)
raise RestException('Unable to compute descriptor or neighbor index from sid %s.' % sid, 500)
# Get appropriate descriptor elements from index for
# setting new adjudication state.
try:
pos_descrs = set(descriptor_index.get_many_descriptors(pos_uuids))
neg_descrs = set(descriptor_index.get_many_descriptors(neg_uuids))
except KeyError as ex:
logger.warn(traceback.format_exc())
raise RestException('Descriptor UUID %s not found in index.' % ex, 404)
# if a new classifier should be made upon the next
# classification request.
diff_pos = pos_descrs.symmetric_difference(iqrs.positive_descriptors)
diff_neg = neg_descrs.symmetric_difference(iqrs.negative_descriptors)
if diff_pos or diff_neg:
logger.debug("[%s] session Classifier dirty", sid)
self.session_classifier_dirty[sid] = True
logger.info("[%s] Setting adjudications", sid)
iqrs.positive_descriptors = pos_descrs
iqrs.negative_descriptors = neg_descrs
logger.info("[%s] Updating working index", sid)
iqrs.update_working_index(neighbor_index)
logger.info("[%s] Refining", sid)
iqrs.refine()
finally:
iqrs.lock.release()
return sid
def uploadChunk(self, upload, chunk):
"""
Stores the uploaded chunk in fixed-sized pieces in the chunks
collection of this assetstore's database.
"""
# If we know the chunk size is too large or small, fail early.
self.checkUploadSize(upload, self.getChunkSize(chunk))
if isinstance(chunk, six.text_type):
chunk = chunk.encode('utf8')
if isinstance(chunk, six.binary_type):
chunk = BytesIO(chunk)
# Restore the internal state of the streaming SHA-512 checksum
checksum = sha512_state.restoreHex(upload['sha512state'])
# This bit of code will only do anything if there is a discrepancy
# between the received count of the upload record and the length of
# the file stored as chunks in the database. This code simply updates
# the sha512 state with the difference before reading the bytes sent
# from the user.
if self.requestOffset(upload) > upload['received']:
cursor = self.chunkColl.find({
'uuid': upload['chunkUuid'],
'n': {'$gte': upload['received'] // CHUNK_SIZE}
}, fields=['data']).sort('n', pymongo.ASCENDING)
for result in cursor:
checksum.update(result['data'])
cursor = self.chunkColl.find({
'uuid': upload['chunkUuid']
}, fields=['n']).sort('n', pymongo.DESCENDING).limit(1)
if cursor.count(True) == 0:
n = 0
else:
n = cursor[0]['n'] + 1
size = 0
startingN = n
while not upload['received']+size > upload['size']:
data = chunk.read(CHUNK_SIZE)
if not data:
break
# If a timeout occurs while we are trying to load data, we might
# have succeeded, in which case we will get a DuplicateKeyError
# when it automatically retries. Therefore, log this error but
# don't stop.
try:
self.chunkColl.insert({
'n': n,
'uuid': upload['chunkUuid'],
'data': bson.binary.Binary(data)
})
except pymongo.errors.DuplicateKeyError:
logger.info('Received a DuplicateKeyError while uploading, '
'probably because we reconnected to the database '
'(chunk uuid %s part %d)', upload['chunkUuid'], n)
n += 1
size += len(data)
checksum.update(data)
chunk.close()
try:
self.checkUploadSize(upload, size)
except ValidationException:
# The user tried to upload too much or too little. Delete
# everything we added
self.chunkColl.remove({'uuid': upload['chunkUuid'],
'n': {'$gte': startingN}}, multi=True)
raise
# Persist the internal state of the checksum
upload['sha512state'] = sha512_state.serializeHex(checksum)
upload['received'] += size
return upload
def performSelect(self, fields, queryProps={}, filters=[], client=None):
"""
Select data from the database. The results are passed back as a
dictionary with the following values:
limit: the limit used in the query
offset: the offset used in the query
sort: the list of sort parameters used in the query.
fields: a list of the fields that are being returned in the order
that they are returned.
data: a list with one entry per row of results. Each entry is a list
with one entry per column.
:param fields: the results from getFieldInfo.
:param queryProps: general query properties, including limit, offset,
and sort.
:param filters: a list of filters to apply.
:param client: if a client is specified, a previous query made by this
client can be cancelled.
:return: the results of the query. See above.
"""
result = super(MongoConnector, self).performSelect(
fields, queryProps, filters)
if queryProps.get('group'):
raise DatabaseConnectorException(
'Group unsupported by this database.')
filterQueryClauses = []
for filt in filters:
filterQueryClauses = self._addFilter(filterQueryClauses, filt)
opts = {}
for k, v in six.iteritems(queryProps):
target = None
if k == 'fields' and v and v != []:
target = 'projection'
v = {field: True for field in v}
if '_id' not in v:
v['_id'] = False
elif k == 'offset':
target = 'skip'
elif k in ['limit', 'no_cursor_timeout', 'cursor_type', 'sort',
'allow_partial_results', 'oplog_replay', 'modifiers']:
target = k
if target is not None:
opts[target] = v
if len(filterQueryClauses) > 0:
opts['filter'] = {'$and': filterQueryClauses}
result['format'] = 'dict'
if queryProps.get('limit') == 0:
result['data'] = []
else:
if queryProps.get('limit') < 0:
opts['limit'] = 0
coll = self.connect()
log.info('Query: %s', bson.json_util.dumps(
opts, check_circular=False, separators=(',', ':'),
sort_keys=False, default=str, indent=None))
cursor = coll.find(**opts)
result['datacount'] = cursor.count(True)
result['data'] = cursor
self.disconnect()
return result
