def queryThisContentHelper(request, response, content):
request.context['logger'].debug('querying the database with the data {}'.format(content))
request.context['logger'].debug(' ---> headers: {}'.format(request.headers))
customHeaders = {}
utils.getCustomHeaders(request.headers, customHeaders)
if customHeaders.get('contentDeliveryMethod') == 'chunk':
## This routes the same as if requested through ./query/cache
storeQueryResults(request, response, customHeaders, content)
else:
## Work on it now, inside the runtime of this thread
queryProcessingInstance = QueryProcessing(request.context['logger'], request.context['dbSession'], content, customHeaders['removePrivateAttributes'], customHeaders['removeEmptyAttributes'], customHeaders['resultsFormat'])
## Message any query parsing errors back to the client so they
## can see what to fix (e.g. 'betweendate' time formats)
if len(queryProcessingInstance.errorList) > 0:
for msg in queryProcessingInstance.errorList:
request.context['payload']['errors'].append(msg)
response.status = HTTP_400
else:
listOrDictResult = queryProcessingInstance.runQuery()
request.context['logger'].debug('listOrDictResult {}'.format(listOrDictResult))
request.context['payload'] = listOrDictResult
request.context['logger'].debug('payload charCount {}'.format(len(str(request.context['payload']))))
## end queryThisContentHelper
return
def countGivenClassHelper(request, response, className):
"""Count instances in given class type, including counts of subtypes.
Example. Given the following scenario:
* 5 base Nodes (not sub-typed)
* 25481 Linux (subtyped from Node, through NodeServer, to Linux)
* 24529 Windows (similarly subtyped Node-> NodeServer-> Windows)
./tool/count/Node:
Standard counts include full counts of all involved types::
{
"Node": 50015,
"NodeServer": 50010,
"Linux": 25481,
"Windows": 24529
}
./tool/countExclusive/Node:
And in contrast, exclusive counts are unique to the lowest subtypes::
{
"Node": 5,
"Linux": 25481,
"Windows": 24529
}
"""
customHeaders = {}
utils.getCustomHeaders(request.headers, customHeaders)
removeEmptyAttributes = customHeaders['removeEmptyAttributes']
validClassObjects = dict()
utils.getValidClassObjects(validClassObjects)
utils.getValidLinksV2(validClassObjects)
classNameMap = dict()
for item in validClassObjects:
classNameMap[item] = validClassObjects[item]['classObject'].__tablename__
if className not in validClassObjects.keys():
request.context['logger'].error('Received unrecognized object class_name: {}'.format(className))
request.context['payload']['errors'].append('Invalid Class Name')
response.status = HTTP_404
else:
tableName = 'data.'+classNameMap[className]
tableObj= tableMapping.Base.metadata.tables[tableName]
stment = select([func.count()]).select_from(tableObj)
val = request.context['dbSession'].execute(stment).first()
request.context['payload'][className] = val[0]
if validClassObjects[className]['children']:
for item in validClassObjects[className]['children']:
tableName = 'data.'+classNameMap[item]
tableObj= tableMapping.Base.metadata.tables[tableName]
stment = select([func.count()]).select_from(tableObj)
val = request.context['dbSession'].execute(stment).first()
if not removeEmptyAttributes or val[0] != 0:
request.context['payload'][item] = val[0]
request.context['dbSession'].close()
## end countGivenClassHelper
return
def countGivenClassExclusiveHelper(request, response, className):
"""Count instances in given class, excluding instances of any subtypes.
Example. Given the following scenario:
* 5 base Nodes (not sub-typed)
* 25481 Linux (subtyped from Node, through NodeServer, to Linux)
* 24529 Windows (similarly subtyped Node-> NodeServer-> Windows)
./tool/countExclusive/Node:
Exclusive counts return numbers unique to the lowest subtypes::
{
"Node": 5,
"Linux": 25481,
"Windows": 24529
}
./tool/count/Node:
And in contrast, standard counts includes full counts of all involved types::
{
"Node": 50015,
"NodeServer": 50010,
"Linux": 25481,
"Windows": 24529
}
"""
customHeaders = {}
utils.getCustomHeaders(request.headers, customHeaders)
validClassObjects = dict()
utils.getValidClassObjects(validClassObjects)
request.context['logger'].info('validClassObjects: {}'.format(validClassObjects))
request.context['logger'].info('validClassObjects keys: {}'.format(validClassObjects.keys()))
utils.getValidLinksV2(validClassObjects)
classNameMap = dict()
for item in validClassObjects:
classNameMap[validClassObjects[item]['classObject'].__tablename__] = item
request.context['logger'].info('classNameMap: {}'.format(classNameMap))
request.context['logger'].info('classNameMap keys: {}'.format(classNameMap.keys()))
if className not in validClassObjects.keys():
request.context['logger'].error('Received unrecognized object class_name: {}'.foramt(className))
request.context['payload']['errors'].append('Invalid Class Name')
response.status = HTTP_404
else:
ResultSet = request.context['dbSession'].query(func.count(validClassObjects[className]['classObject'].object_id), validClassObjects[className]['classObject'].object_type).group_by(validClassObjects[className]['classObject'].object_type).all()
for item in ResultSet:
request.context['payload'][classNameMap[item[1]]] = item[0]
if not customHeaders['removeEmptyAttributes']:
for item in validClassObjects[className]['children']:
if item not in request.context['payload'].keys():
request.context['payload'][item] = 0
## end countGivenClassExclusiveHelper
return
def deleteTaskContentHelper(request, response, content):
customHeaders = {}
utils.getCustomHeaders(request.headers, customHeaders)
request.context['logger'].debug('Deleting content in ./task request from database')
deleteDbContent = DeleteDbObject(request.context['logger'], request.context['dbSession'], content)
if customHeaders['resultsFormat'] == 'Flat':
deletedObjects = deleteDbContent.queryBeforeDeletion()
else:
deletedObjects = deleteDbContent.wrapNested()
payload = deletedObjects
## end deleteTaskContentHelper
return
def createStoredQueryResult(content:hugJson, request, response):
"""Query and store results matching the provided JSON."""
try:
request.context['logger'].debug('querying the database with the data {}'.format(content))
request.context['logger'].debug(' ---> headers: {}'.format(request.headers))
customHeaders = {}
utils.getCustomHeaders(request.headers, customHeaders)
storeQueryResults(request, response, customHeaders, content)
except:
errorMessage(request, response)
## end createStoredQueryResult
return cleanPayload(request)
def deleteExecutingGivenQueries(content:hugJson, request, response):
"""Remove the results of the add-hoc query defined in the content section.
TODO - change the way this works.
Deletes are an incredibly slow ORM operation on inheritance based classes.
Instead of a blocking, single threaded REST operation... break out into a
queued delete operation. Result sizes over something small like 10 objects,
should be split up and sent back out to kafka to be processed across all
available ResultProcessing clients.
Consider something like the cached/chunked functions, where client is given
an id to use to continue checking back with, to see the progress and when
the requested delete operation has fully completed.
Every so often (increasing sleep times 5 sec, 10, 30, 60 - hold at 60?) we
re-run the query as a GET operation and count the objects... adjust estimate
on approximate runtime and completion datetime with expected load on current
number of resultProcessing clients; suggest spinning up more to get done
in less time.
"""
try:
request.context['payload'].clear()
queryThisContentHelper(request, response, content)
## Flat and Nested formats return a dictionary, but Nested-Simple
## returns a list, so we can't mutate the payload... overwrite.
dataToDelete = request.context['payload']
customHeaders = {}
utils.getCustomHeaders(request.headers, customHeaders)
if customHeaders['resultsFormat'].lower() == 'flat':
content = {'objects': dataToDelete.get('objects')}
deleteTaskContentHelper(request, response, content)
## Payload was borrowed to store the results; clear before exiting
request.context['payload'].clear()
elif customHeaders['resultsFormat'].lower() == 'nested':
content = dataToDelete
deleteTaskContentHelper(request, response, content)
request.context['payload'].clear()
else:
request.context['payload'].clear()
request.context['payload']['errors'].append('Unsupported resultsFormat: {}'.format(customHeaders['resultsFormat']))
except:
errorMessage(request, response)
## end deleteExecutingGivenQueries
return cleanPayload(request)
def deleteStoredQueryDataSet(name:text, request, response):
"""Delete all objects returned from running the named query."""
try:
customHeaders = {}
utils.getCustomHeaders(request.headers, customHeaders)
executeSimpleQueryHelper(request, response, name)
dataToDelete = request.context['payload']
if customHeaders['resultsFormat'].lower() == 'flat':
content ={'objects': dataToDelete.get('objects')}
elif customHeaders['resultsFormat'].lower() == 'nested':
content = dataToDelete
deleteTaskContentHelper(request, response, content)
## Payload was borrowed to store the results; clear before exiting
request.context['payload'].clear()
except:
errorMessage(request, response)
## end deleteStoredQueryDataSet
return cleanPayload(request)
def searchThisContentHelper(request, response, filterContent, dbTable, countResult, deleteResult):
## Not using the queryProcessing module to do this; just borrowing
## the processFilter inside, to share the same filter syntax/parsing
customHeaders = {}
utils.getCustomHeaders(request.headers, customHeaders)
qpInstance = QueryProcessing(request.context['logger'], request.context['dbSession'], {'objects': [], 'links': []}, customHeaders['removePrivateAttributes'], customHeaders['removeEmptyAttributes'], customHeaders['resultsFormat'])
createdFilterList = qpInstance.processFilter(filterContent, dbTable)
filterList = []
## Message any query parsing errors back to the client so they
## can see what to fix (e.g. 'betweendate' time formats)
if len(qpInstance.errorList) > 0:
for msg in qpInstance.errorList:
request.context['payload']['errors'].append(msg)
response.status = HTTP_400
## Only do the following if we didn't hit filter errors
else:
if type(createdFilterList) == type(list()):
filterList.extend(createdFilterList)
else:
filterList.append(createdFilterList)
dataHandle = request.context['dbSession'].query(dbTable).filter(*filterList)
## Number the results; makes it easier for users to visually parse
tmpJobResultNumber = 0
objectList = []
for item in dataHandle:
tmpJobResultNumber += 1
if deleteResult:
request.context['dbSession'].delete(item)
request.context['dbSession'].commit()
elif not countResult:
request.context['payload'][tmpJobResultNumber] = {col:getattr(item, col) for col in inspect(item).mapper.c.keys()}
if deleteResult:
request.context['payload']['Results Removed'] = tmpJobResultNumber
request.context['logger'].debug('searchThisContentHelper: removed total objects: {}'.format(tmpJobResultNumber))
elif countResult:
request.context['payload']['Result Count'] = tmpJobResultNumber
## end searchThisContentHelper
return
def getLinkedObjects(object_id: text, request, response):
"""Return all objects linked to the object specified by its id."""
try:
if utils.uuidCheck(object_id):
dataHandle = request.context['dbSession'].query(
tableMapping.BaseObject).filter(
tableMapping.BaseObject.object_id == object_id).first()
if dataHandle:
customHeaders = {}
utils.getCustomHeaders(request.headers, customHeaders)
resultJson = Results(
'tool/link/{object_id}'.format(object_id=object_id), True)
weakLinkObjects = set()
StrongLinkObjects = set()
onceFlag = False
for relation in ['weak_first_objects', 'weak_second_objects']:
entry = set(getattr(dataHandle, relation))
weakLinkObjects = weakLinkObjects.union(entry)
for relation in [
'strong_first_objects', 'strong_second_objects'
]:
if getattr(
inspect(dataHandle).mapper.relationships,
relation).uselist:
entry = set(getattr(dataHandle, relation))
StrongLinkObjects = StrongLinkObjects.union(entry)
else:
if getattr(dataHandle, relation) is not None:
entry = getattr(dataHandle, relation)
StrongLinkObjects.add(entry)
objects = set()
objectList = []
for item in StrongLinkObjects:
objects.add(item.strong_second_object)
objects.add(item.strong_first_object)
for item in weakLinkObjects:
objects.add(item.weak_second_object)
objects.add(item.weak_first_object)
if dataHandle not in objects:
objects.add(dataHandle)
for obj in objects:
objectId = getattr(obj, 'object_id')
data = None
if customHeaders['removePrivateAttributes']:
data = {
col: getattr(obj, col)
for col in inspect(obj).mapper.c.keys()
if col not in
['container', 'object_type', 'reference_id']
}
else:
data = {
col: getattr(obj, col)
for col in inspect(obj).mapper.c.keys()
}
identifier, exists = resultJson.addObject(
inspect(obj).mapper.class_.__name__,
uniqueId=objectId,
**data)
objectList.append(
((obj, inspect(obj).mapper.class_.__name__),
identifier))
objectsDict = dict(objectList)
for item in StrongLinkObjects:
first_id = objectsDict[(
item.strong_first_object,
inspect(
item.strong_first_object).mapper.class_.__name__)]
second_id = objectsDict[(
item.strong_second_object,
inspect(
item.strong_second_object).mapper.class_.__name__)]
resultJson.addLink(inspect(item).mapper.class_.__name__,
firstId=first_id,
secondId=second_id)
## Recreating weak link connection
for item in weakLinkObjects:
if item.weak_first_object is dataHandle:
first_id = objectsDict[(item.weak_first_object,
inspect(
item.weak_first_object).
mapper.class_.__name__)]
resultJson.addLink(
inspect(item).mapper.class_.__name__,
firstId=first_id,
secondId=item.weak_second_object.object_id)
if item.weak_second_object is dataHandle:
second_id = objectsDict[(
item.weak_second_object,
item.weak_second_object.__class__.__name__)]
resultJson.addLink(
inspect(item).mapper.class_.__name__,
firstId=item.weak_first_object.object_id,
secondId=second_id)
tempJson = resultJson.getJson()
## pop off temp_weak_links
tempJson.pop('temp_weak_links', None)
request.context['payload'] = tempJson
resultJson.clearJson()
del resultJson
else:
request.context['payload']['errors'].append(
'The provided object_id {} does not exist in the database.'
.format(object_id))
response.status = HTTP_404
else:
request.context['payload']['errors'].append(
'Expected 32-bit hex for id')
response.status = HTTP_400
except:
errorMessage(request, response)
## end getLinkedObjects
return cleanPayload(request)
