def fromJson(json, protoClass):
"""
Deserialise json into an instance of protobuf class
"""
try:
return json_format.Parse(json, protoClass())
except Exception as e:
raise BadRequestException(str(e))
def _validate_params(subreddit, start, end):
"""
Checks that all parameters exist and
that the time interval is correct
"""
exc = None
if subreddit is None:
exc = BadRequestException(message="Missing 'subreddit' parameter")
elif start is None:
exc = BadRequestException(message="Missing 'from' parameter")
elif end is None:
exc = BadRequestException(message="Missing 'to' parameter")
elif start > end:
exc = BadRequestException(message='Time interval is invalid')
if exc:
_log.error(str(exc), exc_info=True)
raise exc
def _normalize_params(subreddit, start, end):
"""
Makes sure that 'subreddit' is low case
and 'start' and 'end' are floats
"""
try:
subreddit = subreddit.lower()
start = float(start)
end = float(end)
except:
msg = 'Invalid parameters'
_log.error(msg, exc_info=True)
raise BadRequestException(msg)
return subreddit, start, end
def filter():
subreddit = request.args.get("subreddit")
start = request.args.get("from")
end = request.args.get("to")
keyword = request.args.get("keyword")
subreddit, start, end = _normalize_params(subreddit, start, end)
_validate_params(subreddit, start, end)
try:
items = _db.filter(subreddit, start, end, keyword)
except:
_log.error('Unable to query mongo', exc_info=True)
raise BadRequestException(
message='Internal server error',
status_code=500,
)
return jsonify(items)
def create(self, request, *args, **kwargs):
serializer = self.serializer_class(data=request.data)
serializer.is_valid(raise_exception=True)
try:
from_id = request.data[self.from_field]["pk"]
to_id = request.data[self.to_field]["pk"]
except:
raise BadRequestException()
try:
from_qs = self.from_model.objects.get(
pk=from_id, user=request.user.pk
)
to_qs = self.to_model.objects.get(pk=to_id, user=request.user.pk)
except ObjectDoesNotExist:
raise NotFoundException()
transaction = self.model_class(**serializer.data)
setattr(transaction, self.from_field, from_qs)
setattr(transaction, self.to_field, to_qs)
transaction.save()
return Response({"pk": transaction.pk}, status=status.HTTP_201_CREATED)
def searchDataFrames(self, request, from_get=False):
"""
DataFrame Search Function
"""
# enforce project level access control by enforcing restrictions on keyspaces
check_ks = request.get('keyspaceIds', None)
if check_ks is None or (len(check_ks) == 1
and check_ks[0] == unicode('mask-keys')):
raise BadRequestException(
"keyspaceIds required when searching dataframes")
# get proto, validates
jreq = util.fromJson(json.dumps(request), fs.SearchDataFramesRequest)
# handle masks, ommitting contents from endpoint
mask_contents = {"contents": 0}
mask_keys = None
# handle filters
filters = {}
if len(jreq.dataframe_ids) > 0:
filters['_id'] = util.getMongoFieldFilter(jreq.dataframe_ids,
ObjectId,
from_get=from_get)
if len(jreq.keyspace_ids) > 0:
# grab keys mask
keyspace_ids = jreq.keyspace_ids
if from_get:
keyspace_ids = jreq.keyspace_ids[0].split(',')
mask_keys = util.setMask(keyspace_ids, unicode('mask-keys'),
'keys')
# process keyspace ids
if len(keyspace_ids) > 0:
filters.setdefault('$or', []).append({
'major':
util.getMongoFieldFilter(keyspace_ids,
ObjectId,
from_get=from_get)
})
filters['$or'].append({
'minor':
util.getMongoFieldFilter(keyspace_ids,
ObjectId,
from_get=from_get)
})
if len(jreq.unit_ids) > 0:
filters.setdefault('$and', []).append({
'units':
util.getMongoFieldFilter(jreq.unit_ids,
ObjectId,
from_get=from_get)
})
result = self.db.dataframe.find(filters, mask_contents)
# make proto
_protoresp = fs.SearchDataFramesResponse(dataframes=[])
for r in result:
kmaj_name, kmaj_keys = util.getKeySpaceInfo(
self.db, r['major'], mask_keys)
kmin_name, kmin_keys = util.getKeySpaceInfo(
self.db, r['minor'], mask_keys)
dataframe = fs.DataFrame(id=str(r['_id']), \
major=fs.Dimension(keyspace_id=str(r['major']), keys=kmaj_keys), \
minor=fs.Dimension(keyspace_id=str(r['minor']), keys=kmin_keys), \
units=[], contents=[])
for unit in r['units']:
_unit = self.db.units.find_one({'_id': unit})
dataframe.units.extend([fs.Unit(name=_unit['name'], \
description=_unit['description'], \
id=str(_unit['_id']))])
_protoresp.dataframes.extend([dataframe])
return util.toFlaskJson(_protoresp)
def sliceDataFrame(self, request, transpose=False):
# inits
vec_filters = {}
# validate request
jreq = util.fromJson(request, fs.SliceDataFrameRequest)
if not jreq.dataframe_id:
raise BadRequestException(
"dataframeId is required for sliceDataFrame")
# first request to get dataframe
result = self.db.dataframe.find_one(
{"_id": ObjectId(str(jreq.dataframe_id))})
if result is None:
raise DataFrameNotFoundException(jreq.dataframe_id)
# prep vector query
vc = result.get('contents', [])
# save page end for later check
page_end = int(jreq.page_end)
# if page start is outside of dataframe length, return empty
if jreq.page_start > len(vc):
dataframe = {"id": str(result["_id"]), \
"major": {"keyspaceId": str(result['major']), "keys": []}, \
"minor": {"keyspaceId": str(result['minor']), "keys": []}, \
"contents": []}
return util.buildResponse(dataframe)
elif jreq.page_end > len(vc) or len(
jreq.new_minor.keys) > 0 or jreq.page_end == 0:
jreq.page_end = len(vc)
# construct vector filters
vec_filters["_id"] = {"$in": vc[jreq.page_start:jreq.page_end]}
if not transpose:
kmaj_keys = self.setDimensionFilters(jreq.new_major.keys,
jreq.new_minor.keys,
vec_filters)
else:
kmaj_keys = self.setDimensionFilters(jreq.new_minor.keys,
jreq.new_major.keys,
vec_filters)
# seconrd query to backend to get contents
vectors = self.db.vector.find(vec_filters, kmaj_keys)
vectors.batch_size(1000000)
# construct response
contents = {vector["key"]: vector["contents"] for vector in vectors}
if transpose:
# this is the overhead
d = pd.DataFrame.from_dict(contents, orient="index")
contents = d.to_dict()
# avoid invalid keys passing through to keys
# explore impacts on response time
kmaj_keys = []
if len(jreq.new_major.keys) > 0:
kmaj_keys = contents[contents.keys()[0]].keys()
# return keys in dimension,
# if the whole dimension is not returned
kmin_keys = []
# if len(jreq.new_minor.keys) > 0 or page_end < len(vc):
if len(jreq.new_minor.keys) > 0 or page_end == 0:
kmin_keys = contents.keys()
dataframe = {"id": str(result["_id"]), \
"major": {"keyspaceId": str(result['major']), "keys": kmaj_keys}, \
"minor": {"keyspaceId": str(result['minor']), "keys": kmin_keys}, \
"contents": contents}
return util.buildResponse(dataframe)