def sink(bucket):
"""
Allows clients to submit records via HTTP
"""
start_time = unixtime()
request = bottle.request
data = request.json
try:
record = make_record(data)
except ValidationError, oops:
LOG.info('Input data failed sanitization checks', exc_info=True)
bottle.abort(400, oops.message)
def get_values(bucket):
"""
Retrieves the values stored in a single facet.
It accepts a JSON dictionary of query names to facets, e.g.:
{
'tablet-dayone': {'device': ['tablet', 'apple'],
'time': ['2013', '1', '15']}
'tablet-daytwo': {'device': ['tablet', 'apple'],
'time': ['2013', '1', '16']}
}
On success it will respond with the an API standard response including the
'results' key, a dictionary of the values queried for, e.g.:
{
'ok': true,
'status': 200,
'results': {
'tablet-dayone': {
'duration': 1212421,
'datapoints': 238282
},
'tablet-daytwo': {
'duration': 3292382,
'datapoints': 38283
}
}
}
This allows effecient retrieve of any number of facet values from the DB.
"""
start_time = unixtime()
request = bottle.request
query = request.json
if query is None or type(query) != dict:
bottle.abort(400, 'Must POST application/json dictionary')
facet_keys = {}
results = {}
# Prepare facets for query
for key, facet in query.items():
try:
facet = sanitized_facets(query['facets'])
except ValidationError, oops:
LOG.info("Query for '%s' contained invalid facet", key, exc_info=True)
bottle.abort(400, "%s: %s" % (key, oops.message))
facet_keys[key] = split_facet(facet)
def sync_redis(self):
# Sync every 5 minutes, or when 5k entries exist
need_to_sync = self.redis.scard('keys') > 5000
if not need_to_sync:
if self._last_sync is None:
need_to_sync = True
else:
need_to_sync = self._last_sync < (unixtime() - (60))
if need_to_sync:
self.incr_stats('redis.ops')
self.incr_stats('redis.ops.smembers')
for member in self.redis.smembers('keys'):
values = self.redis.hgetall(member)
self.incr_stats('redis.ops')
self.incr_stats('redis.ops.hgetall')
facet = marshal.loads(values['$hs.facet'])
del values['$hs.facet']
self.insert_to_hyperdex(facet, values)
self.show_status()
if self.is_stopping():
break
def find_values(bucket):
"""
Retrieves a list of the facet values underneath a parent facet.
It accepts a JSON dictionry of the parent facet names and a limit on how
many sub-facets to return.
{
'tablet-days': {'facet': {'device': ['tablet', 'apple'],
'time': ['2013', '1', '15']},
'limit': 50}
}
On success it will return the a standard API response including the
'results' key which contains a
{
'ok': true,
'status': 200,
'results': {
'tablet-days': {'1': {'duration': 38289323,
'datapoints': 4919}},
{'2': {'duration': 382829,
'datapoints': 1234}}
}
}
"""
start_time = unixtime()
request = bottle.request
query = request.json
if query is None or type(query) != dict:
bottle.abort(400, 'Must POST application/json dictionary')
# Validate the searches
searches = {}
for name, search in query.items():
if 'facet' not in search:
bottle.abort(400, '"facet" key required for "%s"' % (name,))
if 'limit' not in search:
bottle.abort(400, '"limit" key required for "%s"' % (name,))
try:
limit = int(search.get('limit'))
except ValueError:
bottle.abort(400, 'Invalid "limit" for "%s"' % (name,))
withvalues = bool(search.get('withvalues'))
startkey = None
if 'startkey' in search:
try:
startkey = to_utf8_str(search['startkey'])
except TypeError:
bottle.abort(400, 'Start key for "%s" is invalid type' % (name,))
try:
facet = sanitized_facets(search['facet'])
except ValidationError, oops:
LOG.info("'%s' contained invalid facet", name, exc_info=True)
bottle.abort(400, "%s: %s" % (name, oops.message))
searches[name] = {
'facet': split_facet(facet),
'limit': limit,
'startkey': start,
'withvalues': withvalues
}
# TODO: add NotEqual for the 'start' value too
limit += 1
withvalues = search['withvalues']
results = {} if withvalues else []
searchiter = HDEX.sorted_search(bucket, predicate, 'facet', limit, 'min')
for result in searchiter:
facet = result['facet']
if facet == startkey:
continue
if withvalues:
results[facet] = result['values']
else:
results.append(facet)
all_results[name] = results
end_time = unixtime()
return {
'ok': True,
'status': 200,
'results': results,
'time': end_time - start_time
}
@bottle.route('/<bucket:re:[a-z]+>/get-values', method=['POST'], name='get_values')
def get_values(bucket):
"""
Retrieves the values stored in a single facet.
It accepts a JSON dictionary of query names to facets, e.g.:
def __init__(self, rdb, hdex):
super(AggregatorDaemon, self).__init__(rdb)
assert hdex is not None
self._hdex = hdex
self._last_sync = unixtime()
