def sink(bucket): """ Allows clients to submit records via HTTP """ start_time = unixtime() request = bottle.request data = request.json try: record = make_record(data) except ValidationError, oops: LOG.info('Input data failed sanitization checks', exc_info=True) bottle.abort(400, oops.message)
def get_values(bucket): """ Retrieves the values stored in a single facet. It accepts a JSON dictionary of query names to facets, e.g.: { 'tablet-dayone': {'device': ['tablet', 'apple'], 'time': ['2013', '1', '15']} 'tablet-daytwo': {'device': ['tablet', 'apple'], 'time': ['2013', '1', '16']} } On success it will respond with the an API standard response including the 'results' key, a dictionary of the values queried for, e.g.: { 'ok': true, 'status': 200, 'results': { 'tablet-dayone': { 'duration': 1212421, 'datapoints': 238282 }, 'tablet-daytwo': { 'duration': 3292382, 'datapoints': 38283 } } } This allows effecient retrieve of any number of facet values from the DB. """ start_time = unixtime() request = bottle.request query = request.json if query is None or type(query) != dict: bottle.abort(400, 'Must POST application/json dictionary') facet_keys = {} results = {} # Prepare facets for query for key, facet in query.items(): try: facet = sanitized_facets(query['facets']) except ValidationError, oops: LOG.info("Query for '%s' contained invalid facet", key, exc_info=True) bottle.abort(400, "%s: %s" % (key, oops.message)) facet_keys[key] = split_facet(facet)
def sync_redis(self): # Sync every 5 minutes, or when 5k entries exist need_to_sync = self.redis.scard('keys') > 5000 if not need_to_sync: if self._last_sync is None: need_to_sync = True else: need_to_sync = self._last_sync < (unixtime() - (60)) if need_to_sync: self.incr_stats('redis.ops') self.incr_stats('redis.ops.smembers') for member in self.redis.smembers('keys'): values = self.redis.hgetall(member) self.incr_stats('redis.ops') self.incr_stats('redis.ops.hgetall') facet = marshal.loads(values['$hs.facet']) del values['$hs.facet'] self.insert_to_hyperdex(facet, values) self.show_status() if self.is_stopping(): break
def find_values(bucket): """ Retrieves a list of the facet values underneath a parent facet. It accepts a JSON dictionry of the parent facet names and a limit on how many sub-facets to return. { 'tablet-days': {'facet': {'device': ['tablet', 'apple'], 'time': ['2013', '1', '15']}, 'limit': 50} } On success it will return the a standard API response including the 'results' key which contains a { 'ok': true, 'status': 200, 'results': { 'tablet-days': {'1': {'duration': 38289323, 'datapoints': 4919}}, {'2': {'duration': 382829, 'datapoints': 1234}} } } """ start_time = unixtime() request = bottle.request query = request.json if query is None or type(query) != dict: bottle.abort(400, 'Must POST application/json dictionary') # Validate the searches searches = {} for name, search in query.items(): if 'facet' not in search: bottle.abort(400, '"facet" key required for "%s"' % (name,)) if 'limit' not in search: bottle.abort(400, '"limit" key required for "%s"' % (name,)) try: limit = int(search.get('limit')) except ValueError: bottle.abort(400, 'Invalid "limit" for "%s"' % (name,)) withvalues = bool(search.get('withvalues')) startkey = None if 'startkey' in search: try: startkey = to_utf8_str(search['startkey']) except TypeError: bottle.abort(400, 'Start key for "%s" is invalid type' % (name,)) try: facet = sanitized_facets(search['facet']) except ValidationError, oops: LOG.info("'%s' contained invalid facet", name, exc_info=True) bottle.abort(400, "%s: %s" % (name, oops.message)) searches[name] = { 'facet': split_facet(facet), 'limit': limit, 'startkey': start, 'withvalues': withvalues }
# TODO: add NotEqual for the 'start' value too limit += 1 withvalues = search['withvalues'] results = {} if withvalues else [] searchiter = HDEX.sorted_search(bucket, predicate, 'facet', limit, 'min') for result in searchiter: facet = result['facet'] if facet == startkey: continue if withvalues: results[facet] = result['values'] else: results.append(facet) all_results[name] = results end_time = unixtime() return { 'ok': True, 'status': 200, 'results': results, 'time': end_time - start_time } @bottle.route('/<bucket:re:[a-z]+>/get-values', method=['POST'], name='get_values') def get_values(bucket): """ Retrieves the values stored in a single facet. It accepts a JSON dictionary of query names to facets, e.g.:
def __init__(self, rdb, hdex): super(AggregatorDaemon, self).__init__(rdb) assert hdex is not None self._hdex = hdex self._last_sync = unixtime()