def _hit_endpoint(self, endpoint, data, expected_codes, method=None): if method: kwargs = {'path': endpoint} if method == 'POST': kwargs.update({'data': marshal.dumps(data), 'buffered': True}) response = getattr(self.http_client, method.lower())(**kwargs) elif '/index' in endpoint: response = self.http_client.get(path=endpoint) else: response = self.http_client.post(path=endpoint, data=marshal.dumps(data), buffered=True) self.assertTrue(response.status_code in expected_codes)
def get_events(environment, start_response, headers): """ Retrieve events POST body should contain a JSON encoded version of: { namespace: namespace_name (optional), stream : stream_name, start_time : starting_time_as_kronos_time, end_time : ending_time_as_kronos_time, start_id : only_return_events_with_id_greater_than_me, limit: optional_maximum_number_of_events, order: ResultOrder.ASCENDING or ResultOrder.DESCENDING (default ResultOrder.ASCENDING) } Either start_time or start_id should be specified. If a retrieval breaks while returning results, you can send another retrieval request and specify start_id as the last id that you saw. Kronos will only return events that occurred after the event with that id. """ request_json = environment['json'] try: stream = request_json['stream'] validate_stream(stream) except Exception, e: log.exception('get_events: stream validation failed for `%s`', request_json.get('stream')) start_response('400 Bad Request', headers) yield marshal.dumps({ERRORS_FIELD: [repr(e)], SUCCESS_FIELD: False}) return
def _retrieve(self, namespace, stream, start_id, end_time, order, limit, configuration): """ Yield events from stream starting after the event with id `start_id` until and including events with timestamp `end_time`. """ start_id_event = Event(start_id) end_id_event = Event( uuid_from_kronos_time(end_time, _type=UUIDType.HIGHEST)) stream_events = self.db[namespace][stream] # Find the interval our events belong to. lo = bisect.bisect_left(stream_events, start_id_event) if lo + 1 > len(stream_events): return if stream_events[lo] == start_id_event: lo += 1 hi = bisect.bisect_right(stream_events, end_id_event) if order == ResultOrder.DESCENDING: index_it = xrange(hi - 1, lo - 1, -1) else: index_it = xrange(lo, hi) for i in index_it: if limit <= 0: break limit -= 1 yield marshal.dumps(stream_events[i])
def insert(self, events): if not events: return batch_stmt = BatchStatement(batch_type=BatchType.UNLOGGED, consistency_level=ConsistencyLevel.QUORUM) shard_idx = {} for _id, event in events: shard_time = round_down(event[TIMESTAMP_FIELD], self.width) shard = shard_idx.get(shard_time, random.randint(0, self.shards - 1)) # Insert to index. try: self.index_cache.get((shard_time, shard)) except KeyError: batch_stmt.add(BoundStatement(self.namespace.INDEX_INSERT_STMT, routing_key=self.stream, consistency_level=ConsistencyLevel.QUORUM) .bind((self.stream, shard_time, self.width, shard))) self.index_cache.set((shard_time, shard), None) # Insert to stream. shard_key = StreamShard.get_key(self.stream, shard_time, shard) batch_stmt.add(BoundStatement(self.namespace.INSERT_STMT, routing_key=shard_key, consistency_level=ConsistencyLevel.QUORUM) .bind((shard_key, _id, marshal.dumps(event)))) shard_idx[shard_time] = (shard + 1) % self.shards # Round robin. self.session.execute(batch_stmt)
def _retrieve(self, namespace, stream, start_id, end_time, order, limit, configuration): """ Yield events from stream starting after the event with id `start_id` until and including events with timestamp `end_time`. """ start_id_event = Event(start_id) end_id_event = Event(uuid_from_kronos_time(end_time, _type=UUIDType.HIGHEST)) stream_events = self.db[namespace][stream] # Find the interval our events belong to. lo = bisect.bisect_left(stream_events, start_id_event) if lo + 1 > len(stream_events): return if stream_events[lo] == start_id_event: lo += 1 hi = bisect.bisect_right(stream_events, end_id_event) if order == ResultOrder.DESCENDING: index_it = xrange(hi-1, lo-1, -1) else: index_it = xrange(lo, hi) for i in index_it: if limit <= 0: break limit -= 1 yield marshal.dumps(stream_events[i])
def infer_schema(self, stream, namespace=None): response = self.http_client.post( self.infer_schema_path, data=marshal.dumps({'stream': stream, 'namespace': namespace}), buffered=True) self.assertEqual(response.status_code, 200) return marshal.loads(response.data)
def get_streams(self, namespace=None): data = {} if namespace is not None: data['namespace'] = namespace response = self.http_client.post(self.streams_path, data=marshal.dumps(data), buffered=True) self.assertEqual(response.status_code, 200) return response.data.splitlines()
def get_streams(self, namespace=None): data = {} if namespace is not None: data['namespace'] = namespace response = self.http_client.post(self.streams_path, data=marshal.dumps(data), buffered=True) self.assertEqual(response.status_code, 200) return map(marshal.loads, response.data.splitlines())
def delete(self, stream, start_time, end_time, start_id=None, namespace=None): data = {'stream': stream, 'end_time': end_time} if start_id: data['start_id'] = start_id else: data['start_time'] = start_time if namespace is not None: data['namespace'] = namespace response = self.http_client.post(path=self.delete_path, data=marshal.dumps(data), buffered=True) self.assertEqual(response.status_code, 200) response = marshal.loads(response.data) self.assertTrue(response[SUCCESS_FIELD]) return response
def put(self, stream_or_mapping, events=None, namespace=None): data = {} if isinstance(stream_or_mapping, dict): data['events'] = stream_or_mapping else: self.assertTrue(events is not None) data['events'] = {stream_or_mapping: events} if namespace is not None: data['namespace'] = namespace response = self.http_client.post(path=self.put_path, data=marshal.dumps(data), buffered=True) self.assertEqual(response.status_code, 200) response = marshal.loads(response.data) self.assertTrue(response[SUCCESS_FIELD]) return response
def get(self, stream, start_time, end_time, start_id=None, limit=None, order=None, namespace=None): data = {'stream': stream, 'end_time': end_time} if start_id: data['start_id'] = start_id else: data['start_time'] = start_time if limit is not None: data['limit'] = limit if order is not None: data['order'] = order if namespace is not None: data['namespace'] = namespace response = self.http_client.post(path=self.get_path, data=marshal.dumps(data), buffered=True) self.assertEqual(response.status_code, 200) return map(marshal.loads, response.data.splitlines())
def wrapper(environment, start_response): try: start_time = time.time() if function.func_name not in ( _serving_mode_endpoints[settings.serving_mode]): start_response('403 Forbidden', [('Content-Type', 'application/json')]) return marshal.dumps({ ERRORS_FIELD: [ 'kronosd is configured to block access to this ' 'endpoint.' ], SUCCESS_FIELD: False, TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time)) }) req_method = environment['REQUEST_METHOD'] # If the request method is not allowed, return 405. if req_method not in methods: start_response('405 Method Not Allowed', [('Allow', ', '.join(methods)), ('Content-Type', 'application/json')]) return marshal.dumps({ ERRORS_FIELD: ['%s method not allowed' % req_method], SUCCESS_FIELD: False, TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time)) }) headers = [] remote_origin = environment.get('HTTP_ORIGIN') if req_method == 'OPTIONS': # This is a CORS preflight request so check that the remote domain is # allowed and respond with appropriate CORS headers. # http://www.html5rocks.com/static/images/cors_server_flowchart.png if is_remote_allowed(remote_origin): headers.extend([ ('Access-Control-Allow-Origin', remote_origin), ('Access-Control-Allow-Credentials', 'true'), ('Access-Control-Allow-Headers', ', '.join( ('Accept', 'Content-Type', 'Origin', 'X-Requested-With'))), ('Access-Control-Allow-Methods', ', '.join(methods)) ]) # We just tell the client that CORS is ok. Client will follow up # with another request to get the answer. start_response('200 OK', headers) return '' # All POST bodies must be json, so decode it here. if req_method == 'POST': try: environment['json'] = marshal.loads( environment['wsgi.input'].read()) except ValueError: start_response('400 Bad Request', [('Content-Type', 'application/json')]) return marshal.dumps({ ERRORS_FIELD: ['Request body must be valid JSON.'], SUCCESS_FIELD: False, TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time)) }) # All responses are JSON. headers.append(('Content-Type', 'application/json')) if remote_origin: headers.append( ('Access-Control-Allow-Origin', remote_origin)) response = function(environment, start_response, headers) if not isinstance(response, types.GeneratorType): response[TOOK_FIELD] = '%fms' % ( 1000 * (time.time() - start_time)) response = marshal.dumps(response) return response except Exception, e: log.exception('endpoint: uncaught exception!') start_response('400 Bad Request', [('Content-Type', 'application/json')]) return marshal.dumps({ ERRORS_FIELD: [repr(e)], SUCCESS_FIELD: False, TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time)) })
def wrapper(environment, start_response): try: start_time = time.time() if function.func_name not in (_serving_mode_endpoints [settings.serving_mode]): start_response('403 Forbidden', [('Content-Type', 'application/json')]) return marshal.dumps({ ERRORS_FIELD: ['kronosd is configured to block access to this ' 'endpoint.'], SUCCESS_FIELD: False, TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time)) }) req_method = environment['REQUEST_METHOD'] # If the request method is not allowed, return 405. if req_method not in methods: start_response('405 Method Not Allowed', [('Allow', ', '.join(methods)), ('Content-Type', 'application/json')]) return marshal.dumps({ ERRORS_FIELD: ['%s method not allowed' % req_method], SUCCESS_FIELD: False, TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time)) }) headers = [] remote_origin = environment.get('HTTP_ORIGIN') if req_method == 'OPTIONS': # This is a CORS preflight request so check that the remote domain is # allowed and respond with appropriate CORS headers. # http://www.html5rocks.com/static/images/cors_server_flowchart.png if is_remote_allowed(remote_origin): headers.extend([ ('Access-Control-Allow-Origin', remote_origin), ('Access-Control-Allow-Credentials', 'true'), ('Access-Control-Allow-Headers', ', '.join( ('Accept', 'Content-Type', 'Origin', 'X-Requested-With'))), ('Access-Control-Allow-Methods', ', '.join(methods)) ]) # We just tell the client that CORS is ok. Client will follow up # with another request to get the answer. start_response('200 OK', headers) return '' # All POST bodies must be json, so decode it here. if req_method == 'POST': try: environment['json'] = marshal.loads(environment['wsgi.input'] .read()) except ValueError: start_response('400 Bad Request', [('Content-Type', 'application/json')]) return marshal.dumps({ ERRORS_FIELD: ['Request body must be valid JSON.'], SUCCESS_FIELD: False, TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time)) }) # All responses are JSON. headers.append(('Content-Type', 'application/json')) if remote_origin: headers.append(('Access-Control-Allow-Origin', remote_origin)) response = function(environment, start_response, headers) if not isinstance(response, types.GeneratorType): response[TOOK_FIELD] = '%fms' % (1000 * (time.time() - start_time)) response = marshal.dumps(response) return response except Exception, e: log.exception('endpoint: uncaught exception!') start_response('400 Bad Request', [('Content-Type', 'application/json')]) return marshal.dumps({ ERRORS_FIELD: [repr(e)], SUCCESS_FIELD: False, TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time)) })