def _hit_endpoint(self, endpoint, data, expected_codes, method=None):
if method:
kwargs = {'path': endpoint}
if method == 'POST':
kwargs.update({'data': marshal.dumps(data), 'buffered': True})
response = getattr(self.http_client, method.lower())(**kwargs)
elif '/index' in endpoint:
response = self.http_client.get(path=endpoint)
else:
response = self.http_client.post(path=endpoint,
data=marshal.dumps(data),
buffered=True)
self.assertTrue(response.status_code in expected_codes)
def _hit_endpoint(self, endpoint, data, expected_codes, method=None):
if method:
kwargs = {'path': endpoint}
if method == 'POST':
kwargs.update({'data': marshal.dumps(data),
'buffered': True})
response = getattr(self.http_client, method.lower())(**kwargs)
elif '/index' in endpoint:
response = self.http_client.get(path=endpoint)
else:
response = self.http_client.post(path=endpoint,
data=marshal.dumps(data),
buffered=True)
self.assertTrue(response.status_code in expected_codes)
def get_events(environment, start_response, headers):
"""
Retrieve events
POST body should contain a JSON encoded version of:
{ namespace: namespace_name (optional),
stream : stream_name,
start_time : starting_time_as_kronos_time,
end_time : ending_time_as_kronos_time,
start_id : only_return_events_with_id_greater_than_me,
limit: optional_maximum_number_of_events,
order: ResultOrder.ASCENDING or ResultOrder.DESCENDING (default
ResultOrder.ASCENDING)
}
Either start_time or start_id should be specified. If a retrieval breaks
while returning results, you can send another retrieval request and specify
start_id as the last id that you saw. Kronos will only return events that
occurred after the event with that id.
"""
request_json = environment['json']
try:
stream = request_json['stream']
validate_stream(stream)
except Exception, e:
log.exception('get_events: stream validation failed for `%s`',
request_json.get('stream'))
start_response('400 Bad Request', headers)
yield marshal.dumps({ERRORS_FIELD: [repr(e)], SUCCESS_FIELD: False})
return
def _retrieve(self, namespace, stream, start_id, end_time, order, limit,
configuration):
"""
Yield events from stream starting after the event with id `start_id` until
and including events with timestamp `end_time`.
"""
start_id_event = Event(start_id)
end_id_event = Event(
uuid_from_kronos_time(end_time, _type=UUIDType.HIGHEST))
stream_events = self.db[namespace][stream]
# Find the interval our events belong to.
lo = bisect.bisect_left(stream_events, start_id_event)
if lo + 1 > len(stream_events):
return
if stream_events[lo] == start_id_event:
lo += 1
hi = bisect.bisect_right(stream_events, end_id_event)
if order == ResultOrder.DESCENDING:
index_it = xrange(hi - 1, lo - 1, -1)
else:
index_it = xrange(lo, hi)
for i in index_it:
if limit <= 0:
break
limit -= 1
yield marshal.dumps(stream_events[i])
def insert(self, events):
if not events:
return
batch_stmt = BatchStatement(batch_type=BatchType.UNLOGGED,
consistency_level=ConsistencyLevel.QUORUM)
shard_idx = {}
for _id, event in events:
shard_time = round_down(event[TIMESTAMP_FIELD], self.width)
shard = shard_idx.get(shard_time,
random.randint(0, self.shards - 1))
# Insert to index.
try:
self.index_cache.get((shard_time, shard))
except KeyError:
batch_stmt.add(BoundStatement(self.namespace.INDEX_INSERT_STMT,
routing_key=self.stream,
consistency_level=ConsistencyLevel.QUORUM)
.bind((self.stream, shard_time, self.width, shard)))
self.index_cache.set((shard_time, shard), None)
# Insert to stream.
shard_key = StreamShard.get_key(self.stream, shard_time, shard)
batch_stmt.add(BoundStatement(self.namespace.INSERT_STMT,
routing_key=shard_key,
consistency_level=ConsistencyLevel.QUORUM)
.bind((shard_key,
_id,
marshal.dumps(event))))
shard_idx[shard_time] = (shard + 1) % self.shards # Round robin.
self.session.execute(batch_stmt)
def _retrieve(self, namespace, stream, start_id, end_time, order, limit,
configuration):
"""
Yield events from stream starting after the event with id `start_id` until
and including events with timestamp `end_time`.
"""
start_id_event = Event(start_id)
end_id_event = Event(uuid_from_kronos_time(end_time,
_type=UUIDType.HIGHEST))
stream_events = self.db[namespace][stream]
# Find the interval our events belong to.
lo = bisect.bisect_left(stream_events, start_id_event)
if lo + 1 > len(stream_events):
return
if stream_events[lo] == start_id_event:
lo += 1
hi = bisect.bisect_right(stream_events, end_id_event)
if order == ResultOrder.DESCENDING:
index_it = xrange(hi-1, lo-1, -1)
else:
index_it = xrange(lo, hi)
for i in index_it:
if limit <= 0:
break
limit -= 1
yield marshal.dumps(stream_events[i])
def infer_schema(self, stream, namespace=None):
response = self.http_client.post(
self.infer_schema_path,
data=marshal.dumps({'stream': stream, 'namespace': namespace}),
buffered=True)
self.assertEqual(response.status_code, 200)
return marshal.loads(response.data)
def get_events(environment, start_response, headers):
"""
Retrieve events
POST body should contain a JSON encoded version of:
{ namespace: namespace_name (optional),
stream : stream_name,
start_time : starting_time_as_kronos_time,
end_time : ending_time_as_kronos_time,
start_id : only_return_events_with_id_greater_than_me,
limit: optional_maximum_number_of_events,
order: ResultOrder.ASCENDING or ResultOrder.DESCENDING (default
ResultOrder.ASCENDING)
}
Either start_time or start_id should be specified. If a retrieval breaks
while returning results, you can send another retrieval request and specify
start_id as the last id that you saw. Kronos will only return events that
occurred after the event with that id.
"""
request_json = environment['json']
try:
stream = request_json['stream']
validate_stream(stream)
except Exception, e:
log.exception('get_events: stream validation failed for `%s`',
request_json.get('stream'))
start_response('400 Bad Request', headers)
yield marshal.dumps({ERRORS_FIELD: [repr(e)],
SUCCESS_FIELD: False})
return
def get_streams(self, namespace=None):
data = {}
if namespace is not None:
data['namespace'] = namespace
response = self.http_client.post(self.streams_path,
data=marshal.dumps(data),
buffered=True)
self.assertEqual(response.status_code, 200)
return response.data.splitlines()
def get_streams(self, namespace=None):
data = {}
if namespace is not None:
data['namespace'] = namespace
response = self.http_client.post(self.streams_path,
data=marshal.dumps(data),
buffered=True)
self.assertEqual(response.status_code, 200)
return map(marshal.loads, response.data.splitlines())
def delete(self, stream, start_time, end_time, start_id=None, namespace=None):
data = {'stream': stream, 'end_time': end_time}
if start_id:
data['start_id'] = start_id
else:
data['start_time'] = start_time
if namespace is not None:
data['namespace'] = namespace
response = self.http_client.post(path=self.delete_path,
data=marshal.dumps(data),
buffered=True)
self.assertEqual(response.status_code, 200)
response = marshal.loads(response.data)
self.assertTrue(response[SUCCESS_FIELD])
return response
def delete(self, stream, start_time, end_time, start_id=None, namespace=None):
data = {'stream': stream, 'end_time': end_time}
if start_id:
data['start_id'] = start_id
else:
data['start_time'] = start_time
if namespace is not None:
data['namespace'] = namespace
response = self.http_client.post(path=self.delete_path,
data=marshal.dumps(data),
buffered=True)
self.assertEqual(response.status_code, 200)
response = marshal.loads(response.data)
self.assertTrue(response[SUCCESS_FIELD])
return response
def put(self, stream_or_mapping, events=None, namespace=None):
data = {}
if isinstance(stream_or_mapping, dict):
data['events'] = stream_or_mapping
else:
self.assertTrue(events is not None)
data['events'] = {stream_or_mapping: events}
if namespace is not None:
data['namespace'] = namespace
response = self.http_client.post(path=self.put_path,
data=marshal.dumps(data),
buffered=True)
self.assertEqual(response.status_code, 200)
response = marshal.loads(response.data)
self.assertTrue(response[SUCCESS_FIELD])
return response
def put(self, stream_or_mapping, events=None, namespace=None):
data = {}
if isinstance(stream_or_mapping, dict):
data['events'] = stream_or_mapping
else:
self.assertTrue(events is not None)
data['events'] = {stream_or_mapping: events}
if namespace is not None:
data['namespace'] = namespace
response = self.http_client.post(path=self.put_path,
data=marshal.dumps(data),
buffered=True)
self.assertEqual(response.status_code, 200)
response = marshal.loads(response.data)
self.assertTrue(response[SUCCESS_FIELD])
return response
def get(self, stream, start_time, end_time, start_id=None, limit=None,
order=None, namespace=None):
data = {'stream': stream, 'end_time': end_time}
if start_id:
data['start_id'] = start_id
else:
data['start_time'] = start_time
if limit is not None:
data['limit'] = limit
if order is not None:
data['order'] = order
if namespace is not None:
data['namespace'] = namespace
response = self.http_client.post(path=self.get_path,
data=marshal.dumps(data),
buffered=True)
self.assertEqual(response.status_code, 200)
return map(marshal.loads, response.data.splitlines())
def get(self, stream, start_time, end_time, start_id=None, limit=None,
order=None, namespace=None):
data = {'stream': stream, 'end_time': end_time}
if start_id:
data['start_id'] = start_id
else:
data['start_time'] = start_time
if limit is not None:
data['limit'] = limit
if order is not None:
data['order'] = order
if namespace is not None:
data['namespace'] = namespace
response = self.http_client.post(path=self.get_path,
data=marshal.dumps(data),
buffered=True)
self.assertEqual(response.status_code, 200)
return map(marshal.loads, response.data.splitlines())
def wrapper(environment, start_response):
try:
start_time = time.time()
if function.func_name not in (
_serving_mode_endpoints[settings.serving_mode]):
start_response('403 Forbidden',
[('Content-Type', 'application/json')])
return marshal.dumps({
ERRORS_FIELD: [
'kronosd is configured to block access to this '
'endpoint.'
],
SUCCESS_FIELD:
False,
TOOK_FIELD:
'%fms' % (1000 * (time.time() - start_time))
})
req_method = environment['REQUEST_METHOD']
# If the request method is not allowed, return 405.
if req_method not in methods:
start_response('405 Method Not Allowed',
[('Allow', ', '.join(methods)),
('Content-Type', 'application/json')])
return marshal.dumps({
ERRORS_FIELD: ['%s method not allowed' % req_method],
SUCCESS_FIELD:
False,
TOOK_FIELD:
'%fms' % (1000 * (time.time() - start_time))
})
headers = []
remote_origin = environment.get('HTTP_ORIGIN')
if req_method == 'OPTIONS':
# This is a CORS preflight request so check that the remote domain is
# allowed and respond with appropriate CORS headers.
# http://www.html5rocks.com/static/images/cors_server_flowchart.png
if is_remote_allowed(remote_origin):
headers.extend([
('Access-Control-Allow-Origin', remote_origin),
('Access-Control-Allow-Credentials', 'true'),
('Access-Control-Allow-Headers', ', '.join(
('Accept', 'Content-Type', 'Origin',
'X-Requested-With'))),
('Access-Control-Allow-Methods',
', '.join(methods))
])
# We just tell the client that CORS is ok. Client will follow up
# with another request to get the answer.
start_response('200 OK', headers)
return ''
# All POST bodies must be json, so decode it here.
if req_method == 'POST':
try:
environment['json'] = marshal.loads(
environment['wsgi.input'].read())
except ValueError:
start_response('400 Bad Request',
[('Content-Type', 'application/json')])
return marshal.dumps({
ERRORS_FIELD: ['Request body must be valid JSON.'],
SUCCESS_FIELD:
False,
TOOK_FIELD:
'%fms' % (1000 * (time.time() - start_time))
})
# All responses are JSON.
headers.append(('Content-Type', 'application/json'))
if remote_origin:
headers.append(
('Access-Control-Allow-Origin', remote_origin))
response = function(environment, start_response, headers)
if not isinstance(response, types.GeneratorType):
response[TOOK_FIELD] = '%fms' % (
1000 * (time.time() - start_time))
response = marshal.dumps(response)
return response
except Exception, e:
log.exception('endpoint: uncaught exception!')
start_response('400 Bad Request',
[('Content-Type', 'application/json')])
return marshal.dumps({
ERRORS_FIELD: [repr(e)],
SUCCESS_FIELD:
False,
TOOK_FIELD:
'%fms' % (1000 * (time.time() - start_time))
})
def wrapper(environment, start_response):
try:
start_time = time.time()
if function.func_name not in (_serving_mode_endpoints
[settings.serving_mode]):
start_response('403 Forbidden',
[('Content-Type', 'application/json')])
return marshal.dumps({
ERRORS_FIELD: ['kronosd is configured to block access to this '
'endpoint.'],
SUCCESS_FIELD: False,
TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time))
})
req_method = environment['REQUEST_METHOD']
# If the request method is not allowed, return 405.
if req_method not in methods:
start_response('405 Method Not Allowed',
[('Allow', ', '.join(methods)),
('Content-Type', 'application/json')])
return marshal.dumps({
ERRORS_FIELD: ['%s method not allowed' % req_method],
SUCCESS_FIELD: False,
TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time))
})
headers = []
remote_origin = environment.get('HTTP_ORIGIN')
if req_method == 'OPTIONS':
# This is a CORS preflight request so check that the remote domain is
# allowed and respond with appropriate CORS headers.
# http://www.html5rocks.com/static/images/cors_server_flowchart.png
if is_remote_allowed(remote_origin):
headers.extend([
('Access-Control-Allow-Origin', remote_origin),
('Access-Control-Allow-Credentials', 'true'),
('Access-Control-Allow-Headers', ', '.join(
('Accept', 'Content-Type', 'Origin', 'X-Requested-With'))),
('Access-Control-Allow-Methods', ', '.join(methods))
])
# We just tell the client that CORS is ok. Client will follow up
# with another request to get the answer.
start_response('200 OK', headers)
return ''
# All POST bodies must be json, so decode it here.
if req_method == 'POST':
try:
environment['json'] = marshal.loads(environment['wsgi.input']
.read())
except ValueError:
start_response('400 Bad Request',
[('Content-Type', 'application/json')])
return marshal.dumps({
ERRORS_FIELD: ['Request body must be valid JSON.'],
SUCCESS_FIELD: False,
TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time))
})
# All responses are JSON.
headers.append(('Content-Type', 'application/json'))
if remote_origin:
headers.append(('Access-Control-Allow-Origin', remote_origin))
response = function(environment, start_response, headers)
if not isinstance(response, types.GeneratorType):
response[TOOK_FIELD] = '%fms' % (1000 * (time.time() - start_time))
response = marshal.dumps(response)
return response
except Exception, e:
log.exception('endpoint: uncaught exception!')
start_response('400 Bad Request',
[('Content-Type', 'application/json')])
return marshal.dumps({
ERRORS_FIELD: [repr(e)],
SUCCESS_FIELD: False,
TOOK_FIELD: '%fms' % (1000 * (time.time() - start_time))
})
