def get_events(i):
client = KronosClient(node._host, blocking=True)
start_time = node.start_time + (i * delta)
if i == executor.parallelism - 1:
end_time = node.end_time
else:
end_time = start_time + delta - 1
return list(client.get(node.stream,
start_time,
end_time,
namespace=node._namespace))
def __init__(self,
source,
stream=None,
start_time=None,
end_time=None,
**kwargs):
self._config = app.config['DATA_SOURCES'][source]
self._host = self._config['url']
self._namespace = self._config['namespace']
self._kronos_client = KronosClient(self._host,
namespace=self._namespace)
self.type = Operator.Type.DATA_ACCESS
self.source = source
self.stream = stream
self.start_time = start_time
self.end_time = end_time
super(KronosSource, self).__init__(**kwargs)
class KronosSource(DataAccess):
def __init__(self,
source,
stream=None,
start_time=None,
end_time=None,
**kwargs):
self._config = app.config['DATA_SOURCES'][source]
self._host = self._config['url']
self._namespace = self._config['namespace']
self._kronos_client = KronosClient(self._host,
namespace=self._namespace)
self.type = Operator.Type.DATA_ACCESS
self.source = source
self.stream = stream
self.start_time = start_time
self.end_time = end_time
super(KronosSource, self).__init__(**kwargs)
@classmethod
def parse(cls, _dict):
kronos_source = KronosSource(**_dict)
# TODO(usmanm): Improve validation.
assert isinstance(kronos_source._config, types.DictType)
assert isinstance(kronos_source._host, types.StringTypes)
assert isinstance(kronos_source._namespace, types.StringTypes)
assert isinstance(kronos_source.source, types.StringTypes)
assert (kronos_source.stream is None
or isinstance(kronos_source.stream, types.StringTypes))
assert (kronos_source.start_time is None
or isinstance(kronos_source.start_time, int))
assert (kronos_source.end_time is None
or isinstance(kronos_source.end_time, int))
return kronos_source
def get_streams(self):
kstreams = self._kronos_client.get_streams()
return sorted(kstreams)
def get_schema(self, stream_name):
schema = self._kronos_client.infer_schema(stream_name)
return schema
def __init__(self, source, stream=None, start_time=None, end_time=None,
**kwargs):
self._config = app.config['DATA_SOURCES'][source]
self._host = self._config['url']
self._namespace = self._config['namespace']
self._kronos_client = KronosClient(self._host, namespace=self._namespace)
self.type = Operator.Type.DATA_ACCESS
self.source = source
self.stream = stream
self.start_time = start_time
self.end_time = end_time
super(KronosSource, self).__init__(**kwargs)
class KronosSource(DataAccess):
def __init__(self, source, stream=None, start_time=None, end_time=None,
**kwargs):
self._config = app.config['DATA_SOURCES'][source]
self._host = self._config['url']
self._namespace = self._config['namespace']
self._kronos_client = KronosClient(self._host, namespace=self._namespace)
self.type = Operator.Type.DATA_ACCESS
self.source = source
self.stream = stream
self.start_time = start_time
self.end_time = end_time
super(KronosSource, self).__init__(**kwargs)
@classmethod
def parse(cls, _dict):
kronos_source = KronosSource(**_dict)
# TODO(usmanm): Improve validation.
assert isinstance(kronos_source._config, types.DictType)
assert isinstance(kronos_source._host, types.StringTypes)
assert isinstance(kronos_source._namespace, types.StringTypes)
assert isinstance(kronos_source.source, types.StringTypes)
assert (kronos_source.stream is None or
isinstance(kronos_source.stream, types.StringTypes))
assert (kronos_source.start_time is None or
isinstance(kronos_source.start_time, int))
assert (kronos_source.end_time is None or
isinstance(kronos_source.end_time, int))
return kronos_source
def get_streams(self):
kstreams = self._kronos_client.get_streams()
return sorted(kstreams)
def get_schema(self, stream_name):
schema = self._kronos_client.infer_schema(stream_name)
return schema
def __init__(self):
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
if not hasattr(settings, 'KRONOS_MIDDLEWARE'):
raise ImproperlyConfigured
kronos_config = settings.KRONOS_MIDDLEWARE
if not isinstance(kronos_config, dict):
raise ImproperlyConfigured
if 'host' not in kronos_config:
raise ImproperlyConfigured
if 'stream' not in kronos_config:
raise ImproperlyConfigured
self.client = KronosClient(
kronos_config['host'],
blocking=kronos_config.get('blocking', False),
sleep_block=kronos_config.get('sleep_block', 0.1))
self.stream = kronos_config['stream']
self.namespace = kronos_config.get('namespace')
self.log_exception_stack_trace = kronos_config.get(
'log_exception_stack_trace', False)
self.fail_silently = kronos_config.get('fail_silently', True)
def __init__(self):
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
if not hasattr(settings, 'KRONOS_MIDDLEWARE'):
raise ImproperlyConfigured
kronos_config = settings.KRONOS_MIDDLEWARE
if not isinstance(kronos_config, dict):
raise ImproperlyConfigured
if 'host' not in kronos_config:
raise ImproperlyConfigured
if 'stream' not in kronos_config:
raise ImproperlyConfigured
self.client = KronosClient(
kronos_config['host'],
blocking=kronos_config.get('blocking', False),
sleep_block=kronos_config.get('sleep_block', 0.1))
self.stream = kronos_config['stream']
self.namespace = kronos_config.get('namespace')
self.log_exception_stack_trace = kronos_config.get(
'log_exception_stack_trace', False)
self.fail_silently = kronos_config.get('fail_silently', True)
class KronosLoggingMiddleware(object):
FORWARDED_IP_FIELDS = {'HTTP_X_FORWARDED_FOR',
'HTTP_X_FORWARDED_HOST',
'HTTP_X_FORWARDED_SERVER'}
def __init__(self):
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
if not hasattr(settings, 'KRONOS_MIDDLEWARE'):
raise ImproperlyConfigured
kronos_config = settings.KRONOS_MIDDLEWARE
if not isinstance(kronos_config, dict):
raise ImproperlyConfigured
if 'host' not in kronos_config:
raise ImproperlyConfigured
if 'stream' not in kronos_config:
raise ImproperlyConfigured
self.client = KronosClient(
kronos_config['host'],
blocking=kronos_config.get('blocking', False),
sleep_block=kronos_config.get('sleep_block', 0.1))
self.stream = kronos_config['stream']
self.namespace = kronos_config.get('namespace')
self.log_exception_stack_trace = kronos_config.get(
'log_exception_stack_trace', False)
self.fail_silently = kronos_config.get('fail_silently', True)
def _get_ip(self, request):
if not KronosLoggingMiddleware.FORWARDED_IP_FIELDS & set(request.META):
return request.META.get('REMOTE_ADDR', '').strip()
for field in KronosLoggingMiddleware.FORWARDED_IP_FIELDS:
if field in request.META:
return request.META[field].split(',')[-1].strip()
def process_request(self, request):
request._kronos_event = {
'start_time': time.time(),
'@time': kronos_time_now(),
'method': request.method.upper(),
'path': request.path,
'client_ip': self._get_ip(request)
}
get_dict = request.GET.dict()
if get_dict:
request._kronos_event['get_params'] = get_dict
try:
if not request.body:
return
except:
# The following exception is thrown if the body could not be read.
# Exception("You cannot access body after reading from request's data
# stream")
return
post_dict = request.POST.dict()
if (len(post_dict) == 1 and
post_dict.values()[0] == '' and
post_dict.keys()[0] == request.body):
# There is no real *form* POST.
try:
request._kronos_event['body'] = json.loads(request.body)
except ValueError:
request._kronos_event['body'] = request.body
else:
request._kronos_event['post_params'] = post_dict
def process_exception(self, request, exception):
self.client._log_exception(
request._kronos_event, exception,
sys.exc_info()[2] if self.log_exception_stack_trace else None)
def process_response(self, request, response):
start_time = request._kronos_event.pop('start_time')
request._kronos_event['duration'] = time.time() - start_time
try:
self.client.put({self.stream: [request._kronos_event]},
namespace=self.namespace)
except:
if self.fail_silently:
log.error('Failed to log event to Kronos.', exc_info=True)
else:
raise
return response
class KronosLoggingMiddleware(object):
FORWARDED_IP_FIELDS = {
'HTTP_X_FORWARDED_FOR', 'HTTP_X_FORWARDED_HOST',
'HTTP_X_FORWARDED_SERVER'
}
def __init__(self):
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
if not hasattr(settings, 'KRONOS_MIDDLEWARE'):
raise ImproperlyConfigured
kronos_config = settings.KRONOS_MIDDLEWARE
if not isinstance(kronos_config, dict):
raise ImproperlyConfigured
if 'host' not in kronos_config:
raise ImproperlyConfigured
if 'stream' not in kronos_config:
raise ImproperlyConfigured
self.client = KronosClient(
kronos_config['host'],
blocking=kronos_config.get('blocking', False),
sleep_block=kronos_config.get('sleep_block', 0.1))
self.stream = kronos_config['stream']
self.namespace = kronos_config.get('namespace')
self.log_exception_stack_trace = kronos_config.get(
'log_exception_stack_trace', False)
self.fail_silently = kronos_config.get('fail_silently', True)
def _get_ip(self, request):
if not KronosLoggingMiddleware.FORWARDED_IP_FIELDS & set(request.META):
return request.META.get('REMOTE_ADDR', '').strip()
for field in KronosLoggingMiddleware.FORWARDED_IP_FIELDS:
if field in request.META:
return request.META[field].split(',')[-1].strip()
def process_request(self, request):
request._kronos_event = {
'start_time': time.time(),
'@time': kronos_time_now(),
'method': request.method.upper(),
'path': request.path,
'client_ip': self._get_ip(request)
}
get_dict = request.GET.dict()
if get_dict:
request._kronos_event['get_params'] = get_dict
try:
if not request.body:
return
except:
# The following exception is thrown if the body could not be read.
# Exception("You cannot access body after reading from request's data
# stream")
return
post_dict = request.POST.dict()
if (len(post_dict) == 1 and post_dict.values()[0] == ''
and post_dict.keys()[0] == request.body):
# There is no real *form* POST.
try:
request._kronos_event['body'] = json.loads(request.body)
except ValueError:
request._kronos_event['body'] = request.body
else:
request._kronos_event['post_params'] = post_dict
def process_exception(self, request, exception):
self.client._log_exception(
request._kronos_event, exception,
sys.exc_info()[2] if self.log_exception_stack_trace else None)
def process_response(self, request, response):
start_time = request._kronos_event.pop('start_time')
request._kronos_event['duration'] = time.time() - start_time
try:
self.client.put({self.stream: [request._kronos_event]},
namespace=self.namespace)
except:
if self.fail_silently:
log.error('Failed to log event to Kronos.', exc_info=True)
else:
raise
return response
from pykronos.client import ID_FIELD
from pykronos.client import TIMESTAMP_FIELD
from pykronos.client import KronosClient
from pykronos.client import ResultOrder
from pykronos.common.time import datetime_to_kronos_time
from datetime import datetime
from datetime import timedelta
from dateutil.tz import tzutc
# Creating a client
"""
Create a Kronos client with the URL of a running server. Optionally
provide a `namespace` to explicitly work with events in a particular
namespace.
"""
kc = KronosClient('http://localhost:8151', namespace='demo')
start = datetime.now(tz=tzutc())
## A nonblocking client
"""
Pass a `blocking=False` to the KronosClient constructor for a client
that won't block on the Kronos server when you insert data. A
background thread will batch up data and send it to the server. This
is useful for logging/metrics collection on machines that can't wait
for a write acknowledgement. An optional `sleep_block` argument,
defaulting to `0.1` specifies how many seconds to wait between batches
to the server. If the process running the client crashes before
flushing events, those events will be lost.
"""
nonblocking = KronosClient('http://localhost:8151', namespace='demo',
blocking=False)
def execute(self, node, executor):
client = KronosClient(node._host, blocking=True)
return client.get(node.stream,
node.start_time,
node.end_time,
namespace=node._namespace)
from pykronos.client import TIMESTAMP_FIELD
from pykronos.client import KronosClient
from pykronos.client import ResultOrder
from pykronos.common.time import datetime_to_kronos_time
from datetime import datetime
from datetime import timedelta
from dateutil.tz import tzutc
"""
## Creating A Client
Create a Kronos client with the URL of a running server. Optionally
provide a `namespace` to explicitly work with events in a particular
namespace.
"""
kc = KronosClient('http://localhost:8151', namespace='kronos')
start = datetime.now(tz=tzutc())
"""
### A Non-blocking Client
Pass a `blocking=False` to the KronosClient constructor for a client
that won't block on the Kronos server when you insert data. A
background thread will batch up data and send it to the server. This
is useful for logging/metrics collection on machines that can't wait
for a write acknowledgement. An optional `sleep_block` argument,
defaulting to `0.1` specifies how many seconds to wait between batches
to the server. If the process running the client crashes before
flushing events, those events will be lost.
"""
nonblocking = KronosClient('http://localhost:8151', namespace='kronos',