def __init__(self,
server_url,
parse_event_body=True,
last_event_seen=None):
self.server_url = server_url
self.last_event_seen = last_event_seen
self.deserializer = Deserializer()
self.parse_event_body = parse_event_body
self.stream_finished = False
def __init__(self, server_url, parse_event_body=True,
last_event_seen=None):
self.server_url = server_url
self.last_event_seen = last_event_seen
self.deserializer = Deserializer()
self.parse_event_body = parse_event_body
self.stream_finished = False
class SynchronousClient(object):
"""Synchronous client.
This client should be used in long-polling mode.
"""
def __init__(self, server_url, parse_event_body=True,
last_event_seen=None):
self.server_url = server_url
self.last_event_seen = last_event_seen
self.deserializer = Deserializer()
self.parse_event_body = parse_event_body
self.stream_finished = False
def receive_events(self):
url = self.server_url
if self.last_event_seen is not None:
url += '?last-seen=' + self.last_event_seen
connection = urllib2.urlopen(url)
data = connection.read()
evs = self.deserializer.deserialize(data, complete=True,
parse_body=self.parse_event_body)
connection.close()
if len(evs) > 0:
self.last_event_seen = evs[-1].event_id
for event in evs:
if (isinstance(event, Command)
and event.command == 'Stream-Finished'):
self.stream_finished = True
break
return [e for e in evs if not isinstance(e, Command)]
class SynchronousClient(object):
"""Synchronous client.
This client should be used in long-polling mode.
"""
def __init__(self,
server_url,
parse_event_body=True,
last_event_seen=None):
self.server_url = server_url
self.last_event_seen = last_event_seen
self.deserializer = Deserializer()
self.parse_event_body = parse_event_body
self.stream_finished = False
def receive_events(self):
url = self.server_url
if self.last_event_seen is not None:
url += '?last-seen=' + self.last_event_seen
connection = urllib2.urlopen(url)
data = connection.read()
evs = self.deserializer.deserialize(data,
complete=True,
parse_body=self.parse_event_body)
connection.close()
if len(evs) > 0:
self.last_event_seen = evs[-1].event_id
for event in evs:
if (isinstance(event, Command)
and event.command == 'Stream-Finished'):
self.stream_finished = True
break
return [e for e in evs if not isinstance(e, Command)]
def __init__(self, url, event_callback=None, error_callback=None,
connection_close_callback=None,
source_start_callback=None, source_finish_callback=None,
label=None, retrieve_missing_events=False,
ioloop=None, parse_event_body=True, separate_events=True,
reconnect=True, disable_compression=False):
"""Creates a new client for a given stream URL.
The client connects to the stream URL given by 'url'. For
every single received event, the 'event_callback' function is
invoked. It receives an event object as parameter.
If 'separate_events' is set to None, then the event callback
will receive a list of events instead of a single events.
A 'label' (string) may be set to this client. Setting a label
allows the client to save the id of the latest event it received
and ask for missed events when the client is run again.
Set 'retrieve_missing_events' to True in order to do that.
If 'retrieve_missing_events' is True, a non-empty label must be set.
If a 'ioloop' object is given, the client will block on it
apon calling the 'start()' method. If not, it will block on
the default 'ioloop' of Tornado.
When 'reconnect' is True (which is the default), the client tries
to automatically reconnect when it loses connection
with the server, following an exponential back-off mechanism.
"""
if retrieve_missing_events and not label:
raise ValueError('Retrieving missing events'
' requires a client label')
if isinstance(event_callback, Filter):
event_callback = event_callback.filter_event
self.url = url
self.event_callback = event_callback
self.error_callback = error_callback
self.connection_close_callback = connection_close_callback
self.source_start_callback = source_start_callback
self.source_finish_callback = source_finish_callback
self.label = label
if retrieve_missing_events:
self.status_file = self._create_status_file(False)
elif label:
self.status_file = self._create_status_file(True)
else:
self.status_file = None
self.last_event_id = None
self.ioloop = ioloop or tornado.ioloop.IOLoop.instance()
self.parse_event_body = parse_event_body
self.separate_events = separate_events
self._closed = False
self._looping = False
self._deserializer = Deserializer()
self.reconnect = reconnect
self.disable_compression = disable_compression
self.reconnection = ReconnectionManager()
def __init__(self,
url,
event_callback=None,
error_callback=None,
connection_close_callback=None,
source_start_callback=None,
source_finish_callback=None,
ioloop=None,
parse_event_body=True,
separate_events=True,
reconnect=True,
disable_compression=False):
"""Creates a new client for a given stream URL.
The client connects to the stream URL given by 'url'. For
every single received event, the 'event_callback' function is
invoked. It receives an event object as parameter.
If 'separate_events' is set to None, then the event callback
will receive a list of events instead of a single events.
If a 'ioloop' object is given, the client will block on it
apon calling the 'start()' method. If not, it will block on
the default 'ioloop' of Tornado.
"""
if isinstance(event_callback, Filter):
event_callback = event_callback.filter_event
self.url = url
self.event_callback = event_callback
self.error_callback = error_callback
self.connection_close_callback = connection_close_callback
self.source_start_callback = source_start_callback
self.source_finish_callback = source_finish_callback
self.ioloop = ioloop or tornado.ioloop.IOLoop.instance()
self.parse_event_body = parse_event_body
self.separate_events = separate_events
self._closed = False
self._looping = False
self._deserializer = Deserializer()
self.last_event = None
self.reconnect = reconnect
self.disable_compression = disable_compression
self.connection_attempts = 0
def count_events(filename):
"""Counts the number of events in a file and their total size.
Returns a tuple (num_events, num_bytes).
"""
num_bytes = 0
num_events = 0
if filename.endswith('.gz'):
file_ = gzip.GzipFile(filename, 'r')
else:
file_ = open(filename, 'r')
deserializer = Deserializer()
while True:
data = file_.read(1024)
if data == '':
break
evs = deserializer.deserialize(data, parse_body=False, complete=False)
num_bytes += len(data)
num_events += len(evs)
file_.close()
return num_events, num_bytes
def count_events(filename):
"""Counts the number of events in a file and their total size.
Returns a tuple (num_events, num_bytes).
"""
num_bytes = 0
num_events = 0
if filename.endswith('.gz'):
file_ = gzip.GzipFile(filename, 'r')
else:
file_ = open(filename, 'r')
deserializer = Deserializer()
while True:
data = file_.read(1024)
if data == '':
break
evs = deserializer.deserialize(data, parse_body=False, complete=False)
num_bytes += len(data)
num_events += len(evs)
file_.close()
return num_events, num_bytes
def __init__(self, url, event_callback=None, error_callback=None,
connection_close_callback=None,
source_start_callback=None, source_finish_callback=None,
ioloop=None, parse_event_body=True, separate_events=True,
reconnect=True):
"""Creates a new client for a given stream URL.
The client connects to the stream URL given by 'url'. For
every single received event, the 'event_callback' function is
invoked. It receives an event object as parameter.
If 'separate_events' is set to None, then the event callback
will receive a list of events instead of a single events.
If a 'ioloop' object is given, the client will block on it
apon calling the 'start()' method. If not, it will block on
the default 'ioloop' of Tornado.
"""
if isinstance(event_callback, Filter):
event_callback = event_callback.filter_event
self.url = url
self.event_callback = event_callback
self.error_callback = error_callback
self.connection_close_callback = connection_close_callback
self.source_start_callback = source_start_callback
self.source_finish_callback = source_finish_callback
self.ioloop = ioloop or tornado.ioloop.IOLoop.instance()
self.parse_event_body = parse_event_body
self.separate_events = separate_events
self._closed = False
self._looping = False
self._compressed = False
self._deserializer = Deserializer()
self.last_event = None
self.reconnect = reconnect
self.connection_attempts = 0
class AsyncStreamingClient(object):
"""Asynchronous client for a single event source.
If you need to receive events from several sources, use the class
'Client' instead.
"""
def __init__(self, url, event_callback=None, error_callback=None,
connection_close_callback=None,
source_start_callback=None, source_finish_callback=None,
ioloop=None, parse_event_body=True, separate_events=True,
reconnect=True):
"""Creates a new client for a given stream URL.
The client connects to the stream URL given by 'url'. For
every single received event, the 'event_callback' function is
invoked. It receives an event object as parameter.
If 'separate_events' is set to None, then the event callback
will receive a list of events instead of a single events.
If a 'ioloop' object is given, the client will block on it
apon calling the 'start()' method. If not, it will block on
the default 'ioloop' of Tornado.
"""
if isinstance(event_callback, Filter):
event_callback = event_callback.filter_event
self.url = url
self.event_callback = event_callback
self.error_callback = error_callback
self.connection_close_callback = connection_close_callback
self.source_start_callback = source_start_callback
self.source_finish_callback = source_finish_callback
self.ioloop = ioloop or tornado.ioloop.IOLoop.instance()
self.parse_event_body = parse_event_body
self.separate_events = separate_events
self._closed = False
self._looping = False
self._compressed = False
self._deserializer = Deserializer()
self.last_event = None
self.reconnect = reconnect
self.connection_attempts = 0
# self.data_history = []
def start(self, loop=False):
"""Starts the client.
This function has to be called in order to connect to the
streams and begin to receive events.
If 'loop' is True (the default is False), the server will
block on the ioloop until 'close()' is called.
"""
self._connect()
if loop:
self._looping = True
self.ioloop.start()
self._looping = False
def stop(self, notify_connection_close=True):
"""Stops and closes this client.
The client can no longer be used in the future.
Unless the keywork parameter `notify_connection_close` is set
to false, the connection close callback will be invoked if set
for this client.
If the server is blocked on the ioloop in the 'start()'
method, it is released.
Note: if the backend behind
'tornado.httpclient.AsyncHTTPClient()' is 'SimpleHTTPClient',
invoking 'stop()' does not actually close the HTTP connections
(as of Tornado branch master september 1st 2011).
"""
self._finish_internal(notify_connection_close)
def _connect(self):
http_client = AsyncHTTPClient()
if self.last_event is None:
url = self.url
else:
url = self.url + '?last-seen=' + self.last_event
req = HTTPRequest(url, streaming_callback=self._stream_callback,
request_timeout=0, connect_timeout=0)
http_client.fetch(req, self._request_callback)
self.connection_attempts += 1
def _reconnect(self):
logging.info('Reconnecting to the stream...')
t = 3 + random.expovariate(0.3)
self.ioloop.add_timeout(datetime.timedelta(seconds=t), self._connect)
def _finish_internal(self, notify_connection_close):
if self._closed:
return
if (notify_connection_close
and self.connection_close_callback is not None):
self.connection_close_callback(self)
if self._looping:
self.ioloop.stop()
self._looping = False
self._closed = True
def _stream_callback(self, data):
self.connection_attempts = 0
self._process_received_data(data)
def _request_callback(self, response):
if response.error:
if (self.connection_attempts < 5
and not response.error.code // 100 == 4
and not self._closed
and self.reconnect):
self._reconnect()
finish = False
else:
if self.error_callback is not None:
self.error_callback('Error in HTTP request',
http_error=response.error)
finish = True
elif len(response.body) > 0:
self._process_received_data(response.body)
finish = True
if finish:
logging.info('Finishing client')
self._finish_internal(True)
def _process_received_data(self, data):
global transferred_bytes
transferred_bytes += len(data)
evs = self._deserialize(data, parse_body=self.parse_event_body)
for e in evs:
logger.logger.event_delivered(e)
if self.event_callback is not None:
if not self.separate_events:
self.event_callback(evs)
else:
for ev in evs:
self.event_callback(ev)
if len(evs) > 0:
self.last_event = evs[-1].event_id
def _reset_compression(self):
self._compressed = True
self._decompressor = zlib.decompressobj()
def _deserialize(self, data, parse_body=True):
evs = []
event = None
compressed_len = len(data)
if self._compressed:
data = self._decompressor.decompress(data)
logger.logger.data_received(compressed_len, len(data))
self._deserializer.append_data(data)
event = self._deserializer.deserialize_next(parse_body=parse_body)
while event is not None:
if isinstance(event, Command):
if event.command == 'Set-Compression':
self._reset_compression()
pos = self._deserializer.data_consumed()
self._deserializer.reset()
evs.extend(self._deserialize(data[pos:], parse_body))
return evs
elif event.command == 'Event-Source-Started':
if self.source_start_callback:
self.source_start_callback()
evs.append(event)
elif event.command == 'Event-Source-Finished':
if self.source_finish_callback:
self.source_finish_callback()
evs.append(event)
elif event.command == 'Stream-Finished':
self._finish_internal(True)
## logging.info('Stream finished')
else:
evs.append(event)
event = self._deserializer.deserialize_next(parse_body=parse_body)
return evs
class AsyncStreamingClient(object):
"""Asynchronous client for a single event source.
If you need to receive events from several sources, use the class
'Client' instead.
"""
def __init__(self, url, event_callback=None, error_callback=None,
connection_close_callback=None,
source_start_callback=None, source_finish_callback=None,
label=None, retrieve_missing_events=False,
ioloop=None, parse_event_body=True, separate_events=True,
reconnect=True, disable_compression=False):
"""Creates a new client for a given stream URL.
The client connects to the stream URL given by 'url'. For
every single received event, the 'event_callback' function is
invoked. It receives an event object as parameter.
If 'separate_events' is set to None, then the event callback
will receive a list of events instead of a single events.
A 'label' (string) may be set to this client. Setting a label
allows the client to save the id of the latest event it received
and ask for missed events when the client is run again.
Set 'retrieve_missing_events' to True in order to do that.
If 'retrieve_missing_events' is True, a non-empty label must be set.
If a 'ioloop' object is given, the client will block on it
apon calling the 'start()' method. If not, it will block on
the default 'ioloop' of Tornado.
When 'reconnect' is True (which is the default), the client tries
to automatically reconnect when it loses connection
with the server, following an exponential back-off mechanism.
"""
if retrieve_missing_events and not label:
raise ValueError('Retrieving missing events'
' requires a client label')
if isinstance(event_callback, Filter):
event_callback = event_callback.filter_event
self.url = url
self.event_callback = event_callback
self.error_callback = error_callback
self.connection_close_callback = connection_close_callback
self.source_start_callback = source_start_callback
self.source_finish_callback = source_finish_callback
self.label = label
if retrieve_missing_events:
self.status_file = self._create_status_file(False)
elif label:
self.status_file = self._create_status_file(True)
else:
self.status_file = None
self.last_event_id = None
self.ioloop = ioloop or tornado.ioloop.IOLoop.instance()
self.parse_event_body = parse_event_body
self.separate_events = separate_events
self._closed = False
self._looping = False
self._deserializer = Deserializer()
self.reconnect = reconnect
self.disable_compression = disable_compression
self.reconnection = ReconnectionManager()
# self.data_history = []
def start(self, loop=False):
"""Starts the client.
This function has to be called in order to connect to the
streams and begin to receive events.
If 'loop' is True (the default is False), the server will
block on the ioloop until 'close()' is called.
"""
self._connect()
if loop:
self._looping = True
self.ioloop.start()
self._looping = False
def stop(self, notify_connection_close=True):
"""Stops and closes this client.
The client can no longer be used in the future.
Unless the keywork parameter `notify_connection_close` is set
to false, the connection close callback will be invoked if set
for this client.
If the server is blocked on the ioloop in the 'start()'
method, it is released.
Note: if the backend behind
'tornado.httpclient.AsyncHTTPClient()' is 'SimpleHTTPClient',
invoking 'stop()' does not actually close the HTTP connections
(as of Tornado branch master september 1st 2011).
"""
self._finish_internal(notify_connection_close)
def _connect(self):
http_client = AsyncHTTPClient()
last_event_received = self._read_last_event_id()
if last_event_received is None:
url = self.url
else:
url = self.url + '?last-seen=' + last_event_received
if not self.disable_compression:
headers = {'Accept-Encoding': 'deflate;q=1, identity;q=0.5'}
else:
headers = {'Accept-Encoding': 'identity'}
req = HTTPRequest(url, streaming_callback=self._stream_callback,
headers=headers,
request_timeout=0, connect_timeout=0)
http_client.fetch(req, self._request_callback)
self.reconnection.notify_failure()
logging.info('Connecting to {}'.format(self.url))
def _reconnect(self):
t = self.reconnection.compute_delay()
logging.info('Disconnected from {}. Next attempt in {:.02f}s'\
.format(self.url, t))
self.ioloop.add_timeout(datetime.timedelta(seconds=t), self._connect)
def _finish_internal(self, notify_connection_close):
if self._closed:
return
if (notify_connection_close
and self.connection_close_callback is not None):
self.connection_close_callback(self)
if self._looping:
self.ioloop.stop()
self._looping = False
self._closed = True
def _stream_callback(self, data):
self.reconnection.notify_success()
self._process_received_data(data)
def _request_callback(self, response):
if response.error:
if (not response.error.code // 100 == 4
and not self._closed
and self.reconnect):
self._reconnect()
finish = False
else:
if self.error_callback is not None:
self.error_callback('Error in HTTP request',
http_error=response.error)
finish = True
else:
if len(response.body) > 0:
self._process_received_data(response.body)
self._reconnect()
finish = False
if finish:
logging.info('Finishing client')
self._finish_internal(True)
def _process_received_data(self, data):
global transferred_bytes
transferred_bytes += len(data)
evs = self._deserialize(data, parse_body=self.parse_event_body)
if self.event_callback is not None:
if not self.separate_events:
self.event_callback(evs)
else:
for ev in evs:
self.event_callback(ev)
if len(evs) > 0:
self._write_last_event_id(evs[-1])
def _deserialize(self, data, parse_body=True):
evs = []
event = None
self._deserializer.append_data(data)
event = self._deserializer.deserialize_next(parse_body=parse_body)
while event is not None:
if isinstance(event, Command):
if event.command == 'Event-Source-Started':
if self.source_start_callback:
self.source_start_callback()
evs.append(event)
elif event.command == 'Event-Source-Finished':
if self.source_finish_callback:
self.source_finish_callback()
evs.append(event)
elif event.command == 'Stream-Finished':
self._finish_internal(True)
## logging.info('Stream finished')
else:
evs.append(event)
event = self._deserializer.deserialize_next(parse_body=parse_body)
return evs
def _create_status_file(self, overwrite):
dirname = '.ztreamy-client-' + self.label
if not os.path.exists(dirname):
os.makedirs(dirname)
status_file = os.path.join(dirname, base64.urlsafe_b64encode(self.url))
if overwrite:
mode = 'w'
else:
mode = 'a'
with open(status_file, mode=mode):
pass
return status_file
def _write_last_event_id(self, event):
if self.status_file is not None:
with open(self.status_file, mode='w') as f:
f.write(event.event_id)
else:
self.last_event_id = event.event_id
def _read_last_event_id(self):
if self.status_file is not None:
with open(self.status_file) as f:
event_id_read = f.read().strip()
if event_id_read:
event_id = event_id_read
else:
event_id = None
else:
event_id = self.last_event_id
return event_id
class AsyncStreamingClient(object):
"""Asynchronous client for a single event source.
If you need to receive events from several sources, use the class
'Client' instead.
"""
def __init__(self,
url,
event_callback=None,
error_callback=None,
connection_close_callback=None,
source_start_callback=None,
source_finish_callback=None,
ioloop=None,
parse_event_body=True,
separate_events=True,
reconnect=True,
disable_compression=False):
"""Creates a new client for a given stream URL.
The client connects to the stream URL given by 'url'. For
every single received event, the 'event_callback' function is
invoked. It receives an event object as parameter.
If 'separate_events' is set to None, then the event callback
will receive a list of events instead of a single events.
If a 'ioloop' object is given, the client will block on it
apon calling the 'start()' method. If not, it will block on
the default 'ioloop' of Tornado.
"""
if isinstance(event_callback, Filter):
event_callback = event_callback.filter_event
self.url = url
self.event_callback = event_callback
self.error_callback = error_callback
self.connection_close_callback = connection_close_callback
self.source_start_callback = source_start_callback
self.source_finish_callback = source_finish_callback
self.ioloop = ioloop or tornado.ioloop.IOLoop.instance()
self.parse_event_body = parse_event_body
self.separate_events = separate_events
self._closed = False
self._looping = False
self._deserializer = Deserializer()
self.last_event = None
self.reconnect = reconnect
self.disable_compression = disable_compression
self.connection_attempts = 0
# self.data_history = []
def start(self, loop=False):
"""Starts the client.
This function has to be called in order to connect to the
streams and begin to receive events.
If 'loop' is True (the default is False), the server will
block on the ioloop until 'close()' is called.
"""
self._connect()
if loop:
self._looping = True
self.ioloop.start()
self._looping = False
def stop(self, notify_connection_close=True):
"""Stops and closes this client.
The client can no longer be used in the future.
Unless the keywork parameter `notify_connection_close` is set
to false, the connection close callback will be invoked if set
for this client.
If the server is blocked on the ioloop in the 'start()'
method, it is released.
Note: if the backend behind
'tornado.httpclient.AsyncHTTPClient()' is 'SimpleHTTPClient',
invoking 'stop()' does not actually close the HTTP connections
(as of Tornado branch master september 1st 2011).
"""
self._finish_internal(notify_connection_close)
def _connect(self):
http_client = AsyncHTTPClient()
if self.last_event is None:
url = self.url
else:
url = self.url + '?last-seen=' + self.last_event
if not self.disable_compression:
headers = {'Accept-Encoding': 'deflate;q=1, identity;q=0.5'}
else:
headers = {'Accept-Encoding': 'identity'}
req = HTTPRequest(url,
streaming_callback=self._stream_callback,
headers=headers,
request_timeout=0,
connect_timeout=0)
http_client.fetch(req, self._request_callback)
self.connection_attempts += 1
def _reconnect(self):
logging.info('Reconnecting to the stream...')
t = 3 + random.expovariate(0.3)
self.ioloop.add_timeout(datetime.timedelta(seconds=t), self._connect)
def _finish_internal(self, notify_connection_close):
if self._closed:
return
if (notify_connection_close
and self.connection_close_callback is not None):
self.connection_close_callback(self)
if self._looping:
self.ioloop.stop()
self._looping = False
self._closed = True
def _stream_callback(self, data):
self.connection_attempts = 0
self._process_received_data(data)
def _request_callback(self, response):
if response.error:
if (self.connection_attempts < 5
and not response.error.code // 100 == 4
and not self._closed and self.reconnect):
self._reconnect()
finish = False
else:
if self.error_callback is not None:
self.error_callback('Error in HTTP request',
http_error=response.error)
finish = True
else:
if len(response.body) > 0:
self._process_received_data(response.body)
finish = True
if finish:
logging.info('Finishing client')
self._finish_internal(True)
def _process_received_data(self, data):
global transferred_bytes
transferred_bytes += len(data)
evs = self._deserialize(data, parse_body=self.parse_event_body)
for e in evs:
logger.logger.event_delivered(e)
if self.event_callback is not None:
if not self.separate_events:
self.event_callback(evs)
else:
for ev in evs:
self.event_callback(ev)
if len(evs) > 0:
self.last_event = evs[-1].event_id
def _deserialize(self, data, parse_body=True):
evs = []
event = None
compressed_len = len(data)
logger.logger.data_received(compressed_len, len(data))
self._deserializer.append_data(data)
event = self._deserializer.deserialize_next(parse_body=parse_body)
while event is not None:
if isinstance(event, Command):
if event.command == 'Event-Source-Started':
if self.source_start_callback:
self.source_start_callback()
evs.append(event)
elif event.command == 'Event-Source-Finished':
if self.source_finish_callback:
self.source_finish_callback()
evs.append(event)
elif event.command == 'Stream-Finished':
self._finish_internal(True)
## logging.info('Stream finished')
else:
evs.append(event)
event = self._deserializer.deserialize_next(parse_body=parse_body)
return evs
