def subscribe_all(self, callback, start_from_begining = False):
start_position = 0 if start_from_begining else -1
self.__subscribersAll=SubscribeAllInfo(start_position, start_position, callback)
self._should_subscribe_all=True
def subscribe_all(self, callback, start_from_begining=False):
start_position = 0 if start_from_begining else -1
self.__subscribersAll = SubscribeAllInfo(start_position,
start_position, callback)
self._should_subscribe_all = True
class Client():
def __init__(self, ip_address="127.0.0.1", port = 2113):
self._base_url = "http://{0}:{1}".format(ip_address, str(port))
self._headers = {"content-type" : "application/json", "accept" : "application/json", "extensions" : "json"}
self._read_batch_size = 20
self._tornado_http_sender = TornadoHttpSender()
self._subscribers = []
self._subscribers_thread = thread.start_new_thread(self._handle_subscribers, ())
self._subscribers_all_thread = thread.start_new_thread(self._handle_subscribers_all, ())
self._should_subscribe_all=False
self.projections = Projections(ip_address, port)
def resume(self):
IOLoop.instance().stop()
def wait(self):
IOLoop.instance().start()
def _sync_success(self, response):
self.resume()
response = SyncResponse(True, response)
return response
def _sync_failed(self, response):
self.resume()
response = SyncResponse(False, response)
return response
def create_stream(self, stream_id, metadata=""):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.create_stream_async(stream_id, metadata, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return
else:
raise result.response
def create_stream_async(self, stream_id, metadata="", on_success = None, on_failed = None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_create_stream(stream_id, metadata, on_success, on_failed)
def _start_create_stream(self, stream_id, metadata, on_success, on_failed):
Ensure.is_not_empty_string(stream_id, "stream_id")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
body = to_json(CreateStreamRequestBody(stream_id, metadata))
url = "{0}/streams".format(self._base_url)
self._tornado_http_sender.send_async(url, "POST", self._headers, body, lambda x: self._create_stream_callback(x, on_success, on_failed))
def _create_stream_callback(self, response, on_success, on_failed):
if response.code==201:
on_success(response)
else:
on_failed(FailedAnswer(response.code, response.error.message))
def delete_stream(self, stream_id, expected_version=-2):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.delete_stream_async(stream_id,expected_version, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return
else:
raise result.response
def delete_stream_async(self, stream_id, expected_version=-2, on_success = None, on_failed = None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_delete_stream(stream_id,expected_version, on_success, on_failed)
def _start_delete_stream(self,stream_id, expected_version, on_success, on_failed, ):
Ensure.is_not_empty_string(stream_id, "stream_id")
Ensure.is_number(expected_version, "expected_version")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
stream_id = quote(stream_id)
url = "{0}/streams/{1}".format(self._base_url, stream_id)
body = to_json(DeleteStreamRequestBody(expected_version))
self._tornado_http_sender.send_async(url, "DELETE", self._headers, body,
lambda x: self._delete_stream_callback(x, on_success, on_failed))
def _delete_stream_callback(self, response, on_success, on_failed):
if response.code==204:
on_success(response)
else:
on_failed(FailedAnswer(response.code,response.error.message))
def append_to_stream(self, stream_id, events, expected_version=-2):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.append_to_stream_async(stream_id, events, expected_version, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return
else:
raise result.response
def append_to_stream_async(self, stream_id, events, expected_version=-2, on_success = None, on_failed = None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_append_to_stream(stream_id, events, expected_version, on_success, on_failed)
def _start_append_to_stream(self,stream_id, events, expected_version, on_success, on_failed):
Ensure.is_not_empty_string(stream_id, "stream_id")
Ensure.is_number(expected_version, "expected_version")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
stream_id = quote(stream_id)
if type(events) is not list:
events = [events]
body = to_json(AppendToStreamRequestBody(expected_version, events))
url = "{0}/streams/{1}".format(self._base_url,stream_id)
self._tornado_http_sender.send_async(url,"POST", self._headers, body,
lambda x: self._append_to_stream_callback(x, on_success, on_failed))
def _append_to_stream_callback(self, response, on_success,on_failed):
if response.code==201:
on_success(response)
else:
on_failed(FailedAnswer(response.code,response.error.message))
def read_event(self,stream_id, event_number):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.read_event_async(stream_id, event_number, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return result.response
else:
raise result.response
def read_event_async(self,stream_id, event_number, on_success = None, on_failed = None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_read_event(stream_id, event_number, on_success, on_failed)
def _start_read_event(self,stream_id, event_number, on_success, on_failed):
Ensure.is_not_empty_string(stream_id, "stream_id")
Ensure.is_greater_equal_number_then(0,event_number, "event_number")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
stream_id = quote(stream_id)
url = "{0}/streams/{1}/event/{2}".format(self._base_url, stream_id, str(event_number))
self._tornado_http_sender.send_async(url, "GET", self._headers, None,
lambda x: self._read_event_callback(x, on_success, on_failed))
def _read_event_callback(self, response, on_success, on_failed):
if response.code!=200:
on_failed(FailedAnswer(response.code,response.error.message))
return
responseContent = response.body
event = json.loads(responseContent)
on_success(self._json_to_event(event))
def read_stream_events_backward(self, stream_id, start_position, count):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.read_stream_events_backward_async(stream_id, start_position, count, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return result.response
else:
raise result.response
def read_stream_events_backward_async(self, stream_id, start_position, count, on_success = None, on_failed = None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_read_stream_events_backward(stream_id, start_position, count, on_success, on_failed)
def _start_read_stream_events_backward(self, stream_id, start_position, count, on_success, on_failed):
Ensure.is_not_empty_string(stream_id, "stream_id")
Ensure.is_greater_equal_number_then(-1,start_position, "start_position")
Ensure.is_greater_equal_number_then(1, count, "count")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
stream_id = quote(stream_id)
events = []
batch_counter=0
params = ReadEventsData
params.stream_id = stream_id
params.start_position = start_position
params.count = count
params.batch_counter = batch_counter
params.events = events
self._read_batch_events_backward(params, on_success, on_failed)
def _read_batch_events_backward(self, params, on_success, on_failed, events_count=None):
if events_count is not None and events_count < self._read_batch_size:
on_success(params.events)
return
if params.batch_counter < params.count:
if params.count < params.batch_counter + self._read_batch_size:
params.batch_length = params.count - params.batch_counter
else :
params.batch_length = self._read_batch_size
url = self._base_url + "/streams/{0}/range/{1}/{2}?format=json".format(
params.stream_id,
str(params.start_position-params.batch_counter),
str(params.batch_length))
params.batch_counter+=self._read_batch_size
self._tornado_http_sender.send_async(url, "GET", self._headers, None,
lambda x: self._read_stream_events_backward_async_callback(x, params, on_success, on_failed))
else:
on_success(params.events)
def _read_stream_events_backward_async_callback(self, response, params, on_success, on_failed):
if response.code!=200:
on_failed(FailedAnswer(response.code, "Error occur while reading batch: " + response.error.message))
return
try:
response = json.loads(response.body)
if len(response["entries"])==0:
if len(params.events)!=0:
on_success(params.events)
return
url = "{0}/streams/{1}".format(self._base_url, params.stream_id)
self._tornado_http_sender.send_async(url, "GET", self._headers, None,
lambda x: self._on_read_events_first_response_entries_empty(x, params, on_success, on_failed))
return
batch_events = []
for uri in response["entries"]:
url = uri["links"]
for ur in url:
if "type" not in ur:
continue
if ur["type"] == "application/json":
url = ur["uri"]
self._tornado_http_sender.send_async(url, "GET", self._headers, None,
lambda x: self._event_read_callback(x, params,batch_events, on_success, on_failed,len(response["entries"])))
break
except:
on_failed(FailedAnswer(response, response))
return
def _on_read_events_first_response_entries_empty(self, response, params, on_success, on_failed):
if response.code!=200:
on_failed(FailedAnswer(response.code, "Error occur while reading first page: " + response.error.message))
return
response = json.loads(response.body)
if len(response["entries"])==0:
on_success(params.events)
return
last_event_number = int(response["entries"][0]["id"].split("/")[-1])
if params.count < params.start_position - last_event_number:
on_success(params.events)
return
params.count = params.count + last_event_number - params.start_position
params.start_position = last_event_number
params.batch_counter = 0
self._read_batch_events_backward(params, on_success, on_failed)
def _event_read_callback(self, response, params, batch_events, on_success, on_failed, events_count):
if response.code !=200:
on_failed(FailedAnswer(response.code, response.error.message))
return
try:
event = json.loads(response.body)
batch_events.append(self._json_to_event(event))
if len(batch_events)==events_count:
batch_events = sorted(batch_events, key=lambda ev: ev.event_number, reverse=True)
params.events+=batch_events
self._read_batch_events_backward(params, on_success, on_failed, events_count)
except:
on_failed(FailedAnswer(response.code, "Error occure while reading event: " + response.body))
return
def read_stream_events_forward(self, stream_id, start_position, count):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(list(reversed(x))))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.read_stream_events_forward_async(stream_id, start_position, count, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return result.response
else:
raise result.response
def read_stream_events_forward_async(self, stream_id, start_position, count, on_success = None, on_failed = None):
new_start_position = start_position+count-1
if new_start_position > sys.maxint:
count -= (new_start_position-sys.maxint)
new_start_position = sys.maxint
self._start_read_stream_events_backward(stream_id, int(new_start_position), int(count), on_success, on_failed)
def read_all_events_backward(self, prepare_position, commit_position, count):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self._start_read_all_events_backward(prepare_position, commit_position, count, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return result.response
else:
raise result.response
def read_all_events_backward_async(self, prepare_position, commit_position, count, on_success = None, on_failed = None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_read_all_events_backward(prepare_position, commit_position, count, on_success, on_failed)
def _start_read_all_events_backward(self, prepare_position, commit_position, count, on_success, on_failed):
Ensure.is_number(prepare_position, "prepare_position")
Ensure.is_number(commit_position, "commit_position")
Ensure.is_greater_equal_number_then(1, count, "count")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
events = []
batch_counter = 0
params = ReadEventsData
params.prepare_position = prepare_position
params.commit_position = commit_position
params.count = count
params.batch_counter = batch_counter
params.events = events
self._read_batch_all_events_backward(params, on_success, on_failed)
def _read_batch_all_events_backward(self, params, on_success, on_failed, events_count=None):
if events_count is not None and events_count < self._read_batch_size:
on_success(AllEventsAnswer(params.events, params.prepare_position, params.commit_position))
return
if params.batch_counter < params.count:
hexStartPosition = self._convert_to_hex(params.prepare_position) + self._convert_to_hex(params.commit_position)
if params.count < params.batch_counter + self._read_batch_size:
params.batch_length = params.count - params.batch_counter
else :
params.batch_length = self._read_batch_size
url = "{0}/streams/$all/before/{1}/{2}".format(self._base_url, hexStartPosition, str(params.batch_length))
params.batch_counter+=self._read_batch_size
self._tornado_http_sender.send_async(url, "GET", self._headers, None,
lambda x: self._read_all_events_backward_page_callback(x, params, on_success, on_failed))
else:
on_success(AllEventsAnswer(params.events, params.prepare_position, params.commit_position))
def _read_all_events_backward_page_callback(self, response, params, on_success, on_failed):
if response.code!=200:
on_failed(FailedAnswer(response.code,"Error occur while reading links: " + response.error.message))
read_line = response.body
body = json.loads(read_line)
params.prepare_position = self._get_prepare_position(body["links"][4]["uri"])
params.commit_position = self._get_commit_position(body["links"][4]["uri"])
try:
if body["entries"] == []:
if len(params.events)!=0:
on_success(AllEventsAnswer("", 0,0 ))
else:
self._start_read_all_events_backward(-1,-1, params.count, on_success, on_failed)
return
params.events_count=len(body["entries"])
params.events_count_in_current_butch = len(body["entries"])
params.batch_events = {}
params.url_number_dictionary = {}
event_number = 0
for uri in body["entries"]:
url = uri["links"]
for ur in url:
try:
if ur["type"] == "application/json":
url = ur["uri"]
params.url_number_dictionary[url] = event_number
event_number+=1
self._tornado_http_sender.send_async(url, "GET", self._headers, None,
lambda x: self._read_all_events_backward_callback(x, params, on_success, on_failed))
break
except:
continue
except:
on_failed(FailedAnswer(response.code,response.error.message))
return
def _read_all_events_backward_callback(self, response, params, on_success, on_failed):
is_failed = True
if response.code == 410:
is_failed = False
params.events_count_in_current_butch-=1
elif response.code == 200:
is_failed = False
try:
event = json.loads(response.body)
params.batch_events[params.url_number_dictionary[response.request.url]]= self._json_to_event(event)
except:
on_failed(FailedAnswer(response.code, "Error occure while reading event: " + response.error.message))
return
if len(params.batch_events)==params.events_count_in_current_butch:
i = 0
while i<params.events_count:
try:
params.events.append(params.batch_events[i])
except:
self._do_nothing(i)
i+=1
self._read_batch_all_events_backward(params, on_success, on_failed, params.events_count)
return
if is_failed:
on_failed(FailedAnswer(response.code,"Error occure while reading event: "+response.error.message))
def read_all_events_forward(self, prepare_position, commit_position, count):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.read_all_events_forward_async(prepare_position, commit_position, count, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return result.response
else:
raise result.response
def read_all_events_forward_async(self, prepare_position, commit_position, count, on_success = None, on_failed = None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_read_all_events_forward(prepare_position, commit_position, count, on_success, on_failed)
def _start_read_all_events_forward(self, prepare_position, commit_position, count, on_success, on_failed):
Ensure.is_greater_equal_number_then(-1, prepare_position, "prepare_position")
Ensure.is_greater_equal_number_then(-1, commit_position, "commit_position")
Ensure.is_greater_equal_number_then(1, count, "count")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
events = []
batch_counter = 0
params = ReadEventsData
params.prepare_position = prepare_position
params.commit_position = commit_position
params.count = count
params.batch_counter = batch_counter
params.events = events
self._read_batch_all_events_forward(params, on_success, on_failed)
def _read_batch_all_events_forward(self, params, on_success, on_failed, events_count=None):
if events_count is not None and events_count<self._read_batch_size:
on_success(AllEventsAnswer(params.events, params.prepare_position, params.commit_position))
return
if params.batch_counter<params.count:
hexStartPosition = self._convert_to_hex(params.prepare_position) + self._convert_to_hex(params.commit_position)
if params.batch_counter+self._read_batch_size>params.count:
params.batch_length = params.count - params.batch_counter
else :
params.batch_length = self._read_batch_size
url = "{0}/streams/$all/after/{1}/{2}".format(self._base_url, hexStartPosition, str(params.batch_length))
params.batch_counter+=self._read_batch_size
self._tornado_http_sender.send_async(url, "GET", self._headers, None, lambda x: \
self._read_all_events_forward_page_callback(x, params, on_success, on_failed))
else:
on_success(AllEventsAnswer(params.events, params.prepare_position, params.commit_position))
def _read_all_events_forward_page_callback(self, response, params, on_success, on_failed):
if response.code!=200:
on_failed(FailedAnswer(response.code,"Error occur while reading links: " + response.error.message))
read_line = response.body
body = json.loads(read_line)
params.prepare_position = self._get_prepare_position(body["links"][3]["uri"])
params.commit_position = self._get_commit_position(body["links"][3]["uri"])
try:
if not body["entries"]:
on_success(AllEventsAnswer(params.events, params.prepare_position, params.commit_position ))
return
params.events_count=len(body["entries"])
params.events_count_in_current_butch = len(body["entries"])
params.batch_events = {}
params.url_number_dictionary = {}
event_number = 0
for uri in body["entries"]:
url = uri["links"]
for ur in url:
try:
if ur["type"] == "application/json":
url = ur["uri"]
params.url_number_dictionary[url] = event_number
event_number+=1
self._tornado_http_sender.send_async(url, "GET", self._headers, None,
lambda x: self._read_all_events_forward_callback(x, params, on_success, on_failed))
break
except:
continue
except:
on_failed(FailedAnswer(response.code,response.error.message))
return
def _read_all_events_forward_callback(self, response, params, on_success, on_failed):
is_failed = True
if response.code == 410:
is_failed = False
params.events_count_in_current_butch-=1
elif response.code == 200:
is_failed = False
try:
event = json.loads(response.body)
params.batch_events[params.url_number_dictionary[response.request.url]]= self._json_to_event(event)
except:
on_failed(FailedAnswer(response.code, "Error occure while reading event: " + response.error.message))
return
if len(params.batch_events)==params.events_count_in_current_butch:
i = params.events_count-1
while i>=0:
try:
params.events.append(params.batch_events[i])
except:
self._do_nothing(i)
i-=1
self._read_batch_all_events_forward(params, on_success, on_failed, params.events_count)
return
if is_failed:
on_failed(FailedAnswer(response.code, "Error occure while reading event: " + response.error.message))
def _get_prepare_position(self, link):
position = link.split("/")[6]
result =int(position[0:16], 16)
return result
def _get_commit_position(self, link):
position = link.split("/")[6]
result =int(position[16:32], 16)
return result
def _convert_to_hex(self, number):
if number<0:
return"ffffffffffffffff"
hexVal = hex(number)
number = str(hexVal).split("x")[1]
while len(number)!=16:
number = "0"+number
return number
def subscribe(self, stream_id, callback, start_from_begining = False):
start_position = 0 if start_from_begining else self._get_stream_event_position(stream_id) + 1
self._subscribers.append(SubscribeInfo(stream_id,start_position, callback))
def _handle_subscribers(self):
while True:
if len(self._subscribers)>0:
processed_streams=0
for i in range(len(self._subscribers)):
self.read_stream_events_forward_async(self._subscribers[i].stream_id, self._subscribers[i].last_position, sys.maxint,
lambda s: self._handle_subscribers_success(s, len(self._subscribers), processed_streams, i),
lambda s: self._handle_subscribers_failed(len(self._subscribers), processed_streams))
self.wait()
time.sleep(1)
def _handle_subscribers_success(self, response, stream_count, processed_streams, subscriber_index):
processed_streams+=1
self._subscribers[subscriber_index].last_position += len(response)
for i in reversed(response):
self._subscribers[subscriber_index].callback(i)
if processed_streams==stream_count:
self.resume()
def _handle_subscribers_failed(self, stream_count, processed_streams):
processed_streams+=1
if processed_streams==stream_count:
self.resume()
def unsubscribe(self, stream_id):
for i in self._subscribers:
if i.stream_id == stream_id:
self._subscribers.remove(i)
def subscribe_all(self, callback, start_from_begining = False):
start_position = 0 if start_from_begining else -1
self.__subscribersAll=SubscribeAllInfo(start_position, start_position, callback)
self._should_subscribe_all=True
def _handle_subscribers_all(self):
while True:
if self._should_subscribe_all:
if self.__subscribersAll.commit_position ==-1:
answer = self.read_all_events_backward(self.__subscribersAll.prepare_position,self.__subscribersAll.commit_position, 1)
else:
try:
answer = self.read_all_events_forward(self.__subscribersAll.prepare_position,self.__subscribersAll.commit_position, sys.maxint-1)
for i in answer.events:
self.__subscribersAll.callback(i)
except:
time.sleep(0.1)
self.__subscribersAll.commit_position = int(answer.commit_position)
self.__subscribersAll.prepare_position = int(answer.prepare_position)
time.sleep(1)
def unsubscribe_all(self):
self._should_subscribe_all = False
def _get_stream_event_position(self,stream_id):
try:
events = self.read_stream_events_backward(stream_id, -1, 1)
return int(events[0].event_number)
except:
return 0
def _do_nothing(self, x):
a=0
def _json_to_event(self, json):
event = ReadEvent(json["data"], json["metadata"], json["eventType"], json["eventNumber"], )
return event
class Client():
def __init__(self, ip_address="127.0.0.1", port=2113):
self._base_url = "http://{0}:{1}".format(ip_address, str(port))
self._headers = {
"content-type": "application/json",
"accept": "application/json",
"extensions": "json"
}
self._read_batch_size = 20
self._tornado_http_sender = TornadoHttpSender()
self._subscribers = []
self._subscribers_thread = thread.start_new_thread(
self._handle_subscribers, ())
self._subscribers_all_thread = thread.start_new_thread(
self._handle_subscribers_all, ())
self._should_subscribe_all = False
self.projections = Projections(ip_address, port)
def resume(self):
IOLoop.instance().stop()
def wait(self):
IOLoop.instance().start()
def _sync_success(self, response):
self.resume()
response = SyncResponse(True, response)
return response
def _sync_failed(self, response):
self.resume()
response = SyncResponse(False, response)
return response
def create_stream(self, stream_id, metadata=""):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.create_stream_async(stream_id, metadata, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return
else:
raise result.response
def create_stream_async(self,
stream_id,
metadata="",
on_success=None,
on_failed=None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_create_stream(stream_id, metadata, on_success, on_failed)
def _start_create_stream(self, stream_id, metadata, on_success, on_failed):
Ensure.is_not_empty_string(stream_id, "stream_id")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
body = to_json(CreateStreamRequestBody(stream_id, metadata))
url = "{0}/streams".format(self._base_url)
self._tornado_http_sender.send_async(
url, "POST", self._headers, body,
lambda x: self._create_stream_callback(x, on_success, on_failed))
def _create_stream_callback(self, response, on_success, on_failed):
if response.code == 201:
on_success(response)
else:
on_failed(FailedAnswer(response.code, response.error.message))
def delete_stream(self, stream_id, expected_version=-2):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.delete_stream_async(stream_id, expected_version, on_success,
on_failed)
self.wait()
result = queue.popleft()
if result.success:
return
else:
raise result.response
def delete_stream_async(self,
stream_id,
expected_version=-2,
on_success=None,
on_failed=None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_delete_stream(stream_id, expected_version, on_success,
on_failed)
def _start_delete_stream(
self,
stream_id,
expected_version,
on_success,
on_failed,
):
Ensure.is_not_empty_string(stream_id, "stream_id")
Ensure.is_number(expected_version, "expected_version")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
stream_id = quote(stream_id)
url = "{0}/streams/{1}".format(self._base_url, stream_id)
body = to_json(DeleteStreamRequestBody(expected_version))
self._tornado_http_sender.send_async(
url, "DELETE", self._headers, body,
lambda x: self._delete_stream_callback(x, on_success, on_failed))
def _delete_stream_callback(self, response, on_success, on_failed):
if response.code == 204:
on_success(response)
else:
on_failed(FailedAnswer(response.code, response.error.message))
def append_to_stream(self, stream_id, events, expected_version=-2):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.append_to_stream_async(stream_id, events, expected_version,
on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return
else:
raise result.response
def append_to_stream_async(self,
stream_id,
events,
expected_version=-2,
on_success=None,
on_failed=None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_append_to_stream(stream_id, events, expected_version,
on_success, on_failed)
def _start_append_to_stream(self, stream_id, events, expected_version,
on_success, on_failed):
Ensure.is_not_empty_string(stream_id, "stream_id")
Ensure.is_number(expected_version, "expected_version")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
stream_id = quote(stream_id)
if type(events) is not list:
events = [events]
body = to_json(AppendToStreamRequestBody(expected_version, events))
url = "{0}/streams/{1}".format(self._base_url, stream_id)
self._tornado_http_sender.send_async(
url, "POST", self._headers,
body, lambda x: self._append_to_stream_callback(
x, on_success, on_failed))
def _append_to_stream_callback(self, response, on_success, on_failed):
if response.code == 201:
on_success(response)
else:
on_failed(FailedAnswer(response.code, response.error.message))
def read_event(self, stream_id, event_number):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.read_event_async(stream_id, event_number, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return result.response
else:
raise result.response
def read_event_async(self,
stream_id,
event_number,
on_success=None,
on_failed=None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_read_event(stream_id, event_number, on_success, on_failed)
def _start_read_event(self, stream_id, event_number, on_success,
on_failed):
Ensure.is_not_empty_string(stream_id, "stream_id")
Ensure.is_greater_equal_number_then(0, event_number, "event_number")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
stream_id = quote(stream_id)
url = "{0}/streams/{1}/event/{2}".format(self._base_url, stream_id,
str(event_number))
self._tornado_http_sender.send_async(
url, "GET", self._headers, None,
lambda x: self._read_event_callback(x, on_success, on_failed))
def _read_event_callback(self, response, on_success, on_failed):
if response.code != 200:
on_failed(FailedAnswer(response.code, response.error.message))
return
responseContent = response.body
event = json.loads(responseContent)
on_success(self._json_to_event(event))
def read_stream_events_backward(self, stream_id, start_position, count):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.read_stream_events_backward_async(stream_id, start_position,
count, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return result.response
else:
raise result.response
def read_stream_events_backward_async(self,
stream_id,
start_position,
count,
on_success=None,
on_failed=None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_read_stream_events_backward(stream_id, start_position,
count, on_success, on_failed)
def _start_read_stream_events_backward(self, stream_id, start_position,
count, on_success, on_failed):
Ensure.is_not_empty_string(stream_id, "stream_id")
Ensure.is_greater_equal_number_then(-1, start_position,
"start_position")
Ensure.is_greater_equal_number_then(1, count, "count")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
stream_id = quote(stream_id)
events = []
batch_counter = 0
params = ReadEventsData
params.stream_id = stream_id
params.start_position = start_position
params.count = count
params.batch_counter = batch_counter
params.events = events
self._read_batch_events_backward(params, on_success, on_failed)
def _read_batch_events_backward(self,
params,
on_success,
on_failed,
events_count=None):
if events_count is not None and events_count < self._read_batch_size:
on_success(params.events)
return
if params.batch_counter < params.count:
if params.count < params.batch_counter + self._read_batch_size:
params.batch_length = params.count - params.batch_counter
else:
params.batch_length = self._read_batch_size
url = self._base_url + "/streams/{0}/range/{1}/{2}?format=json".format(
params.stream_id,
str(params.start_position - params.batch_counter),
str(params.batch_length))
params.batch_counter += self._read_batch_size
self._tornado_http_sender.send_async(
url, "GET", self._headers, None,
lambda x: self._read_stream_events_backward_async_callback(
x, params, on_success, on_failed))
else:
on_success(params.events)
def _read_stream_events_backward_async_callback(self, response, params,
on_success, on_failed):
if response.code != 200:
on_failed(
FailedAnswer(
response.code, "Error occur while reading batch: " +
response.error.message))
return
try:
response = json.loads(response.body)
if len(response["entries"]) == 0:
if len(params.events) != 0:
on_success(params.events)
return
url = "{0}/streams/{1}".format(self._base_url,
params.stream_id)
self._tornado_http_sender.send_async(
url, "GET", self._headers, None, lambda x: self.
_on_read_events_first_response_entries_empty(
x, params, on_success, on_failed))
return
batch_events = []
for uri in response["entries"]:
url = uri["links"]
for ur in url:
if "type" not in ur:
continue
if ur["type"] == "application/json":
url = ur["uri"]
self._tornado_http_sender.send_async(
url, "GET", self._headers, None,
lambda x: self._event_read_callback(
x, params, batch_events, on_success, on_failed,
len(response["entries"])))
break
except:
on_failed(FailedAnswer(response, response))
return
def _on_read_events_first_response_entries_empty(self, response, params,
on_success, on_failed):
if response.code != 200:
on_failed(
FailedAnswer(
response.code, "Error occur while reading first page: " +
response.error.message))
return
response = json.loads(response.body)
if len(response["entries"]) == 0:
on_success(params.events)
return
last_event_number = int(response["entries"][0]["id"].split("/")[-1])
if params.count < params.start_position - last_event_number:
on_success(params.events)
return
params.count = params.count + last_event_number - params.start_position
params.start_position = last_event_number
params.batch_counter = 0
self._read_batch_events_backward(params, on_success, on_failed)
def _event_read_callback(self, response, params, batch_events, on_success,
on_failed, events_count):
if response.code != 200:
on_failed(FailedAnswer(response.code, response.error.message))
return
try:
event = json.loads(response.body)
batch_events.append(self._json_to_event(event))
if len(batch_events) == events_count:
batch_events = sorted(batch_events,
key=lambda ev: ev.event_number,
reverse=True)
params.events += batch_events
self._read_batch_events_backward(params, on_success, on_failed,
events_count)
except:
on_failed(
FailedAnswer(
response.code,
"Error occure while reading event: " + response.body))
return
def read_stream_events_forward(self, stream_id, start_position, count):
queue = deque()
on_success = lambda x: queue.append(
self._sync_success(list(reversed(x))))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.read_stream_events_forward_async(stream_id, start_position, count,
on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return result.response
else:
raise result.response
def read_stream_events_forward_async(self,
stream_id,
start_position,
count,
on_success=None,
on_failed=None):
new_start_position = start_position + count - 1
if new_start_position > sys.maxint:
count -= (new_start_position - sys.maxint)
new_start_position = sys.maxint
self._start_read_stream_events_backward(stream_id,
int(new_start_position),
int(count), on_success,
on_failed)
def read_all_events_backward(self, prepare_position, commit_position,
count):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self._start_read_all_events_backward(prepare_position, commit_position,
count, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return result.response
else:
raise result.response
def read_all_events_backward_async(self,
prepare_position,
commit_position,
count,
on_success=None,
on_failed=None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_read_all_events_backward(prepare_position, commit_position,
count, on_success, on_failed)
def _start_read_all_events_backward(self, prepare_position,
commit_position, count, on_success,
on_failed):
Ensure.is_number(prepare_position, "prepare_position")
Ensure.is_number(commit_position, "commit_position")
Ensure.is_greater_equal_number_then(1, count, "count")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
events = []
batch_counter = 0
params = ReadEventsData
params.prepare_position = prepare_position
params.commit_position = commit_position
params.count = count
params.batch_counter = batch_counter
params.events = events
self._read_batch_all_events_backward(params, on_success, on_failed)
def _read_batch_all_events_backward(self,
params,
on_success,
on_failed,
events_count=None):
if events_count is not None and events_count < self._read_batch_size:
on_success(
AllEventsAnswer(params.events, params.prepare_position,
params.commit_position))
return
if params.batch_counter < params.count:
hexStartPosition = self._convert_to_hex(
params.prepare_position) + self._convert_to_hex(
params.commit_position)
if params.count < params.batch_counter + self._read_batch_size:
params.batch_length = params.count - params.batch_counter
else:
params.batch_length = self._read_batch_size
url = "{0}/streams/$all/before/{1}/{2}".format(
self._base_url, hexStartPosition, str(params.batch_length))
params.batch_counter += self._read_batch_size
self._tornado_http_sender.send_async(
url, "GET", self._headers, None,
lambda x: self._read_all_events_backward_page_callback(
x, params, on_success, on_failed))
else:
on_success(
AllEventsAnswer(params.events, params.prepare_position,
params.commit_position))
def _read_all_events_backward_page_callback(self, response, params,
on_success, on_failed):
if response.code != 200:
on_failed(
FailedAnswer(
response.code, "Error occur while reading links: " +
response.error.message))
read_line = response.body
body = json.loads(read_line)
params.prepare_position = self._get_prepare_position(
body["links"][4]["uri"])
params.commit_position = self._get_commit_position(
body["links"][4]["uri"])
try:
if body["entries"] == []:
if len(params.events) != 0:
on_success(AllEventsAnswer("", 0, 0))
else:
self._start_read_all_events_backward(
-1, -1, params.count, on_success, on_failed)
return
params.events_count = len(body["entries"])
params.events_count_in_current_butch = len(body["entries"])
params.batch_events = {}
params.url_number_dictionary = {}
event_number = 0
for uri in body["entries"]:
url = uri["links"]
for ur in url:
try:
if ur["type"] == "application/json":
url = ur["uri"]
params.url_number_dictionary[url] = event_number
event_number += 1
self._tornado_http_sender.send_async(
url, "GET", self._headers, None, lambda x: self
._read_all_events_backward_callback(
x, params, on_success, on_failed))
break
except:
continue
except:
on_failed(FailedAnswer(response.code, response.error.message))
return
def _read_all_events_backward_callback(self, response, params, on_success,
on_failed):
is_failed = True
if response.code == 410:
is_failed = False
params.events_count_in_current_butch -= 1
elif response.code == 200:
is_failed = False
try:
event = json.loads(response.body)
params.batch_events[params.url_number_dictionary[
response.request.url]] = self._json_to_event(event)
except:
on_failed(
FailedAnswer(
response.code, "Error occure while reading event: " +
response.error.message))
return
if len(params.batch_events) == params.events_count_in_current_butch:
i = 0
while i < params.events_count:
try:
params.events.append(params.batch_events[i])
except:
self._do_nothing(i)
i += 1
self._read_batch_all_events_backward(params, on_success, on_failed,
params.events_count)
return
if is_failed:
on_failed(
FailedAnswer(
response.code, "Error occure while reading event: " +
response.error.message))
def read_all_events_forward(self, prepare_position, commit_position,
count):
queue = deque()
on_success = lambda x: queue.append(self._sync_success(x))
on_failed = lambda x: queue.append(self._sync_failed(x))
self.read_all_events_forward_async(prepare_position, commit_position,
count, on_success, on_failed)
self.wait()
result = queue.popleft()
if result.success:
return result.response
else:
raise result.response
def read_all_events_forward_async(self,
prepare_position,
commit_position,
count,
on_success=None,
on_failed=None):
if on_success is None: on_success = lambda x: self._do_nothing(x)
if on_failed is None: on_failed = lambda x: self._do_nothing(x)
self._start_read_all_events_forward(prepare_position, commit_position,
count, on_success, on_failed)
def _start_read_all_events_forward(self, prepare_position, commit_position,
count, on_success, on_failed):
Ensure.is_greater_equal_number_then(-1, prepare_position,
"prepare_position")
Ensure.is_greater_equal_number_then(-1, commit_position,
"commit_position")
Ensure.is_greater_equal_number_then(1, count, "count")
Ensure.is_function(on_success, "on_success")
Ensure.is_function(on_failed, "on_failed")
events = []
batch_counter = 0
params = ReadEventsData
params.prepare_position = prepare_position
params.commit_position = commit_position
params.count = count
params.batch_counter = batch_counter
params.events = events
self._read_batch_all_events_forward(params, on_success, on_failed)
def _read_batch_all_events_forward(self,
params,
on_success,
on_failed,
events_count=None):
if events_count is not None and events_count < self._read_batch_size:
on_success(
AllEventsAnswer(params.events, params.prepare_position,
params.commit_position))
return
if params.batch_counter < params.count:
hexStartPosition = self._convert_to_hex(
params.prepare_position) + self._convert_to_hex(
params.commit_position)
if params.batch_counter + self._read_batch_size > params.count:
params.batch_length = params.count - params.batch_counter
else:
params.batch_length = self._read_batch_size
url = "{0}/streams/$all/after/{1}/{2}".format(
self._base_url, hexStartPosition, str(params.batch_length))
params.batch_counter += self._read_batch_size
self._tornado_http_sender.send_async(url, "GET", self._headers, None, lambda x: \
self._read_all_events_forward_page_callback(x, params, on_success, on_failed))
else:
on_success(
AllEventsAnswer(params.events, params.prepare_position,
params.commit_position))
def _read_all_events_forward_page_callback(self, response, params,
on_success, on_failed):
if response.code != 200:
on_failed(
FailedAnswer(
response.code, "Error occur while reading links: " +
response.error.message))
read_line = response.body
body = json.loads(read_line)
params.prepare_position = self._get_prepare_position(
body["links"][3]["uri"])
params.commit_position = self._get_commit_position(
body["links"][3]["uri"])
try:
if not body["entries"]:
on_success(
AllEventsAnswer(params.events, params.prepare_position,
params.commit_position))
return
params.events_count = len(body["entries"])
params.events_count_in_current_butch = len(body["entries"])
params.batch_events = {}
params.url_number_dictionary = {}
event_number = 0
for uri in body["entries"]:
url = uri["links"]
for ur in url:
try:
if ur["type"] == "application/json":
url = ur["uri"]
params.url_number_dictionary[url] = event_number
event_number += 1
self._tornado_http_sender.send_async(
url, "GET", self._headers, None, lambda x: self
._read_all_events_forward_callback(
x, params, on_success, on_failed))
break
except:
continue
except:
on_failed(FailedAnswer(response.code, response.error.message))
return
def _read_all_events_forward_callback(self, response, params, on_success,
on_failed):
is_failed = True
if response.code == 410:
is_failed = False
params.events_count_in_current_butch -= 1
elif response.code == 200:
is_failed = False
try:
event = json.loads(response.body)
params.batch_events[params.url_number_dictionary[
response.request.url]] = self._json_to_event(event)
except:
on_failed(
FailedAnswer(
response.code, "Error occure while reading event: " +
response.error.message))
return
if len(params.batch_events) == params.events_count_in_current_butch:
i = params.events_count - 1
while i >= 0:
try:
params.events.append(params.batch_events[i])
except:
self._do_nothing(i)
i -= 1
self._read_batch_all_events_forward(params, on_success, on_failed,
params.events_count)
return
if is_failed:
on_failed(
FailedAnswer(
response.code, "Error occure while reading event: " +
response.error.message))
def _get_prepare_position(self, link):
position = link.split("/")[6]
result = int(position[0:16], 16)
return result
def _get_commit_position(self, link):
position = link.split("/")[6]
result = int(position[16:32], 16)
return result
def _convert_to_hex(self, number):
if number < 0:
return "ffffffffffffffff"
hexVal = hex(number)
number = str(hexVal).split("x")[1]
while len(number) != 16:
number = "0" + number
return number
def subscribe(self, stream_id, callback, start_from_begining=False):
start_position = 0 if start_from_begining else self._get_stream_event_position(
stream_id) + 1
self._subscribers.append(
SubscribeInfo(stream_id, start_position, callback))
def _handle_subscribers(self):
while True:
if len(self._subscribers) > 0:
processed_streams = 0
for i in range(len(self._subscribers)):
self.read_stream_events_forward_async(
self._subscribers[i].stream_id,
self._subscribers[i].last_position, sys.maxint,
lambda s: self._handle_subscribers_success(
s, len(self._subscribers), processed_streams, i),
lambda s: self._handle_subscribers_failed(
len(self._subscribers), processed_streams))
self.wait()
time.sleep(1)
def _handle_subscribers_success(self, response, stream_count,
processed_streams, subscriber_index):
processed_streams += 1
self._subscribers[subscriber_index].last_position += len(response)
for i in reversed(response):
self._subscribers[subscriber_index].callback(i)
if processed_streams == stream_count:
self.resume()
def _handle_subscribers_failed(self, stream_count, processed_streams):
processed_streams += 1
if processed_streams == stream_count:
self.resume()
def unsubscribe(self, stream_id):
for i in self._subscribers:
if i.stream_id == stream_id:
self._subscribers.remove(i)
def subscribe_all(self, callback, start_from_begining=False):
start_position = 0 if start_from_begining else -1
self.__subscribersAll = SubscribeAllInfo(start_position,
start_position, callback)
self._should_subscribe_all = True
def _handle_subscribers_all(self):
while True:
if self._should_subscribe_all:
if self.__subscribersAll.commit_position == -1:
answer = self.read_all_events_backward(
self.__subscribersAll.prepare_position,
self.__subscribersAll.commit_position, 1)
else:
try:
answer = self.read_all_events_forward(
self.__subscribersAll.prepare_position,
self.__subscribersAll.commit_position,
sys.maxint - 1)
for i in answer.events:
self.__subscribersAll.callback(i)
except:
time.sleep(0.1)
self.__subscribersAll.commit_position = int(
answer.commit_position)
self.__subscribersAll.prepare_position = int(
answer.prepare_position)
time.sleep(1)
def unsubscribe_all(self):
self._should_subscribe_all = False
def _get_stream_event_position(self, stream_id):
try:
events = self.read_stream_events_backward(stream_id, -1, 1)
return int(events[0].event_number)
except:
return 0
def _do_nothing(self, x):
a = 0
def _json_to_event(self, json):
event = ReadEvent(
json["data"],
json["metadata"],
json["eventType"],
json["eventNumber"],
)
return event
