def test_frequency(self):
ramp_uuid = str(uuid.uuid4())
intersection_uuid = str(uuid.uuid4())
msg = Message(1337, "split this string please", producer_uuid=ramp_uuid)
msg.send_control_message(self.controller_sock, datetime.timedelta(seconds=2), process_name=ramp_uuid, destination_uuid=intersection_uuid, sender='ramp')
self.run_controller_process_method()
self.controller.update()
self.assertEqual(self.controller.process_statistics[ramp_uuid]['frequency'][2], 1)
def next(self):
for msg in self.queue.get_messages(num_messages=10, wait_time_seconds=5, visibility_timeout=10*60):
# Gets max 10 messages, waiting for max 5 seconds to receive them and blocks other from receiving it for 10m
self.messages[msg.id] = msg
if self.json_group_key:
body = msg.get_body()
yield Message(msg.id, body, grouping_value=body[self.json_group_key])
else:
yield Message(msg.id, msg.get_body())
def _process(self, receive_sock, controller_sock=None):
try:
if getattr(self.process, 'batch_process', None):
poller = zmq.Poller()
poller.register(receive_sock, zmq.POLLIN)
value = []
end_time = _time() + self.process.wait
while end_time > _time() and len(value) < self.process.limit:
socks = dict(poller.poll(timeout=1000))
if socks.get(receive_sock) == zmq.POLLIN:
value.append(receive_sock.recv_json())
message_count = len(value)
else:
poller = zmq.Poller()
poller.register(receive_sock, zmq.POLLIN)
socks = dict(poller.poll(timeout=1000))
if socks.get(receive_sock) == zmq.POLLIN:
value = receive_sock.recv_json()
else:
return
message_count = 1
if value:
self.messages_processed += message_count
if isinstance(value, list):
message = [
Message.from_message(m,
controller_sock,
process_name=self.process_uuid)
for m in value
]
else:
message = Message.from_message(
value, controller_sock, process_name=self.process_uuid)
try:
self.message_batch_start = datetime.datetime.now()
for generated_message in self.process(message):
if generated_message is not None and self.send_socks:
self.send_message(
generated_message,
self.process_uuid,
time_consumed=(datetime.datetime.now() -
self.message_batch_start),
control_message=self.send_control_messages)
self.message_batch_start = datetime.datetime.now()
except Exception as e:
logger.error(unicode(e), exc_info=True)
# Don't send control messages if e.g. web server or other system process
if self.send_control_messages:
if isinstance(message, list):
[m.fail() for m in message]
else:
message.fail()
except Empty: # Didn't receive anything from ZMQ
pass
def next(self):
for msg in self.queue.receive_messages(MaxNumberOfMessages=10,
WaitTimeSeconds=5,
VisibilityTimeout=10 * 60):
# Gets max 10 messages, waiting for max 5 seconds to receive them and blocks other from receiving it for 10m
self.messages[
msg.
message_id] = msg.receipt_handle # we need the receipt_handle to delete the message and we always want to store the latest one, according to the sqs docs
body = json.loads(msg.body)
if self.json_group_key:
yield Message(msg.message_id,
body,
grouping_value=body[self.json_group_key])
else:
yield Message(msg.message_id, body)
def next(self):
for account in self.recurly.Account.all():
yield Message(account.account_code, {
'company_name': account.company_name,
'account_code': account.account_code,
})
time.sleep(3600)
def process(self, messages):
for message in messages:
# time.sleep(1)
self._count[message.content] += 1
self.ack(message)
# time.sleep(0.5)
yield Message(str(uuid.uuid4()), self._count)
def next(self):
msg = self.insertion_queue.get()
try:
yield Message(msg['SequenceNumber'],
json.loads(msg['Data']),
grouping_value=msg['PartitionKey'])
except ValueError as e:
logger.exception(e)
def test_message_timeout(self):
ramp_uuid = str(uuid.uuid4())
intersection_uuid = str(uuid.uuid4())
with freeze_time("2014-01-10 12:00:01"):
msg_in_ramp = Message(1337, "split this string please", producer_uuid=ramp_uuid)
msg_in_ramp.send_control_message(self.controller_sock, datetime.timedelta(seconds=2), process_name=ramp_uuid, destination_uuid=intersection_uuid)
self.run_controller_process_method()
self.controller.update()
self.assertEqual(self.controller.process_statistics[ramp_uuid]['waiting'], 0)
self.assertEqual(self.controller.process_statistics[intersection_uuid]['waiting'], 1)
with freeze_time("2014-01-10 12:59:01"):
self.controller.update()
self.assertEqual(self.controller.process_statistics[ramp_uuid]['waiting'], 0)
self.assertEqual(self.controller.process_statistics[intersection_uuid]['waiting'], 0)
self.run_controller_process_method()
self.assertEqual(self.controller.process_statistics[ramp_uuid]['histogram'][59]['timeout_count'], 0)
self.assertEqual(self.controller.process_statistics[intersection_uuid]['histogram'][59]['timeout_count'], 1)
def _process(self, receive_sock, controller_sock=None):
try:
if getattr(self.process, 'batch_process', None):
poller = zmq.Poller()
poller.register(receive_sock, zmq.POLLIN)
value = []
end_time = _time() + self.process.wait
while end_time > _time() and len(value) < self.process.limit:
socks = dict(poller.poll(timeout=1000))
if socks.get(receive_sock) == zmq.POLLIN:
value.append(receive_sock.recv_json())
message_count = len(value)
else:
poller = zmq.Poller()
poller.register(receive_sock, zmq.POLLIN)
socks = dict(poller.poll(timeout=1000))
if socks.get(receive_sock) == zmq.POLLIN:
value = receive_sock.recv_json()
else:
return
message_count = 1
if value:
self.messages_processed += message_count
if isinstance(value, list):
message = [Message.from_message(m, controller_sock, process_name=self.process_uuid) for m in value]
else:
message = Message.from_message(value, controller_sock, process_name=self.process_uuid)
try:
self.message_batch_start = datetime.datetime.now()
for generated_message in self.process(message):
if generated_message is not None and self.send_socks:
self.send_message(generated_message, self.process_uuid, time_consumed=(datetime.datetime.now() - self.message_batch_start), control_message=self.send_control_messages)
self.message_batch_start = datetime.datetime.now()
except Exception as e:
logger.error(str(e), exc_info=True)
# Don't send control messages if e.g. web server or other system process
if self.send_control_messages:
if isinstance(message, list):
[m.fail() for m in message]
else:
message.fail()
except Empty: # Didn't receive anything from ZMQ
pass
def update(self):
now = datetime.datetime.now()
process_uuid_to_address = {
uuid: address
for address, uuid in self.process_address_to_uuid.items()
}
# Check message status
waiting_messages = {}
for unique_id, lst in self.messages.items():
original_process, ack_value, start_time, process = lst
if unique_id in self.failed_messages:
del self.messages[
unique_id] # This failed somewhere else in the chain and it was notificed already
elif process_uuid_to_address and process not in process_uuid_to_address: # check if dict is empty before checking if the process is in it (mainly for tests)
del self.messages[unique_id] # clean up
self.process_statistics[process]['histogram'][
datetime.datetime.now().minute]['timeout_count'] += 1
self.fail(unique_id,
original_process,
error_message="Assigned processed disappeared")
elif (now - start_time) > datetime.timedelta(
minutes=self.MESSAGE_TIMEOUT):
del self.messages[unique_id] # clean up
self.process_statistics[process]['histogram'][
datetime.datetime.now().minute]['timeout_count'] += 1
self.fail(unique_id,
original_process,
error_message="Message timed out")
elif ack_value > 0:
waiting_messages[process] = waiting_messages.get(process,
0) + 1
# Update histograms
for process in self.process_statistics.keys():
self.process_statistics[process]['histogram'][(
now + datetime.timedelta(minutes=1)
).minute] = self.get_default_process_dict()['histogram'][
0] # reset next minute
self.process_statistics[process]['waiting'] = waiting_messages.get(
process, 0)
# Prepare message but copy the dictionaries so we avoid them being changed while we send it over ZMQ
message = Message("_controller-%s" % uuid.uuid4(), {
'process_id_to_name': self.process_id_to_name.copy(),
'process_statistics': self.process_statistics.copy(),
'stream_consumers': self.stream_consumers.copy(),
'failed_messages': self.failed_messages.copy(),
},
grouping_value=str(self.process_uuid))
if self.send_socks: # if we have at least one destination, send the updates
self.send_message(message,
self.process_uuid,
control_message=False)
def next(self):
# yield Message(uuid.uuid4().int, self.sentences[random.randint(0, len(self.sentences) -1)])
if self.progress <= self.limit:
self.progress += 1
# time.sleep(10)
sentence = self.sentences[random.randint(0,
len(self.sentences) - 1)]
yield Message(uuid.uuid4().int, sentence, grouping_value=sentence)
else:
time.sleep(1)
def next(self):
updates = self.session.post("https://%s/cometd/37.0" %
self.sf.sf_instance,
headers=self.sf.headers,
data=json.dumps({
"channel":
"/meta/connect",
"connectionType":
"long-polling",
"clientId":
self.connection['clientId']
}))
for update in updates.json():
if 'data' in update:
if 'sobject' in update['data']:
yield Message(update['data']['sobject']['Id'],
update['data']['sobject'])
def next(self):
for invoice in self.recurly.Invoice.all():
yield Message(invoice.uuid, {
'uuid': invoice.uuid,
'invoice_number': invoice.invoice_number,
'vat_number': invoice.vat_number,
'total_in_cents': invoice.total_in_cents,
'tax_in_cents': invoice.tax_in_cents,
'subtotal_in_cents': invoice.subtotal_in_cents,
'state': invoice.state,
'collection_method': invoice.collection_method,
'currency': invoice.currency,
'account': invoice.account().account_code, # TODO: Parse this from the href in the XML
'created_at': invoice.created_at,
'updated_at': invoice.updated_at,
'closed_at': invoice.closed_at,
})
time.sleep(3600)
def test_waiting(self):
ramp_uuid = str(uuid.uuid4())
intersection_uuid = str(uuid.uuid4())
msg_in_ramp = Message(1337, "split this string please", producer_uuid=ramp_uuid)
msg_in_ramp.send_control_message(self.controller_sock, datetime.timedelta(seconds=2), process_name=ramp_uuid, destination_uuid=intersection_uuid)
self.run_controller_process_method()
self.controller.update()
self.assertEqual(self.controller.process_statistics[ramp_uuid]['waiting'], 0)
self.assertEqual(self.controller.process_statistics[intersection_uuid]['waiting'], 1)
msg_in_intersection = Message.from_message(msg_in_ramp._message(), self.controller_sock, process_name=intersection_uuid)
msg_in_intersection.ack()
self.run_controller_process_method()
self.controller.update()
self.assertEqual(self.controller.process_statistics[ramp_uuid]['waiting'], 0)
self.assertEqual(self.controller.process_statistics[intersection_uuid]['waiting'], 0)
def test_histogram_error(self):
ramp_uuid = str(uuid.uuid4())
intersection_uuid = str(uuid.uuid4())
with freeze_time("2014-01-10 12:00:01"):
msg_in_ramp = Message(1337, "split this string please", producer_uuid=ramp_uuid)
msg_in_ramp.send_control_message(self.controller_sock, datetime.timedelta(seconds=2), process_name=ramp_uuid, destination_uuid=intersection_uuid)
self.run_controller_process_method()
self.controller.update()
msg_in_intersection = Message.from_message(msg_in_ramp._message(), self.controller_sock)
msg_in_intersection.fail("some error message", False)
self.run_controller_process_method()
self.controller.update()
self.assertEqual(self.controller.process_statistics[ramp_uuid]['histogram'][0]['error_count'], 1)
with freeze_time("2014-01-10 12:59:01"):
self.controller.update()
self.assertEqual(self.controller.process_statistics[ramp_uuid]['histogram'][0]['error_count'], 0)
def next(self):
if not self.completed:
for _id, sentence in enumerate(self.sentences):
yield Message(_id, sentence)
self.completed = True
def process(self, message):
self._count.update(message.content)
yield Message(str(uuid.uuid4()), self._count)
self.ack(message)
def process(self, message):
for word, count in message.content.items():
yield Message.new(message, {'word': word, 'count': count})
self.ack(message)
def test_message_duration(self):
ramp_uuid = str(uuid.uuid4())
intersection_uuid = str(uuid.uuid4())
intersection_two_uuid = str(uuid.uuid4())
msg_in_ramp = Message(1337, "split this string please", producer_uuid=ramp_uuid)
msg_in_ramp.send_control_message(self.controller_sock, datetime.timedelta(seconds=30), process_name=ramp_uuid, destination_uuid=intersection_uuid, sender='ramp')
self.run_controller_process_method()
self.controller.update()
self.assertEqual(self.controller.process_statistics[ramp_uuid]['time_taken'], datetime.timedelta(seconds=30))
self.assertEqual(self.controller.process_statistics[ramp_uuid]['avg_time_taken'], datetime.timedelta(seconds=30))
self.assertEqual(self.controller.process_statistics[ramp_uuid]['total_frequency'], 1)
self.assertEqual(self.controller.process_statistics[intersection_uuid]['time_taken'], datetime.timedelta(seconds=0))
self.assertEqual(self.controller.process_statistics[intersection_uuid]['avg_time_taken'], datetime.timedelta(seconds=0))
self.assertEqual(self.controller.process_statistics[intersection_uuid]['total_frequency'], 0)
msg_in_intersection = Message.from_message(msg_in_ramp._message(), self.controller_sock, process_name=intersection_uuid)
new_msg_in_intersection = Message(1338, "yo", producer_uuid=intersection_uuid)
new_msg_in_intersection.send_control_message(self.controller_sock, datetime.timedelta(seconds=7), process_name=intersection_uuid, destination_uuid=intersection_uuid)
msg_in_intersection.ack(time_consumed=datetime.timedelta(seconds=15))
self.run_controller_process_method()
# self.run_controller_process_method()
self.controller.update()
self.assertEqual(self.controller.process_statistics[ramp_uuid]['time_taken'], datetime.timedelta(seconds=30))
self.assertEqual(self.controller.process_statistics[ramp_uuid]['avg_time_taken'], datetime.timedelta(seconds=30))
self.assertEqual(self.controller.process_statistics[ramp_uuid]['total_frequency'], 1)
# self.assertEqual(self.controller.process_statistics[intersection_uuid]['time_taken'], datetime.timedelta(seconds=15))
# self.assertEqual(self.controller.process_statistics[intersection_uuid]['avg_time_taken'], datetime.timedelta(seconds=15))
self.assertEqual(self.controller.process_statistics[intersection_uuid]['total_frequency'], 1)
msg_in_intersection_two = Message.from_message(msg_in_ramp._message(), self.controller_sock, process_name=intersection_two_uuid)
msg_in_intersection_two.ack()
self.run_controller_process_method()
self.controller.update()
self.assertEqual(self.controller.process_statistics[ramp_uuid]['time_taken'], datetime.timedelta(seconds=30))
self.assertEqual(self.controller.process_statistics[ramp_uuid]['avg_time_taken'], datetime.timedelta(seconds=30))
self.assertEqual(self.controller.process_statistics[ramp_uuid]['total_frequency'], 1)
# self.assertEqual(self.controller.process_statistics[intersection_uuid]['time_taken'], datetime.timedelta(seconds=15))
# self.assertEqual(self.controller.process_statistics[intersection_uuid]['avg_time_taken'], datetime.timedelta(seconds=15))
self.assertEqual(self.controller.process_statistics[intersection_uuid]['total_frequency'], 1)
def process(self, message):
for word in message.content.split(" "):
yield Message.new(message, word, grouping_value=word)
self.ack(message)
def get_control_messages(self):
messages = []
while self.control_messages:
messages.append(Message.from_message(self.control_messages.pop(0), self.controller_sock))
return messages
def connection_thread(self,
context=None,
refresh_connection_stream=None,
input_queue=None,
output_queue=None,
grouper_cls=None,
set_controller_sock=True):
refresh_connection_sock = context.socket(zmq.SUB)
refresh_connection_sock.connect(refresh_connection_stream)
refresh_connection_sock.setsockopt(
zmq.SUBSCRIBE,
'') # You must subscribe to something, so this means *all*}
set_timeouts_on_socket(refresh_connection_sock)
connections = get_connections_block('_update_connections',
refresh_connection_sock)
while not self.receive_port:
time.sleep(1)
# Register as consumer of input stream
update_connection_sock = context.socket(zmq.PUSH)
update_connection_sock.connect(
connections['_update_connections']['streams'][0])
intersection_connection_info = {
'streams': {
input_queue: ['tcp://%s:%s' % (get_ip(), self.receive_port)]
},
'meta': {
'id': self.process_uuid,
'name': self.process_name,
'grouping': None if not grouper_cls else grouper_cls.
__name__ # Specify how messages sent to this intersection should be grouped
}
}
Message(None, intersection_connection_info).send(
update_connection_sock, producer_uuid=self.process_uuid)
last_send_time = datetime.datetime.now()
if set_controller_sock:
connections = get_connections_block(
'_message_ack',
refresh_connection_sock,
existing_connections=connections)
self.controller_sock = context.socket(zmq.PUSH)
self.controller_sock.connect(
connections['_message_ack']['streams'][0])
set_timeouts_on_socket(self.controller_sock)
while True:
try:
connections = refresh_connection_sock.recv_json()
if output_queue in connections:
self.set_send_socks(connections, output_queue, context)
self.process_id_to_name = {
process['process_id']: process['process_name']
for process in itertools.chain.from_iterable([
queue_details['stream_heartbeats'].values()
for queue_details in connections.values()
])
}
self.process_address_to_uuid = {
address: process['process_id']
for address, process in itertools.chain.from_iterable([
queue_details['stream_heartbeats'].items()
for queue_details in connections.values()
])
}
except zmq.Again:
time.sleep(1)
if last_send_time + datetime.timedelta(
seconds=10) < datetime.datetime.now():
Message(None, intersection_connection_info).send(
update_connection_sock, producer_uuid=self.process_uuid)
last_send_time = datetime.datetime.now()
def next(self):
for obj in self.sf.query_all(self.query)['records']:
attributes = obj.pop('attributes')
yield Message(obj['Id'], obj)
time.sleep(self.sleep_time)
def process(self, message):
for word in message.content.split(" "):
yield Message.new(message, word, grouping_value=word)
self.ack(message)
def process(self, message):
for word in message.content.split(" "):
yield Message(str(uuid.uuid4()), word, grouping_value=word)
self.ack(message)