def set_next_poll_time(self, msecs):
"""!
@brief Specify how long time the Eventloop should wait before polling
the status of this job again.
"""
self.next_poll_time = eva.now_with_timezone() + datetime.timedelta(milliseconds=msecs)
self.logger.debug("Next poll for this job: %s", eva.strftime_iso8601(self.next_poll_time))
def test_sort_queue(self):
"""!
@brief Test that the event queue is sorted according to specs.
* RPC events go first in queue
* Events are sorted by their timestamps in chronological or reverse
chronological order, according to EVA_QUEUE_ORDER being either FIFO
or LIFO
"""
timestamp = eva.now_with_timezone()
future = timestamp + datetime.timedelta(seconds=26)
self.eventloop.event_queue = [
eva.event.ProductstatusLocalEvent(None, 1, timestamp=future),
eva.event.ProductstatusLocalEvent(None, 2, timestamp=timestamp),
eva.event.RPCEvent(None, 3, timestamp=timestamp),
eva.event.ProductstatusLocalEvent(None, 4, timestamp=timestamp),
eva.event.RPCEvent(None, 5, timestamp=future),
eva.event.ProductstatusLocalEvent(None, 6, timestamp=future),
eva.event.ProductstatusLocalEvent(None, 7, timestamp=timestamp),
eva.event.RPCEvent(None, 8, timestamp=timestamp),
]
self.eventloop.sort_queue()
order = [3, 8, 5, 2, 4, 7, 1, 6, ]
self.assertEqual(len(self.eventloop.event_queue), len(order))
for i, n in enumerate(order):
self.assertEqual(self.eventloop.event_queue[i].data, n)
self.eventloop.queue_order = self.eventloop.QUEUE_ORDER_LIFO
self.eventloop.sort_queue()
order = [5, 3, 8, 1, 6, 2, 4, 7, ]
for i, n in enumerate(order):
self.assertEqual(self.eventloop.event_queue[i].data, n)
def poll_listeners(self):
"""!
@brief Poll for new messages from all message listeners.
"""
for listener in self.listeners:
try:
event = listener.get_next_event(self.adapter.resource_matches_input_config)
assert isinstance(event, eva.event.Event)
except eva.exceptions.EventTimeoutException:
continue
except eva.exceptions.InvalidEventException as e:
self.logger.debug('Received invalid event: %s', e)
continue
except self.RECOVERABLE_EXCEPTIONS as e:
self.logger.warning('Exception while receiving event: %s', e)
continue
# Duplicate events in queue should not happen
if event.id() in [x.id() for x in self.event_queue]:
self.logger.warning('Event with id %s is already in the event queue. This is most probably due to a previous Kafka commit error. The message has been discarded.', event.id())
continue
if event.id() in [x.id() for x in self.process_list]:
self.logger.warning('Event with id %s is already in the process list. This is most probably due to a previous Kafka commit error. The message has been discarded.', event.id())
continue
# Accept heartbeats without adding them to queue
if isinstance(event, eva.event.ProductstatusHeartbeatEvent):
listener.acknowledge()
self.set_health_check_timestamp(eva.now_with_timezone())
continue
if self.add_event_to_queue(event):
listener.acknowledge()
def reference_time_threshold(self):
"""!
@brief Return a DateTime object which represent the oldest reference time that will be processed.
"""
if self.reference_time_threshold_delta is not None:
return eva.now_with_timezone() - self.reference_time_threshold_delta
return eva.epoch_with_timezone()
def __init__(self, id, logger):
self.id = id
self.logger = self.create_logger(logger)
self.command = "" # a multi-line string containing the commands to be run
self.exit_code = None # process exit code
self.stdout = [] # multi-line standard output
self.stderr = [] # multi-line standard error
self.set_status(INITIALIZED) # what state the job is in
self.next_poll_time = eva.now_with_timezone()
def calculate_status(self):
if self.skip_heartbeat or self.heartbeat_interval == 0:
self.server.ok = True
return
next_heartbeat = self.heartbeat_timestamp + datetime.timedelta(seconds=self.heartbeat_interval + self.heartbeat_timeout)
if next_heartbeat > eva.now_with_timezone():
self.server.ok = True
else:
self.server.ok = False
def test_sort_queue_adaptive(self):
now = eva.now_with_timezone()
# create six mock objects
mocks = [mock.MagicMock() for x in range(6)]
# set type to 'datainstance' for the first four objects, and set reference time to now
for x in [0, 1, 2, 3]:
mocks[x]._collection._resource_name = 'datainstance'
mocks[x].data.productinstance.reference_time = now
# set future reference time for the last two objects datainstance objects
for x in [2, 3]:
mocks[x].data.productinstance.reference_time = eva.coerce_to_utc(datetime.datetime(2100, 1, 1, 12, 0, 0))
# set event queue contents to two ProductstatusResourceEvents and
# three ProductstatusLocalEvents, data pointing to the mock objects,
# with increasing timestamps
self.eventloop.event_queue = [
eva.event.ProductstatusLocalEvent(str(x), mocks[x], timestamp=now + datetime.timedelta(hours=x + 1))
for x in [0, 1, 4]
]
self.eventloop.event_queue += [
eva.event.ProductstatusResourceEvent(str(x), mocks[x], timestamp=now + datetime.timedelta(hours=x + 1))
for x in [2, 3]
]
# add an rpc event to the event queue
self.eventloop.event_queue += [eva.event.RPCEvent(len(mocks) - 1, mocks[-1], timestamp=now + datetime.timedelta(hours=12))]
# set queue sort order
self.eventloop.queue_order = self.eventloop.QUEUE_ORDER_ADAPTIVE
# sort queue
logging.info('=== PRE-SORT ===')
[logging.info("%d: ts=%s type=%s reference_time=%s", i, x.timestamp(), x.__class__.__name__, x.data.data.productinstance.reference_time) for i, x in enumerate(self.eventloop.event_queue)]
self.eventloop.sort_queue()
logging.info('=== POST-SORT ===')
[logging.info("%d: ts=%s type=%s reference_time=%s", i, x.timestamp(), x.__class__.__name__, x.data.data.productinstance.reference_time) for i, x in enumerate(self.eventloop.event_queue)]
# test expected order: 1 rpc event first, then 2 objects with future
# reftimes, then 2 objects with "now" reference time, then 1
# non-datainstance object
expected_order = [5, 2, 3, 0, 1, 4]
real_order = [int(x.message) for x in self.eventloop.event_queue]
for i, n in enumerate(expected_order):
self.assertEqual(self.eventloop.event_queue[i].data, mocks[n],
msg='Queue was sorted incorrectly. Expected %s but got %s' % (expected_order, real_order))
def poll_time_reached(self):
"""!
@brief Returns True if the Job object can be polled according to
Job.next_poll_time.
"""
return eva.now_with_timezone() >= self.next_poll_time
def test_fill_process_list(self):
self.eventloop.concurrency = 4
self.eventloop.event_queue = [eva.event.ProductstatusLocalEvent(None, {}, timestamp=eva.now_with_timezone()) for x in range(5)]
self.eventloop.fill_process_list()
self.assertEqual(len(self.eventloop.process_list), 4)
self.assertEqual(len(self.eventloop.event_queue), 1)
def expiry_from_hours(self, hours):
"""!
@returns a DateTime object representing an absolute DataInstance expiry
time, calculated from the current time.
"""
return eva.now_with_timezone() + datetime.timedelta(hours=hours)
def test_set_next_poll_time(self):
self.job.set_next_poll_time(1000)
self.assertGreater(self.job.next_poll_time, eva.now_with_timezone())
self.assertFalse(self.job.poll_time_reached())