def _process_msg_inject(self, addr, msg):
"""
Processes incoming message for clients involved in an injection session
:param addr: The address of the sender
:param msg: The message dictionary
"""
# We process status messages for connections that are in the queue
is_status, status = MessageClient.is_status_message(msg)
if is_status and status == MessageClient.CONNECTION_LOST_MSG:
# If connection has been lost with an host, we remove its pendingTasks entry
if not self._suppressOutput:
self._writers[addr].write_entry(
MessageBuilder.status_connection(time()))
elif is_status and status == MessageClient.CONNECTION_RESTORED_MSG:
# If connection has been restored with an host, we send a new session start command
self._client.send_msg(
addr, MessageBuilder.command_session(self._session_id))
self._client.send_msg(
addr,
MessageBuilder.command_set_time(self._get_timestamp(time())))
elif is_status and status == MessageClient.CONNECTION_FINALIZED_MSG:
self._pendingTasks.pop(addr, None)
# If all connections to servers were finalized we assume that the injection can be terminated
if len(self._pendingTasks) == 0:
self._endReached = True
self._reader.close()
else:
msg_type = msg[MessageBuilder.FIELD_TYPE]
if msg_type != MessageBuilder.ACK_YES and msg_type != MessageBuilder.ACK_NO:
# Ack messages are not written to the output log
if not self._suppressOutput:
self._writers[addr].write_entry(msg)
# We log on the terminal the content of the message in a pretty form
if msg_type == MessageBuilder.STATUS_START:
InjectorController.logger.info(
"Task %s started on host %s" %
(msg[MessageBuilder.FIELD_DATA], formatipport(addr)))
elif msg_type == MessageBuilder.STATUS_RESTART:
InjectorController.logger.info(
"Task %s restarted on host %s" %
(msg[MessageBuilder.FIELD_DATA], formatipport(addr)))
elif msg_type == MessageBuilder.STATUS_END:
InjectorController.logger.info(
"Task %s terminated successfully on host %s" %
(msg[MessageBuilder.FIELD_DATA], formatipport(addr)))
# If a task terminates, we remove its sequence number from the set of pending tasks for the host
self._pendingTasks[addr].discard(
msg[MessageBuilder.FIELD_SEQNUM])
if not self._suppressOutput:
self._write_task_output(addr, msg)
elif msg_type == MessageBuilder.STATUS_ERR:
InjectorController.logger.error(
"Task %s terminated with error code %s on host %s" %
(msg[MessageBuilder.FIELD_DATA],
str(msg[MessageBuilder.FIELD_ERR]), formatipport(addr)))
self._pendingTasks[addr].discard(
msg[MessageBuilder.FIELD_SEQNUM])
if not self._suppressOutput:
self._write_task_output(addr, msg)
elif msg_type == MessageBuilder.ACK_YES:
# ACK messages after the initialization phase are received ONLY when a connection is restored,
# and the session must be resumed
InjectorController.logger.warning(
"Session resumed with engine %s" % formatipport(addr))
# If the ack msg contains an error, it means all previously running tasks have been lost
if not self._suppressOutput:
self._writers[addr].write_entry(
MessageBuilder.status_connection(time(),
restored=True))
if MessageBuilder.FIELD_ERR in msg:
self._pendingTasks[addr] = set()
if not self._suppressOutput:
self._writers[addr].write_entry(
MessageBuilder.status_reset(
msg[MessageBuilder.FIELD_TIME]))
elif msg_type == MessageBuilder.ACK_NO:
InjectorController.logger.warning(
"Session cannot be resumed with engine %s" %
formatipport(addr))
self._client.remove_host(addr)
def _inject(self, reader, max_tasks=None):
"""
Starts the injection process with a given workload, issuing commands to start tasks on remote hosts and
collecting their result
:param reader: a valid Reader object
:param max_tasks: The maximum number of tasks to be processed before terminating. Useful for debugging
"""
self._reader = reader
assert isinstance(
reader, Reader), '_inject method only supports Reader objects!'
task = reader.read_entry()
if task is None:
InjectorController.logger.warning(
"Input workload appears to be empty. Aborting...")
return
self._client.start()
# Initializing the injection session
session_accepted, session_id = self._init_session(
workload_name=splitext(basename(reader.get_path()))[0])
if session_accepted == 0:
InjectorController.logger.warning(
"No valid hosts for injection detected. Aborting...")
return
self._session_id = session_id
# Determines if we have reached the end of the workload
self._endReached = False
read_tasks = 0
# Start timestamp for the workload, computed from its first entry, minus the specified padding value
self._start_timestamp = task.timestamp - self._workloadPadding
# Synchronizes the time with all of the connected hosts
self._client.broadcast_msg(
MessageBuilder.command_set_time(self._start_timestamp))
# Absolute timestamp associated to the workload's starting timestamp
self._start_timestamp_abs = time()
# Timestamp of the last correction that was applied to the clock of remote hosts
last_clock_correction = self._start_timestamp_abs
while not self._endReached or self._tasks_are_pending():
# While some tasks are still running, and there are tasks from the workload that still need to be read, we
# keep looping
while self._client.peek_msg_queue() > 0:
# We process all messages in the input queue, and write their content to the execution log for the
# given host
addr, msg = self._client.pop_msg_queue()
self._process_msg_inject(addr, msg)
# We compute the new "virtual" timestamp, in function of the workload's starting time
now_timestamp_abs = time()
now_timestamp = self._get_timestamp(now_timestamp_abs)
# We perform periodically a correction of the clock of the remote hosts. This has impact only when there
# is a very large drift between the clocks, of several minutes
# If the sliding window for the task injection is disabled there is no need to perform clock correction
if now_timestamp_abs - last_clock_correction > self._clockCorrectionPeriod and self._preSendInterval >= 0:
msg = MessageBuilder.command_correct_time(now_timestamp)
self._client.broadcast_msg(msg)
last_clock_correction = now_timestamp_abs
while not self._endReached and (
task.timestamp < now_timestamp + self._preSendInterval
or self._preSendInterval < 0):
# We read all entries from the workload that correspond to tasks scheduled to start in the next
# minutes (specified by presendinterval), and issue the related commands. This supposes that the
# workload entries are ordered by their timestamp
msg = MessageBuilder.command_start(task)
self._client.broadcast_msg(msg)
for s in self._pendingTasks.values():
s.add(task.seqNum)
task = reader.read_entry()
read_tasks += 1
if task is None or (max_tasks is not None
and read_tasks >= max_tasks):
self._endReached = True
reader.close()
# This is a busy loop, with a short sleep period of roughly one second
sleep(self._sleepPeriod)
self._end_session()