def _process_msg_inject(self, addr, msg): """ Processes incoming message for clients involved in an injection session :param addr: The address of the sender :param msg: The message dictionary """ # We process status messages for connections that are in the queue is_status, status = MessageClient.is_status_message(msg) if is_status and status == MessageClient.CONNECTION_LOST_MSG: # If connection has been lost with an host, we remove its pendingTasks entry if not self._suppressOutput: self._writers[addr].write_entry( MessageBuilder.status_connection(time())) elif is_status and status == MessageClient.CONNECTION_RESTORED_MSG: # If connection has been restored with an host, we send a new session start command self._client.send_msg( addr, MessageBuilder.command_session(self._session_id)) self._client.send_msg( addr, MessageBuilder.command_set_time(self._get_timestamp(time()))) elif is_status and status == MessageClient.CONNECTION_FINALIZED_MSG: self._pendingTasks.pop(addr, None) # If all connections to servers were finalized we assume that the injection can be terminated if len(self._pendingTasks) == 0: self._endReached = True self._reader.close() else: msg_type = msg[MessageBuilder.FIELD_TYPE] if msg_type != MessageBuilder.ACK_YES and msg_type != MessageBuilder.ACK_NO: # Ack messages are not written to the output log if not self._suppressOutput: self._writers[addr].write_entry(msg) # We log on the terminal the content of the message in a pretty form if msg_type == MessageBuilder.STATUS_START: InjectorController.logger.info( "Task %s started on host %s" % (msg[MessageBuilder.FIELD_DATA], formatipport(addr))) elif msg_type == MessageBuilder.STATUS_RESTART: InjectorController.logger.info( "Task %s restarted on host %s" % (msg[MessageBuilder.FIELD_DATA], formatipport(addr))) elif msg_type == MessageBuilder.STATUS_END: InjectorController.logger.info( "Task %s terminated successfully on host %s" % (msg[MessageBuilder.FIELD_DATA], formatipport(addr))) # If a task terminates, we remove its sequence number from the set of pending tasks for the host self._pendingTasks[addr].discard( msg[MessageBuilder.FIELD_SEQNUM]) if not self._suppressOutput: self._write_task_output(addr, msg) elif msg_type == MessageBuilder.STATUS_ERR: InjectorController.logger.error( "Task %s terminated with error code %s on host %s" % (msg[MessageBuilder.FIELD_DATA], str(msg[MessageBuilder.FIELD_ERR]), formatipport(addr))) self._pendingTasks[addr].discard( msg[MessageBuilder.FIELD_SEQNUM]) if not self._suppressOutput: self._write_task_output(addr, msg) elif msg_type == MessageBuilder.ACK_YES: # ACK messages after the initialization phase are received ONLY when a connection is restored, # and the session must be resumed InjectorController.logger.warning( "Session resumed with engine %s" % formatipport(addr)) # If the ack msg contains an error, it means all previously running tasks have been lost if not self._suppressOutput: self._writers[addr].write_entry( MessageBuilder.status_connection(time(), restored=True)) if MessageBuilder.FIELD_ERR in msg: self._pendingTasks[addr] = set() if not self._suppressOutput: self._writers[addr].write_entry( MessageBuilder.status_reset( msg[MessageBuilder.FIELD_TIME])) elif msg_type == MessageBuilder.ACK_NO: InjectorController.logger.warning( "Session cannot be resumed with engine %s" % formatipport(addr)) self._client.remove_host(addr)
def _inject(self, reader, max_tasks=None): """ Starts the injection process with a given workload, issuing commands to start tasks on remote hosts and collecting their result :param reader: a valid Reader object :param max_tasks: The maximum number of tasks to be processed before terminating. Useful for debugging """ self._reader = reader assert isinstance( reader, Reader), '_inject method only supports Reader objects!' task = reader.read_entry() if task is None: InjectorController.logger.warning( "Input workload appears to be empty. Aborting...") return self._client.start() # Initializing the injection session session_accepted, session_id = self._init_session( workload_name=splitext(basename(reader.get_path()))[0]) if session_accepted == 0: InjectorController.logger.warning( "No valid hosts for injection detected. Aborting...") return self._session_id = session_id # Determines if we have reached the end of the workload self._endReached = False read_tasks = 0 # Start timestamp for the workload, computed from its first entry, minus the specified padding value self._start_timestamp = task.timestamp - self._workloadPadding # Synchronizes the time with all of the connected hosts self._client.broadcast_msg( MessageBuilder.command_set_time(self._start_timestamp)) # Absolute timestamp associated to the workload's starting timestamp self._start_timestamp_abs = time() # Timestamp of the last correction that was applied to the clock of remote hosts last_clock_correction = self._start_timestamp_abs while not self._endReached or self._tasks_are_pending(): # While some tasks are still running, and there are tasks from the workload that still need to be read, we # keep looping while self._client.peek_msg_queue() > 0: # We process all messages in the input queue, and write their content to the execution log for the # given host addr, msg = self._client.pop_msg_queue() self._process_msg_inject(addr, msg) # We compute the new "virtual" timestamp, in function of the workload's starting time now_timestamp_abs = time() now_timestamp = self._get_timestamp(now_timestamp_abs) # We perform periodically a correction of the clock of the remote hosts. This has impact only when there # is a very large drift between the clocks, of several minutes # If the sliding window for the task injection is disabled there is no need to perform clock correction if now_timestamp_abs - last_clock_correction > self._clockCorrectionPeriod and self._preSendInterval >= 0: msg = MessageBuilder.command_correct_time(now_timestamp) self._client.broadcast_msg(msg) last_clock_correction = now_timestamp_abs while not self._endReached and ( task.timestamp < now_timestamp + self._preSendInterval or self._preSendInterval < 0): # We read all entries from the workload that correspond to tasks scheduled to start in the next # minutes (specified by presendinterval), and issue the related commands. This supposes that the # workload entries are ordered by their timestamp msg = MessageBuilder.command_start(task) self._client.broadcast_msg(msg) for s in self._pendingTasks.values(): s.add(task.seqNum) task = reader.read_entry() read_tasks += 1 if task is None or (max_tasks is not None and read_tasks >= max_tasks): self._endReached = True reader.close() # This is a busy loop, with a short sleep period of roughly one second sleep(self._sleepPeriod) self._end_session()