class CTRexServer(object): """This class defines the server side of the RESTfull interaction with TRex""" DEFAULT_TREX_PATH = '/auto/proj-pcube-b/apps/PL-b/tools/bp_sim2/v1.55/' #'/auto/proj-pcube-b/apps/PL-b/tools/nightly/trex_latest' TREX_START_CMD = './t-rex-64' DEFAULT_FILE_PATH = '/tmp/trex_files/' def __init__(self, trex_path, trex_files_path, trex_host='0.0.0.0', trex_daemon_port=8090, trex_zmq_port=4500, trex_nice=-19): """ Parameters ---------- trex_host : str a string of the TRex ip address or hostname. default value: machine hostname as fetched from socket.gethostname() trex_daemon_port : int the port number on which the trex-daemon server can be reached default value: 8090 trex_zmq_port : int the port number on which trex's zmq module will interact with daemon server default value: 4500 nice: int priority of the TRex process Instantiate a TRex client object, and connecting it to listening daemon-server """ self.TREX_PATH = os.path.abspath(os.path.dirname(trex_path + '/')) self.trex_files_path = os.path.abspath( os.path.dirname(trex_files_path + '/')) self.__check_trex_path_validity() self.__check_files_path_validity() self.trex = CTRex() self.trex_version = None self.trex_host = trex_host self.trex_daemon_port = trex_daemon_port self.trex_zmq_port = trex_zmq_port self.trex_server_path = "http://{hostname}:{port}".format( hostname=trex_host, port=trex_daemon_port) self.start_lock = threading.Lock() self.__reservation = None self.zmq_monitor = ZmqMonitorSession( self.trex, self.trex_zmq_port ) # intiate single ZMQ monitor thread for server usage self.trex_nice = int(trex_nice) if self.trex_nice < -20 or self.trex_nice > 19: err = "Parameter 'nice' should be integer in range [-20, 19]" print(err) logger.error(err) raise Exception(err) def add(self, x, y): logger.info( "Processing add function. Parameters are: {0}, {1} ".format(x, y)) return x + y # return Fault(-10, "") def push_file(self, filename, bin_data): logger.info("Processing push_file() command.") try: filepath = os.path.join(self.trex_files_path, os.path.basename(filename)) with open(filepath, 'wb') as f: f.write(binascii.a2b_base64(bin_data)) logger.info("push_file() command finished. File is saved as %s" % filepath) return True except IOError as inst: logger.error("push_file method failed. " + str(inst)) return False def connectivity_check(self): logger.info("Processing connectivity_check function.") return True def start(self): """This method fires up the daemon server based on initialized parameters of the class""" # initialize the server instance with given resources register_socket('trex_daemon_server') try: print("Firing up TRex REST daemon @ port {trex_port} ...\n".format( trex_port=self.trex_daemon_port)) logger.info( "Firing up TRex REST daemon @ port {trex_port} ...".format( trex_port=self.trex_daemon_port)) logger.info("current working dir is: {0}".format(self.TREX_PATH)) logger.info("current files dir is : {0}".format( self.trex_files_path)) logger.debug( "Starting TRex server. Registering methods to process.") logger.info(self.get_trex_version(base64=False)) self.server = SimpleJSONRPCServer( (self.trex_host, self.trex_daemon_port)) except socket.error as e: if e.errno == errno.EADDRINUSE: logger.error( "TRex server requested address already in use. Aborting server launching." ) print( "TRex server requested address already in use. Aborting server launching." ) raise socket.error( errno.EADDRINUSE, "TRex daemon requested address already in use. " "Server launch aborted. Please make sure no other process is " "using the desired server properties.") elif isinstance(e, socket.gaierror) and e.errno == -3: # handling Temporary failure in name resolution exception raise socket.gaierror( -3, "Temporary failure in name resolution.\n" "Make sure provided hostname has DNS resolving.") else: raise # set further functionality and peripherals to server instance self.server.register_function(self.add) self.server.register_function(self.cancel_reservation) self.server.register_function(self.connectivity_check) self.server.register_function(self.connectivity_check, 'check_connectivity') # alias self.server.register_function(self.force_trex_kill) self.server.register_function(self.get_file) self.server.register_function(self.get_files_list) self.server.register_function(self.get_files_path) self.server.register_function(self.get_running_info) self.server.register_function(self.get_running_status) self.server.register_function(self.get_trex_cmds) self.server.register_function(self.get_trex_config) self.server.register_function(self.get_trex_daemon_log) self.server.register_function(self.get_trex_log) self.server.register_function(self.get_trex_version) self.server.register_function(self.is_reserved) self.server.register_function(self.is_running) self.server.register_function(self.kill_all_trexes) self.server.register_function(self.push_file) self.server.register_function(self.reserve_trex) self.server.register_function(self.start_trex) self.server.register_function(self.stop_trex) self.server.register_function(self.wait_until_kickoff_finish) signal.signal(signal.SIGTSTP, self.stop_handler) signal.signal(signal.SIGTERM, self.stop_handler) try: self.zmq_monitor.start() self.server.serve_forever() except KeyboardInterrupt: logger.info("Daemon shutdown request detected.") finally: self.zmq_monitor.join() # close ZMQ monitor thread resources self.server.shutdown() #self.server.server_close() # get files from Trex server and return their content (mainly for logs) @staticmethod def _pull_file(filepath): try: with open(filepath, 'rb') as f: file_content = f.read() return binascii.b2a_base64(file_content).decode( errors='replace') except Exception as e: err_str = "Can't get requested file %s: %s" % (filepath, e) logger.error(err_str) return Fault(-33, err_str) # returns True if given path is under TRex package or under /tmp/trex_files def _check_path_under_TRex_or_temp(self, path): if not os.path.relpath(path, self.trex_files_path).startswith( os.pardir): return True if not os.path.relpath(path, self.TREX_PATH).startswith(os.pardir): return True return False # gets the file content encoded base64 either from /tmp/trex_files or TRex server dir def get_file(self, filepath): try: logger.info("Processing get_file() command.") if not self._check_path_under_TRex_or_temp(filepath): raise Exception( 'Given path should be under current TRex package or /tmp/trex_files' ) return self._pull_file(filepath) except Exception as e: err_str = "Can't get requested file %s: %s" % (filepath, e) logger.error(err_str) return Fault(-33, err_str) # get tuple (dirs, files) with directories and files lists from given path (limited under TRex package or /tmp/trex_files) def get_files_list(self, path): try: logger.info("Processing get_files_list() command, given path: %s" % path) if not self._check_path_under_TRex_or_temp(path): raise Exception( 'Given path should be under current TRex package or /tmp/trex_files' ) return os.walk(path).next()[1:3] except Exception as e: err_str = "Error processing get_files_list(): %s" % e logger.error(err_str) return Fault(-33, err_str) # get Trex log /tmp/trex.txt def get_trex_log(self): logger.info("Processing get_trex_log() command.") return self._pull_file('/tmp/trex.txt') # get /etc/trex_cfg.yaml def get_trex_config(self): logger.info("Processing get_trex_config() command.") return self._pull_file('/etc/trex_cfg.yaml') # get daemon log /var/log/trex/trex_daemon_server.log def get_trex_daemon_log(self): logger.info("Processing get_trex_daemon_log() command.") return self._pull_file('/var/log/trex/trex_daemon_server.log') # get Trex version from ./t-rex-64 --help (last lines starting with "Version : ...") def get_trex_version(self, base64=True): try: logger.info("Processing get_trex_version() command.") if not self.trex_version: ret_code, stdout, stderr = run_command('./t-rex-64 --help', cwd=self.TREX_PATH) search_result = re.search('\n\s*(Version\s*:.+)', stdout, re.DOTALL) if not search_result: raise Exception( 'Could not determine version from ./t-rex-64 --help') self.trex_version = binascii.b2a_base64( search_result.group(1).encode(errors='replace')) if base64: return self.trex_version.decode(errors='replace') else: return binascii.a2b_base64( self.trex_version).decode(errors='replace') except Exception as e: err_str = "Can't get trex version, error: %s" % e logger.error(err_str) return Fault(-33, err_str) def stop_handler(self, *args, **kwargs): logger.info("Daemon STOP request detected.") if self.is_running(): # in case TRex process is currently running, stop it before terminating server process self.stop_trex(self.trex.get_seq()) sys.exit(0) def assert_zmq_ok(self): if self.trex.zmq_error: raise Exception('ZMQ thread got error: %s' % self.trex.zmq_error) if not self.zmq_monitor.is_alive(): if self.trex.get_status() != TRexStatus.Idle: self.force_trex_kill() raise Exception('ZMQ thread is dead.') def is_running(self): run_status = self.trex.get_status() logger.info( "Processing is_running() command. Running status is: {stat}". format(stat=run_status)) if run_status == TRexStatus.Running: return True else: return False def is_reserved(self): logger.info("Processing is_reserved() command.") return bool(self.__reservation) def get_running_status(self): run_status = self.trex.get_status() logger.info( "Processing get_running_status() command. Running status is: {stat}" .format(stat=run_status)) return { 'state': run_status.value, 'verbose': self.trex.get_verbose_status() } def get_files_path(self): logger.info("Processing get_files_path() command.") return self.trex_files_path def reserve_trex(self, user): if user == "": logger.info( "TRex reservation cannot apply to empty string user. Request denied." ) return Fault( -33, "TRex reservation cannot apply to empty string user. Request denied." ) with self.start_lock: logger.info("Processing reserve_trex() command.") if self.is_reserved(): if user == self.__reservation['user']: # return True is the same user is asking and already has the resrvation logger.info( "the same user is asking and already has the resrvation. Re-reserving TRex." ) return True logger.info( "TRex is already reserved to another user ({res_user}), cannot reserve to another user." .format(res_user=self.__reservation['user'])) return Fault( -33, "TRex is already reserved to another user ({res_user}). Please make sure TRex is free before reserving it." .format(res_user=self.__reservation['user'] )) # raise at client TRexInUseError elif self.trex.get_status() != TRexStatus.Idle: logger.info( "TRex is currently running, cannot reserve TRex unless in Idle state." ) return Fault( -13, 'TRex is currently running, cannot reserve TRex unless in Idle state. Please try again when TRex run finished.' ) # raise at client TRexInUseError else: logger.info( "TRex is now reserved for user ({res_user}).".format( res_user=user)) self.__reservation = {'user': user, 'since': time.ctime()} logger.debug("Reservation details: " + str(self.__reservation)) return True def cancel_reservation(self, user): with self.start_lock: logger.info("Processing cancel_reservation() command.") if self.is_reserved(): if self.__reservation['user'] == user: logger.info( "TRex reservation to {res_user} has been canceled successfully." .format(res_user=self.__reservation['user'])) self.__reservation = None return True else: logger.warning( "TRex is reserved to different user than the provided one. Reservation wasn't canceled." ) return Fault( -33, "Cancel reservation request is available to the user that holds the reservation. Request denied" ) # raise at client TRexRequestDenied else: logger.info( "TRex is not reserved to anyone. No need to cancel anything" ) assert (self.__reservation is None) return False def start_trex(self, trex_cmd_options, user, block_to_success=True, timeout=40, stateless=False, debug_image=False, trex_args=''): self.assert_zmq_ok() with self.start_lock: logger.info("Processing start_trex() command.") if self.is_reserved(): # check if this is not the user to which TRex is reserved if self.__reservation['user'] != user: logger.info( "TRex is reserved to another user ({res_user}). Only that user is allowed to initiate new runs." .format(res_user=self.__reservation['user'])) return Fault( -33, "TRex is reserved to another user ({res_user}). Only that user is allowed to initiate new runs." .format(res_user=self.__reservation['user'] )) # raise at client TRexRequestDenied elif self.trex.get_status() != TRexStatus.Idle: logger.info( "TRex is already taken, cannot create another run until done." ) return Fault(-13, '') # raise at client TRexInUseError try: server_cmd_data = self.generate_run_cmd( stateless=stateless, debug_image=debug_image, trex_args=trex_args, **trex_cmd_options) self.zmq_monitor.first_dump = True self.trex.start_trex(self.TREX_PATH, server_cmd_data) logger.info("TRex session has been successfully initiated.") if block_to_success: # delay server response until TRex is at 'Running' state. start_time = time.time() trex_state = None while (time.time() - start_time) < timeout: trex_state = self.trex.get_status() if trex_state != TRexStatus.Starting: break else: time.sleep(0.5) self.assert_zmq_ok() # check for TRex run started normally if trex_state == TRexStatus.Starting: # reached timeout logger.warning( "TimeoutError: TRex initiation outcome could not be obtained, since TRex stays at Starting state beyond defined timeout." ) return Fault( -12, 'TimeoutError: TRex initiation outcome could not be obtained, since TRex stays at Starting state beyond defined timeout.' ) # raise at client TRexWarning elif trex_state == TRexStatus.Idle: return Fault(-11, self.trex.get_verbose_status() ) # raise at client TRexError # reach here only if TRex is at 'Running' state self.trex.gen_seq() return self.trex.get_seq( ) # return unique seq number to client except TypeError as e: logger.error( "TRex command generation failed, probably because either -f (traffic generation .yaml file) and -c (num of cores) was not specified correctly.\nReceived params: {params}" .format(params=trex_cmd_options)) raise TypeError( 'TRex -f (traffic generation .yaml file) and -c (num of cores) must be specified. %s' % e) def stop_trex(self, seq): logger.info("Processing stop_trex() command.") if self.trex.get_seq() == seq: logger.debug("Abort request legit since seq# match") return self.trex.stop_trex() else: if self.trex.get_status() != TRexStatus.Idle: logger.warning( "Abort request is only allowed to process initiated the run. Request denied." ) return Fault( -33, 'Abort request is only allowed to process initiated the run. Request denied.' ) # raise at client TRexRequestDenied else: return False def force_trex_kill(self): logger.info( "Processing force_trex_kill() command. --> Killing TRex session indiscriminately." ) return self.trex.stop_trex() # returns list of tuples (pid, command line) of running TRex(es) def get_trex_cmds(self): logger.info('Processing get_trex_cmds() command.') ret_code, stdout, stderr = run_command( 'ps -u root --format pid,comm,cmd') if ret_code: raise Exception( 'Failed to determine running processes, stderr: %s' % stderr) trex_cmds_list = [] for line in stdout.splitlines(): pid, proc_name, full_cmd = line.strip().split(' ', 2) pid = pid.strip() full_cmd = full_cmd.strip() if proc_name.find('t-rex-64') >= 0: trex_cmds_list.append((pid, full_cmd)) return trex_cmds_list # Silently tries to kill TRexes with given signal. # Responsibility of client to verify with get_trex_cmds. def kill_all_trexes(self, signal_name): logger.info('Processing kill_all_trexes() command.') trex_cmds_list = self.get_trex_cmds() for pid, cmd in trex_cmds_list: logger.info('Killing with signal %s process %s %s' % (signal_name, pid, cmd)) os.kill(int(pid), signal_name) def wait_until_kickoff_finish(self, timeout=40): # block until TRex exits Starting state logger.info("Processing wait_until_kickoff_finish() command.") trex_state = None start_time = time.time() while (time.time() - start_time) < timeout: self.assert_zmq_ok() trex_state = self.trex.get_status() if trex_state != TRexStatus.Starting: return sleep(0.1) return Fault( -12, 'TimeoutError: TRex initiation outcome could not be obtained, since TRex stays at Starting state beyond defined timeout.' ) # raise at client TRexWarning def get_running_info(self): self.assert_zmq_ok() logger.info("Processing get_running_info() command.") return self.trex.get_running_info() def generate_run_cmd(self, iom=0, export_path="/tmp/trex.txt", stateless=False, debug_image=False, trex_args='', **kwargs): """ generate_run_cmd(self, iom, export_path, kwargs) -> str Generates a custom running command for the kick-off of the TRex traffic generator. Returns a tuple of command (string) and export path (string) to be issued on the trex server Parameters ---------- iom: int 0 = don't print stats screen to log, 1 = print stats (can generate huge logs) stateless: boolean True = run as stateless, False = require -f and -d arguments kwargs: dictionary Dictionary of parameters for trex. For example: (c=1, nc=True, l_pkt_mode=3). Notice that when sending command line parameters that has -, you need to replace it with _. for example, to have on command line "--l-pkt-mode 3", you need to send l_pkt_mode=3 export_path : str Full system path to which the results of the trex-run will be logged. """ if 'results_file_path' in kwargs: export_path = kwargs['results_file_path'] del kwargs['results_file_path'] if stateless: kwargs['i'] = True # adding additional options to the command trex_cmd_options = '' for key, value in kwargs.items(): tmp_key = key.replace('_', '-').lstrip('-') dash = ' -' if (len(key) == 1) else ' --' if value is True: trex_cmd_options += (dash + tmp_key) elif value is False: continue else: trex_cmd_options += (dash + '{k} {val}'.format(k=tmp_key, val=value)) if trex_args: trex_cmd_options += ' %s' % trex_args if not stateless: if 'f' not in kwargs: raise Exception( 'Argument -f should be specified in stateful command') if 'd' not in kwargs: raise Exception( 'Argument -d should be specified in stateful command') cmd = "{nice}{run_command}{debug_image} --iom {io} {cmd_options} --no-key".format( # -- iom 0 disables the periodic log to the screen (not needed) nice='' if self.trex_nice == 0 else 'nice -n %s ' % self.trex_nice, run_command=self.TREX_START_CMD, debug_image='-debug' if debug_image else '', cmd_options=trex_cmd_options, io=iom) logger.info("TREX FULL COMMAND: {command}".format(command=cmd)) return (cmd, export_path, kwargs.get('d', 0)) def __check_trex_path_validity(self): # check for executable existance if not os.path.exists(self.TREX_PATH + '/t-rex-64'): print( "The provided TRex path do not contain an executable TRex file.\nPlease check the path and retry." ) logger.error( "The provided TRex path do not contain an executable TRex file" ) exit(-1) # check for executable permissions st = os.stat(self.TREX_PATH + '/t-rex-64') if not bool(st.st_mode & (stat.S_IXUSR)): print( "The provided TRex path do not contain an TRex file with execution privileges.\nPlease check the files permissions and retry." ) logger.error( "The provided TRex path do not contain an TRex file with execution privileges" ) exit(-1) else: return def __check_files_path_validity(self): # first, check for path existance. otherwise, try creating it with appropriate credentials if not os.path.exists(self.trex_files_path): try: os.makedirs(self.trex_files_path, 0o660) return except os.error as inst: print( "The provided files path does not exist and cannot be created with needed access credentials using root user.\nPlease check the path's permissions and retry." ) logger.error( "The provided files path does not exist and cannot be created with needed access credentials using root user." ) exit(-1) elif os.access(self.trex_files_path, os.W_OK): return else: print( "The provided files path has insufficient access credentials for root user.\nPlease check the path's permissions and retry." ) logger.error( "The provided files path has insufficient access credentials for root user" ) exit(-1)
class CTRex(object): def __init__(self): self.status = TRexStatus.Idle self.verbose_status = 'TRex is Idle' self.errcode = None self.session = None self.zmq_monitor = None self.__zmq_dump = {} self.zmq_dump_lock = threading.Lock() self.zmq_error = None self.seq = None self.expect_trex = threading.Event() def __del__(self): if self.zmq_monitor: self.zmq_monitor.join() if self.session: self.session.join() def get_status(self): return self.status def set_status(self, new_status): self.status = new_status def get_verbose_status(self): return self.verbose_status def set_verbose_status(self, new_status): self.verbose_status = new_status def gen_seq(self): self.seq = randrange(1, 1000) def get_seq(self): return self.seq def get_latest_dump(self): with self.zmq_dump_lock: return json.dumps(self.__zmq_dump) def update_zmq_dump_key(self, key, val): with self.zmq_dump_lock: self.__zmq_dump[key] = val def clear_zmq_dump(self): with self.zmq_dump_lock: self.__zmq_dump = {} def get_running_info(self): if self.status == TRexStatus.Running: return self.get_latest_dump() else: logger.info( "TRex isn't running. Running information isn't available.") if self.status == TRexStatus.Idle: if self.errcode is not None: # some error occured logger.info( "TRex is in Idle state, with errors. returning fault") return Fault( self.errcode, self.verbose_status ) # raise at client relevant exception, depending on the reason the error occured else: logger.info( "TRex is in Idle state, no errors. returning {}") return u'{}' return Fault( -12, self.verbose_status ) # raise at client TRexWarning, indicating TRex is back to Idle state or still in Starting state def stop_trex(self): if self.status == TRexStatus.Idle: # TRex isn't running, nothing to abort logger.info("TRex isn't running. No need to stop anything.") if self.errcode is not None: # some error occurred, notify client despite TRex already stopped return Fault( self.errcode, self.verbose_status ) # raise at client relevant exception, depending on the reason the error occured return False else: # handle stopping TRex's run self.session.join() logger.info("TRex session has been successfully aborted.") return True def start_trex(self, trex_launch_path, trex_cmd, zmq_port): self.set_status(TRexStatus.Starting) logger.info("TRex running state changed to 'Starting'.") self.set_verbose_status('TRex is starting (data is not available yet)') if not self.zmq_monitor: logger.info('Starting ZMQ monitor on port %s' % zmq_port) self.zmq_monitor = ZmqMonitorSession(self, zmq_port) self.zmq_monitor.start() else: if not self.zmq_monitor.is_alive( ) or self.zmq_monitor.zmq_port != zmq_port: if not self.zmq_monitor.is_alive(): logger.info('Old ZMQ monitor is dead, starting new') else: logger.info( 'ZMQ port is changed to %s, starting new monitor' % zmq_port) self.zmq_monitor.join() self.zmq_monitor = ZmqMonitorSession(self, zmq_port) self.zmq_monitor.start() self.zmq_monitor.first_dump = True self.errcode = None self.session = AsynchronousTRexSession(self, trex_launch_path, trex_cmd) self.session.start() self.expect_trex.set()
class CTRexServer(object): """This class defines the server side of the RESTfull interaction with TRex""" DEFAULT_TREX_PATH = ( "/auto/proj-pcube-b/apps/PL-b/tools/bp_sim2/v1.55/" ) #'/auto/proj-pcube-b/apps/PL-b/tools/nightly/trex_latest' TREX_START_CMD = "./t-rex-64" DEFAULT_FILE_PATH = "/tmp/trex_files/" def __init__( self, trex_path, trex_files_path, trex_host="0.0.0.0", trex_daemon_port=8090, trex_zmq_port=4500, trex_nice=-19 ): """ Parameters ---------- trex_host : str a string of the TRex ip address or hostname. default value: machine hostname as fetched from socket.gethostname() trex_daemon_port : int the port number on which the trex-daemon server can be reached default value: 8090 trex_zmq_port : int the port number on which trex's zmq module will interact with daemon server default value: 4500 nice: int priority of the TRex process Instantiate a TRex client object, and connecting it to listening daemon-server """ self.TREX_PATH = os.path.abspath(os.path.dirname(trex_path + "/")) self.trex_files_path = os.path.abspath(os.path.dirname(trex_files_path + "/")) self.__check_trex_path_validity() self.__check_files_path_validity() self.trex = CTRex() self.trex_version = None self.trex_host = trex_host self.trex_daemon_port = trex_daemon_port self.trex_zmq_port = trex_zmq_port self.trex_server_path = "http://{hostname}:{port}".format(hostname=trex_host, port=trex_daemon_port) self.start_lock = threading.Lock() self.__reservation = None self.zmq_monitor = ZmqMonitorSession( self.trex, self.trex_zmq_port ) # intiate single ZMQ monitor thread for server usage self.trex_nice = int(trex_nice) if self.trex_nice < -20 or self.trex_nice > 19: err = "Parameter 'nice' should be integer in range [-20, 19]" print(err) logger.error(err) raise Exception(err) def add(self, x, y): logger.info("Processing add function. Parameters are: {0}, {1} ".format(x, y)) return x + y # return Fault(-10, "") def push_file(self, filename, bin_data): logger.info("Processing push_file() command.") try: filepath = os.path.join(self.trex_files_path, os.path.basename(filename)) with open(filepath, "wb") as f: f.write(binascii.a2b_base64(bin_data)) logger.info("push_file() command finished. File is saved as %s" % filepath) return True except IOError as inst: logger.error("push_file method failed. " + str(inst)) return False def connectivity_check(self): logger.info("Processing connectivity_check function.") return True def start(self): """This method fires up the daemon server based on initialized parameters of the class""" # initialize the server instance with given resources register_socket("trex_daemon_server") try: print("Firing up TRex REST daemon @ port {trex_port} ...\n".format(trex_port=self.trex_daemon_port)) logger.info("Firing up TRex REST daemon @ port {trex_port} ...".format(trex_port=self.trex_daemon_port)) logger.info("current working dir is: {0}".format(self.TREX_PATH)) logger.info("current files dir is : {0}".format(self.trex_files_path)) logger.debug("Starting TRex server. Registering methods to process.") logger.info(self.get_trex_version(base64=False)) self.server = SimpleJSONRPCServer((self.trex_host, self.trex_daemon_port)) except socket.error as e: if e.errno == errno.EADDRINUSE: logger.error("TRex server requested address already in use. Aborting server launching.") print("TRex server requested address already in use. Aborting server launching.") raise socket.error( errno.EADDRINUSE, "TRex daemon requested address already in use. " "Server launch aborted. Please make sure no other process is " "using the desired server properties.", ) elif isinstance(e, socket.gaierror) and e.errno == -3: # handling Temporary failure in name resolution exception raise socket.gaierror( -3, "Temporary failure in name resolution.\n" "Make sure provided hostname has DNS resolving." ) else: raise # set further functionality and peripherals to server instance self.server.register_function(self.add) self.server.register_function(self.cancel_reservation) self.server.register_function(self.connectivity_check) self.server.register_function(self.connectivity_check, "check_connectivity") # alias self.server.register_function(self.force_trex_kill) self.server.register_function(self.get_file) self.server.register_function(self.get_files_list) self.server.register_function(self.get_files_path) self.server.register_function(self.get_running_info) self.server.register_function(self.get_running_status) self.server.register_function(self.get_trex_cmds) self.server.register_function(self.get_trex_config) self.server.register_function(self.get_trex_daemon_log) self.server.register_function(self.get_trex_log) self.server.register_function(self.get_trex_version) self.server.register_function(self.is_reserved) self.server.register_function(self.is_running) self.server.register_function(self.kill_all_trexes) self.server.register_function(self.push_file) self.server.register_function(self.reserve_trex) self.server.register_function(self.start_trex) self.server.register_function(self.stop_trex) self.server.register_function(self.wait_until_kickoff_finish) signal.signal(signal.SIGTSTP, self.stop_handler) signal.signal(signal.SIGTERM, self.stop_handler) try: self.zmq_monitor.start() self.server.serve_forever() except KeyboardInterrupt: logger.info("Daemon shutdown request detected.") finally: self.zmq_monitor.join() # close ZMQ monitor thread resources self.server.shutdown() # self.server.server_close() # get files from Trex server and return their content (mainly for logs) @staticmethod def _pull_file(filepath): try: with open(filepath, "rb") as f: file_content = f.read() return binascii.b2a_base64(file_content).decode(errors="replace") except Exception as e: err_str = "Can't get requested file %s: %s" % (filepath, e) logger.error(err_str) return Fault(-33, err_str) # returns True if given path is under TRex package or under /tmp/trex_files def _check_path_under_TRex_or_temp(self, path): if not os.path.relpath(path, self.trex_files_path).startswith(os.pardir): return True if not os.path.relpath(path, self.TREX_PATH).startswith(os.pardir): return True return False # gets the file content encoded base64 either from /tmp/trex_files or TRex server dir def get_file(self, filepath): try: logger.info("Processing get_file() command.") if not self._check_path_under_TRex_or_temp(filepath): raise Exception("Given path should be under current TRex package or /tmp/trex_files") return self._pull_file(filepath) except Exception as e: err_str = "Can't get requested file %s: %s" % (filepath, e) logger.error(err_str) return Fault(-33, err_str) # get tuple (dirs, files) with directories and files lists from given path (limited under TRex package or /tmp/trex_files) def get_files_list(self, path): try: logger.info("Processing get_files_list() command, given path: %s" % path) if not self._check_path_under_TRex_or_temp(path): raise Exception("Given path should be under current TRex package or /tmp/trex_files") return os.walk(path).next()[1:3] except Exception as e: err_str = "Error processing get_files_list(): %s" % e logger.error(err_str) return Fault(-33, err_str) # get Trex log /tmp/trex.txt def get_trex_log(self): logger.info("Processing get_trex_log() command.") return self._pull_file("/tmp/trex.txt") # get /etc/trex_cfg.yaml def get_trex_config(self): logger.info("Processing get_trex_config() command.") return self._pull_file("/etc/trex_cfg.yaml") # get daemon log /var/log/trex/trex_daemon_server.log def get_trex_daemon_log(self): logger.info("Processing get_trex_daemon_log() command.") return self._pull_file("/var/log/trex/trex_daemon_server.log") # get Trex version from ./t-rex-64 --help (last lines starting with "Version : ...") def get_trex_version(self, base64=True): try: logger.info("Processing get_trex_version() command.") if not self.trex_version: ret_code, stdout, stderr = run_command("./t-rex-64 --help", cwd=self.TREX_PATH) search_result = re.search("\n\s*(Version\s*:.+)", stdout, re.DOTALL) if not search_result: raise Exception("Could not determine version from ./t-rex-64 --help") self.trex_version = binascii.b2a_base64(search_result.group(1).encode(errors="replace")) if base64: return self.trex_version.decode(errors="replace") else: return binascii.a2b_base64(self.trex_version).decode(errors="replace") except Exception as e: err_str = "Can't get trex version, error: %s" % e logger.error(err_str) return Fault(-33, err_str) def stop_handler(self, *args, **kwargs): logger.info("Daemon STOP request detected.") if self.is_running(): # in case TRex process is currently running, stop it before terminating server process self.stop_trex(self.trex.get_seq()) sys.exit(0) def assert_zmq_ok(self): if self.trex.zmq_error: raise Exception("ZMQ thread got error: %s" % self.trex.zmq_error) if not self.zmq_monitor.is_alive(): if self.trex.get_status() != TRexStatus.Idle: self.force_trex_kill() raise Exception("ZMQ thread is dead.") def is_running(self): run_status = self.trex.get_status() logger.info("Processing is_running() command. Running status is: {stat}".format(stat=run_status)) if run_status == TRexStatus.Running: return True else: return False def is_reserved(self): logger.info("Processing is_reserved() command.") return bool(self.__reservation) def get_running_status(self): run_status = self.trex.get_status() logger.info("Processing get_running_status() command. Running status is: {stat}".format(stat=run_status)) return {"state": run_status.value, "verbose": self.trex.get_verbose_status()} def get_files_path(self): logger.info("Processing get_files_path() command.") return self.trex_files_path def reserve_trex(self, user): if user == "": logger.info("TRex reservation cannot apply to empty string user. Request denied.") return Fault(-33, "TRex reservation cannot apply to empty string user. Request denied.") with self.start_lock: logger.info("Processing reserve_trex() command.") if self.is_reserved(): if user == self.__reservation["user"]: # return True is the same user is asking and already has the resrvation logger.info("the same user is asking and already has the resrvation. Re-reserving TRex.") return True logger.info( "TRex is already reserved to another user ({res_user}), cannot reserve to another user.".format( res_user=self.__reservation["user"] ) ) return Fault( -33, "TRex is already reserved to another user ({res_user}). Please make sure TRex is free before reserving it.".format( res_user=self.__reservation["user"] ), ) # raise at client TRexInUseError elif self.trex.get_status() != TRexStatus.Idle: logger.info("TRex is currently running, cannot reserve TRex unless in Idle state.") return Fault( -13, "TRex is currently running, cannot reserve TRex unless in Idle state. Please try again when TRex run finished.", ) # raise at client TRexInUseError else: logger.info("TRex is now reserved for user ({res_user}).".format(res_user=user)) self.__reservation = {"user": user, "since": time.ctime()} logger.debug("Reservation details: " + str(self.__reservation)) return True def cancel_reservation(self, user): with self.start_lock: logger.info("Processing cancel_reservation() command.") if self.is_reserved(): if self.__reservation["user"] == user: logger.info( "TRex reservation to {res_user} has been canceled successfully.".format( res_user=self.__reservation["user"] ) ) self.__reservation = None return True else: logger.warning( "TRex is reserved to different user than the provided one. Reservation wasn't canceled." ) return Fault( -33, "Cancel reservation request is available to the user that holds the reservation. Request denied", ) # raise at client TRexRequestDenied else: logger.info("TRex is not reserved to anyone. No need to cancel anything") assert self.__reservation is None return False def start_trex( self, trex_cmd_options, user, block_to_success=True, timeout=40, stateless=False, debug_image=False, trex_args="", ): self.assert_zmq_ok() with self.start_lock: logger.info("Processing start_trex() command.") if self.is_reserved(): # check if this is not the user to which TRex is reserved if self.__reservation["user"] != user: logger.info( "TRex is reserved to another user ({res_user}). Only that user is allowed to initiate new runs.".format( res_user=self.__reservation["user"] ) ) return Fault( -33, "TRex is reserved to another user ({res_user}). Only that user is allowed to initiate new runs.".format( res_user=self.__reservation["user"] ), ) # raise at client TRexRequestDenied elif self.trex.get_status() != TRexStatus.Idle: logger.info("TRex is already taken, cannot create another run until done.") return Fault(-13, "") # raise at client TRexInUseError try: server_cmd_data = self.generate_run_cmd( stateless=stateless, debug_image=debug_image, trex_args=trex_args, **trex_cmd_options ) self.zmq_monitor.first_dump = True self.trex.start_trex(self.TREX_PATH, server_cmd_data) logger.info("TRex session has been successfully initiated.") if block_to_success: # delay server response until TRex is at 'Running' state. start_time = time.time() trex_state = None while (time.time() - start_time) < timeout: trex_state = self.trex.get_status() if trex_state != TRexStatus.Starting: break else: time.sleep(0.5) self.assert_zmq_ok() # check for TRex run started normally if trex_state == TRexStatus.Starting: # reached timeout logger.warning( "TimeoutError: TRex initiation outcome could not be obtained, since TRex stays at Starting state beyond defined timeout." ) return Fault( -12, "TimeoutError: TRex initiation outcome could not be obtained, since TRex stays at Starting state beyond defined timeout.", ) # raise at client TRexWarning elif trex_state == TRexStatus.Idle: return Fault(-11, self.trex.get_verbose_status()) # raise at client TRexError # reach here only if TRex is at 'Running' state self.trex.gen_seq() return self.trex.get_seq() # return unique seq number to client except TypeError as e: logger.error( "TRex command generation failed, probably because either -f (traffic generation .yaml file) and -c (num of cores) was not specified correctly.\nReceived params: {params}".format( params=trex_cmd_options ) ) raise TypeError( "TRex -f (traffic generation .yaml file) and -c (num of cores) must be specified. %s" % e ) def stop_trex(self, seq): logger.info("Processing stop_trex() command.") if self.trex.get_seq() == seq: logger.debug("Abort request legit since seq# match") return self.trex.stop_trex() else: if self.trex.get_status() != TRexStatus.Idle: logger.warning("Abort request is only allowed to process initiated the run. Request denied.") return Fault( -33, "Abort request is only allowed to process initiated the run. Request denied." ) # raise at client TRexRequestDenied else: return False def force_trex_kill(self): logger.info("Processing force_trex_kill() command. --> Killing TRex session indiscriminately.") return self.trex.stop_trex() # returns list of tuples (pid, command line) of running TRex(es) def get_trex_cmds(self): logger.info("Processing get_trex_cmds() command.") ret_code, stdout, stderr = run_command("ps -u root --format pid,comm,cmd") if ret_code: raise Exception("Failed to determine running processes, stderr: %s" % stderr) trex_cmds_list = [] for line in stdout.splitlines(): pid, proc_name, full_cmd = line.strip().split(" ", 2) pid = pid.strip() full_cmd = full_cmd.strip() if proc_name.find("t-rex-64") >= 0: trex_cmds_list.append((pid, full_cmd)) return trex_cmds_list # Silently tries to kill TRexes with given signal. # Responsibility of client to verify with get_trex_cmds. def kill_all_trexes(self, signal_name): logger.info("Processing kill_all_trexes() command.") trex_cmds_list = self.get_trex_cmds() for pid, cmd in trex_cmds_list: logger.info("Killing with signal %s process %s %s" % (signal_name, pid, cmd)) os.kill(int(pid), signal_name) def wait_until_kickoff_finish(self, timeout=40): # block until TRex exits Starting state logger.info("Processing wait_until_kickoff_finish() command.") trex_state = None start_time = time.time() while (time.time() - start_time) < timeout: self.assert_zmq_ok() trex_state = self.trex.get_status() if trex_state != TRexStatus.Starting: return sleep(0.1) return Fault( -12, "TimeoutError: TRex initiation outcome could not be obtained, since TRex stays at Starting state beyond defined timeout.", ) # raise at client TRexWarning def get_running_info(self): self.assert_zmq_ok() logger.info("Processing get_running_info() command.") return self.trex.get_running_info() def generate_run_cmd( self, iom=0, export_path="/tmp/trex.txt", stateless=False, debug_image=False, trex_args="", **kwargs ): """ generate_run_cmd(self, iom, export_path, kwargs) -> str Generates a custom running command for the kick-off of the TRex traffic generator. Returns a tuple of command (string) and export path (string) to be issued on the trex server Parameters ---------- iom: int 0 = don't print stats screen to log, 1 = print stats (can generate huge logs) stateless: boolean True = run as stateless, False = require -f and -d arguments kwargs: dictionary Dictionary of parameters for trex. For example: (c=1, nc=True, l_pkt_mode=3). Notice that when sending command line parameters that has -, you need to replace it with _. for example, to have on command line "--l-pkt-mode 3", you need to send l_pkt_mode=3 export_path : str Full system path to which the results of the trex-run will be logged. """ if "results_file_path" in kwargs: export_path = kwargs["results_file_path"] del kwargs["results_file_path"] if stateless: kwargs["i"] = True # adding additional options to the command trex_cmd_options = "" for key, value in kwargs.items(): tmp_key = key.replace("_", "-").lstrip("-") dash = " -" if (len(key) == 1) else " --" if value is True: trex_cmd_options += dash + tmp_key elif value is False: continue else: trex_cmd_options += dash + "{k} {val}".format(k=tmp_key, val=value) if trex_args: trex_cmd_options += " %s" % trex_args if not stateless: if "f" not in kwargs: raise Exception("Argument -f should be specified in stateful command") if "d" not in kwargs: raise Exception("Argument -d should be specified in stateful command") cmd = "{nice}{run_command}{debug_image} --iom {io} {cmd_options} --no-key".format( # -- iom 0 disables the periodic log to the screen (not needed) nice="" if self.trex_nice == 0 else "nice -n %s " % self.trex_nice, run_command=self.TREX_START_CMD, debug_image="-debug" if debug_image else "", cmd_options=trex_cmd_options, io=iom, ) logger.info("TREX FULL COMMAND: {command}".format(command=cmd)) return (cmd, export_path, kwargs.get("d", 0)) def __check_trex_path_validity(self): # check for executable existance if not os.path.exists(self.TREX_PATH + "/t-rex-64"): print("The provided TRex path do not contain an executable TRex file.\nPlease check the path and retry.") logger.error("The provided TRex path do not contain an executable TRex file") exit(-1) # check for executable permissions st = os.stat(self.TREX_PATH + "/t-rex-64") if not bool(st.st_mode & (stat.S_IXUSR)): print( "The provided TRex path do not contain an TRex file with execution privileges.\nPlease check the files permissions and retry." ) logger.error("The provided TRex path do not contain an TRex file with execution privileges") exit(-1) else: return def __check_files_path_validity(self): # first, check for path existance. otherwise, try creating it with appropriate credentials if not os.path.exists(self.trex_files_path): try: os.makedirs(self.trex_files_path, 0o660) return except os.error as inst: print( "The provided files path does not exist and cannot be created with needed access credentials using root user.\nPlease check the path's permissions and retry." ) logger.error( "The provided files path does not exist and cannot be created with needed access credentials using root user." ) exit(-1) elif os.access(self.trex_files_path, os.W_OK): return else: print( "The provided files path has insufficient access credentials for root user.\nPlease check the path's permissions and retry." ) logger.error("The provided files path has insufficient access credentials for root user") exit(-1)