def _stop_event_subscribers(self): """ Stop event subscribers on cleanup. """ log.info('cleanup: _stop_event_subscribers called.') for sub in self._event_subscribers: sub.stop()
def consume_event(*args, **kwargs): log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) if self._no_events and self._no_events == len( self._event_received): self._async_event_result.set()
def memory_usage(self): """ Get the current memory usage for the current driver process. @returns memory usage in KB of the current driver process """ driver_pid = self.getpid() if not driver_pid: log.warn("no process running") return 0 ps_process = subprocess.Popen(["ps", "-o rss,pid", "-p %s" % self.getpid()], stdout=subprocess.PIPE) ps_process.poll() usage = 0 for line in ps_process.stdout: if not line.strip().startswith('RSS'): try: fields = line.split() pid = int(fields[1]) if pid == driver_pid: usage = int(fields[0]) except: log.warn("Failed to parse output for memory usage: %s" % line) usage = 0 if usage: log.info("process memory usage: %dk" % usage) else: log.warn("process not running") return usage
def _transform(self, obj): # Note: This check to detect an IonObject is a bit risky (only type_) if isinstance(obj, dict) and "type_" in obj: objc = obj otype = objc['type_'].encode('ascii') # Correct? # don't supply a dict - we want the object to initialize with all its defaults intact, # which preserves things like IonEnumObject and invokes the setattr behavior we want there. ion_obj = self._obj_registry.new(otype) # get outdated attributes in data that are not defined in the current schema extra_attributes = objc.viewkeys() - ion_obj._schema.viewkeys() - BUILT_IN_ATTRS for extra in extra_attributes: objc.pop(extra) log.info('discard %s not in current schema' % extra) for k, v in objc.iteritems(): # unicode translate to utf8 if isinstance(v, unicode): v = str(v.encode('utf8')) # CouchDB adds _attachments and puts metadata in it # in pyon metadata is in the document # so we discard _attachments while transforming between the two if k not in ("type_", "_attachments", "_conflicts"): setattr(ion_obj, k, v) if k == "_conflicts": log.warn("CouchDB conflict detected for ID=%S (ignored): %s", obj.get('_id', None), v) return ion_obj return obj
def test_25_get_set_params(self): self._initialize_and_run() p1 = TrhphParameter.TIME_BETWEEN_BURSTS result = self._get_params([p1]) seconds = result[p1] new_seconds = seconds + 5 if new_seconds > 30 or new_seconds < 15: new_seconds = 15 p2 = TrhphParameter.VERBOSE_MODE new_verbose = self._get_verbose_flag_for_set_test() valid_params = {p1: new_seconds, p2: new_verbose} log.info("setting: %s" % str(valid_params)) self._set_params(valid_params) result = self._get_params([p1, p2]) seconds = result[p1] self.assertEqual(seconds, new_seconds) verbose = result[p2] self.assertEqual(verbose, new_verbose)
def _launch(self): """ @brief Launch the port agent process. If the address isn't localhost then we don't start anything @retval return the command port the process is listening on. """ log.info("Startup Unix Port Agent") # Create port agent object. this_pid = os.getpid() if self._test_mode else None log.debug(" -- our pid: %s" % this_pid) log.debug(" -- command port: %s" % self._command_port) log.debug(" -- address: %s, port: %s" % (self._device_addr, self._device_port)) command_line = [self._binary_path, self._type, self._data_port, self._command_port, self._device_addr] if self._type == PortAgentType.ETHERNET: command_line.extend([self._device_port]) elif self._type == PortAgentType.RSN: command_line.extend([self._device_port, self._device_cmd_port]) elif self._type == PortAgentType.BOTPT: command_line.extend([self._device_rx_port, self._device_tx_port]) if self._sniffer_port: command_line.append('--sniff=%d' % self._sniffer_port) command_line = [str(arg) for arg in command_line] self._pid = self.run_command(command_line) return self._command_port
def memory_usage(self): """ Get the current memory usage for the current driver process. @returns memory usage in KB of the current driver process """ driver_pid = self.getpid() if not driver_pid: log.warn("no process running") return 0 ps_process = subprocess.Popen( ["ps", "-o rss,pid", "-p %s" % self.getpid()], stdout=subprocess.PIPE) retcode = ps_process.poll() usage = 0 for line in ps_process.stdout: if not line.strip().startswith('RSS'): try: fields = line.split() pid = int(fields[1]) if pid == driver_pid: usage = int(fields[0]) except: log.warn("Failed to parse output for memory usage: %s" % line) usage = 0 if usage: log.info("process memory usage: %dk" % usage) else: log.warn("process not running") return usage
def test_99_execute_start_autosample(self): self._initialize_and_run() log.info("executing start_autosample") cmd = AgentCommand(command='start_autosample', kwargs=dict(timeout=self._timeout)) reply = self._ia_client.execute(cmd) log.info("start_autosample reply = %s" % str(reply))
def _stop_data_subscribers(self): """ Stop the data subscribers on cleanup. """ log.info('cleanup: _stop_data_subscribers called.') for sub in self._data_subscribers: sub.stop() for gl in self._data_greenlets: gl.kill()
def stop(self): if self.port_agent: pid = self.port_agent.get_pid() if pid: log.info('Stopping pagent pid %i' % pid) self.port_agent.stop() else: log.info('No port agent running.')
def test_70_execute_get_metadata(self): self._initialize_and_run() log.info("getting metadata") cmd = AgentCommand(command='get_metadata', kwargs=dict(timeout=self._timeout)) reply = self._ia_client.execute(cmd) log.info("get_metadata reply = %s" % str(reply)) self.assertTrue(isinstance(reply.result, dict))
def test_90_execute_get_power_statuses(self): self._initialize_and_run() log.info("executing get_power_statuses") cmd = AgentCommand(command='get_power_statuses', kwargs=dict(timeout=self._timeout)) reply = self._ia_client.execute(cmd) log.info("get_power_statuses reply = %s" % str(reply)) self.assertTrue(isinstance(reply.result, dict))
def test_80_execute_diagnostics(self): self._initialize_and_run() log.info("executing diagnostics") num_scans = 11 cmd = AgentCommand(command='diagnostics', kwargs=dict(num_scans=num_scans, timeout=self._timeout)) reply = self._ia_client.execute(cmd) log.info("diagnostics reply = %s" % str(reply)) self.assertTrue(isinstance(reply.result, list)) self.assertEqual(len(reply.result), num_scans)
def _reset(self): """ Set as a clean-up to make sure the agent is reset after each test, so the driver is stopped. """ log.info("_reset called: sending 'reset' command to agent") cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_agent_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def launch(self): """ @brief Launch the driver process and driver client. This is used in the integration and qualification tests. The port agent abstracts the physical interface with the instrument. @retval return the pid to the logger process """ log.info("Startup Port Agent") # Create port agent object. this_pid = os.getpid() if self._test_mode else None log.debug( " -- our pid: %s" % this_pid) log.debug( " -- address: %s, port: %s" % (self._device_addr, self._device_port)) # Working dir and delim are hard coded here because this launch process # will change with the new port agent. self.port_agent = EthernetDeviceLogger.launch_process( self._device_addr, self._device_port, self._working_dir, self._delimiter, this_pid) log.debug( " Port agent object created" ) start_time = time.time() expire_time = start_time + int(self._timeout) pid = self.port_agent.get_pid() while not pid: gevent.sleep(.1) pid = self.port_agent.get_pid() if time.time() > expire_time: log.error("!!!! Failed to start Port Agent !!!!") raise PortAgentTimeout('port agent could not be started') self._pid = pid port = self.port_agent.get_port() start_time = time.time() expire_time = start_time + int(self._timeout) while not port: gevent.sleep(.1) port = self.port_agent.get_port() if time.time() > expire_time: log.error("!!!! Port Agent could not bind to port !!!!") self.stop() raise PortAgentTimeout('port agent could not bind to port') self._data_port = port log.info('Started port agent pid %s listening at port %s' % (pid, port)) return port
def launch(self): ''' @brief Launch a port agent process if it is supposed to run on the local host Otherwise do nothing. @return the command port the port agent is listening on. ''' if (self._pa_addr == LOCALHOST): self._launch() else: self._pid = None log.info("Port Agent Address: %s" % (self._pa_addr)) log.info("Not starting port agent") return self._command_port
def launch(self): """ @brief Launch the driver process and driver client. This is used in the integration and qualification tests. The port agent abstracts the physical interface with the instrument. @retval return the pid to the logger process """ log.info("Startup Port Agent") # Create port agent object. this_pid = os.getpid() if self._test_mode else None log.debug(" -- our pid: %s" % this_pid) log.debug(" -- address: %s, port: %s" % (self._device_addr, self._device_port)) # Working dir and delim are hard coded here because this launch process # will change with the new port agent. self.port_agent = EthernetDeviceLogger.launch_process( self._device_addr, self._device_port, self._working_dir, self._delimiter, this_pid) log.debug(" Port agent object created") start_time = time.time() expire_time = start_time + int(self._timeout) pid = self.port_agent.get_pid() while not pid: gevent.sleep(.1) pid = self.port_agent.get_pid() if time.time() > expire_time: log.error("!!!! Failed to start Port Agent !!!!") raise PortAgentTimeout('port agent could not be started') self._pid = pid port = self.port_agent.get_port() start_time = time.time() expire_time = start_time + int(self._timeout) while not port: gevent.sleep(.1) port = self.port_agent.get_port() if time.time() > expire_time: log.error("!!!! Port Agent could not bind to port !!!!") self.stop() raise PortAgentTimeout('port agent could not bind to port') self._data_port = port log.info('Started port agent pid %s listening at port %s' % (pid, port)) return port
def launch(self): ''' @brief Launch a port agent process if it is supposed to run on the local host Otherwise do nothing. @return the command port the port agent is listening on. ''' if self._pa_addr == LOCALHOST: self._launch() else: self._pid = None log.info("Port Agent Address: %s" % self._pa_addr) log.info("Not starting port agent") return self._command_port
def _readline(self, timeout=None): # must be used by all servers that inherit this class to allow the server greenlet to detect that # server shut down is requested # used to support the 'login' dialog for the telnet client start_time = time.time() input_data = '' while True: input_data += self._get_data(1) if '\r\n' in input_data: return input_data.split('\r\n')[0] # don't pass back the line terminators if timeout: # if a timeout was specified then check for if it has elapsed if ((time.time() - start_time) > timeout): log.info("TcpServer._readline(): timeout, rcvd <%s>" %input_data) self._exit_handler(SessionCloseReasons.telnet_setup_timeout)
def _get_verbose_flag_for_set_test(self): """ Gets the value to use for the verbose flag in the "set" operations. If we are testing against the real instrument (self._is_real_instrument is true), this always returns False because the associated interfaces with verbose=True are not implemented yet. Otherwise it returns a random boolean value. Note, in this case, which means we are testing against the simulator, the actual verbose value does not have any effect of the other interface elements, so it is ok to set any value here. TODO align this when the verbose flag is handled completely, both in the driver and in the simulator. """ if self._is_real_instrument: log.info("setting verbose=False because _is_real_instrument") return False else: return 0 == random.randint(0, 1)
def _set_params(self, params): """ Sets the given parameters, which are assumed to be all valid. """ result = self._ia_client.set_param(params) log.info("set result = %s" % str(result)) if result is None: # TODO check why self._ia_client.set_param returns None return assert isinstance(result, dict) # check all requested params are in the result for (p, v) in params.items(): self.assertTrue(p in result) return result
def _read_pid(self): pid_file = PID_FILE % (PROCESS_BASE_DIR, self._command_port) start_time = time.time() boo = 0 log.debug("read pid file: " + pid_file) while (start_time + DEFAULT_TIMEOUT > time.time()): try: file = open(pid_file) pid = file.read().strip('\0\n\r') if (pid): int(pid) log.info("port agent pid: [%s]" % (pid)) return int(pid) except ValueError, e: log.warn("Failed to convert %s to an int '%s" % (pid, e)) break except:
def _read_pid(self): pid_file = PID_FILE % (PROCESS_BASE_DIR, self._command_port) start_time = time.time() boo = 0; log.debug("read pid file: " + pid_file) while(start_time + DEFAULT_TIMEOUT > time.time()): try: file = open(pid_file) pid = file.read().strip('\0\n\r') if(pid): int(pid) log.info("port agent pid: [%s]" % (pid)) return int(pid) except ValueError, e: log.warn("Failed to convert %s to an int '%s" % (pid, e) ) break except:
def stop(self, force=False): """ Stop the driver process. We try to stop gracefully using the driver client if we can, otherwise a simple kill does the job. """ if self._driver_process: if not force and self._driver_client: try: log.info('Stopping driver process.') self._driver_client.done() self._driver_process.wait() log.info('Driver process stopped.') except: try: log.error( 'Exception stopping driver process...killing.') self._driver_client.stop_messaging() self._driver_process.poll() if not self._driver_process.returncode: self._driver_process.kill() self._driver_process.wait() log.info('Driver process killed.') except Exception as ex: log.error('Exception killing driver process') log.error(type(ex)) log.error(ex) else: try: log.info('Killing driver process.') self._driver_client.stop_messaging() self._driver_process.poll() if not self._driver_process.returncode: self._driver_process.kill() self._driver_process.wait() log.info('Driver process killed.') except Exception as ex: log.error('Exception killing driver process.') log.error(type(ex)) log.error(ex) self._driver_process = None self._driver_client = None
def stop(self, force=False): """ Stop the driver process. We try to stop gracefully using the driver client if we can, otherwise a simple kill does the job. """ if self._driver_process: if not force and self._driver_client: try: log.info('Stopping driver process.') self._driver_client.done() self._driver_process.wait() log.info('Driver process stopped.') except: try: log.error('Exception stopping driver process...killing.') self._driver_client.stop_messaging() self._driver_process.poll() if not self._driver_process.returncode: self._driver_process.kill() self._driver_process.wait() log.info('Driver process killed.') except Exception as ex: log.error('Exception killing driver process') log.error(type(ex)) log.error(ex) else: try: log.info('Killing driver process.') self._driver_client.stop_messaging() self._driver_process.poll() if not self._driver_process.returncode: self._driver_process.kill() self._driver_process.wait() log.info('Driver process killed.') except Exception as ex: log.error('Exception killing driver process.') log.error(type(ex)) log.error(ex) self._driver_process = None self._driver_client = None
def launch(self): """ Launch the driver process. Once the process is launched read the two status files that contain the event port and command port for the driver. @raises DriverLaunchException """ log.info("Launch driver process") cmd = self._process_command() self._driver_process = self._spawn(cmd) if not self._driver_process and not self.poll(): log.error("Failed to launch driver: %s", cmd) raise DriverLaunchException('Error starting driver process') log.debug("driver process started, pid: %s", self.getpid()) self._command_port = self._get_port_from_file(self._driver_command_port_file()) self._event_port = self._get_port_from_file(self._driver_event_port_file()) log.debug("-- command port: %s, event port: %s", self._command_port, self._event_port)
def _launch(self): """ @brief Launch the port agent process. If the address isn't localhost then we don't start anything @retval return the command port the process is listening on. """ log.info("Startup Unix Port Agent") # Create port agent object. this_pid = os.getpid() if self._test_mode else None log.debug( " -- our pid: %s" % this_pid) log.debug( " -- command port: %s" % (self._command_port)) log.debug( " -- address: %s, port: %s" % (self._device_addr, self._device_port)) command_line = [ self._binary_path ] if(self._log_level > 0): for num in range(1, self._log_level): command_line.append("-v"); if(self._test_mode): this_pid = os.getpid(); command_line.append("--ppid") command_line.append(str(this_pid)) command_line.append("-p") command_line.append("%s" % (self._command_port)); command_line.append("-c") command_line.append(self._tmp_config.name); self.run_command(command_line); self._pid = self._read_pid(); return self._command_port;
def stop(self): log.info('Stop port agent') # When calling stop, IMS grabs a new port agent process via PortAgentProcess.get_process # So self._pid is None and needs to be initialized pid_file = PID_FILE % (PROCESS_BASE_DIR, self._command_port) try: with open(pid_file, 'r') as f: pid = f.read().strip('\0\n\4') if pid: try: self._pid = int(pid) except ValueError: pass except IOError: log.exception('Port agent pid file not found!') command_line = [self._binary_path] command_line.append("-c") command_line.append(self._tmp_config.name) command_line.append("-k") command_line.append("-p") command_line.append("%s" % (self._command_port)) self.run_command(command_line) timeout = Timeout(5) timeout.start() try: while self.poll(): log.warn('WAITING HERE with pid %s' % self._pid) gevent.sleep(1) except Timeout, t: log.error( 'Timed out waiting for pagent to die. Going in for kill.') os.kill(self._pid, signal.SIGKILL)
def launch(self): """ Launch the driver process. Once the process is launched read the two status files that contain the event port and command port for the driver. @raises DriverLaunchException """ log.info("Launch driver process") cmd = self._process_command() self._driver_process = self._spawn(cmd) if not self._driver_process and not self.poll(): log.error("Failed to launch driver: %s", cmd) raise DriverLaunchException('Error starting driver process') log.debug("driver process started, pid: %s", self.getpid()) self._command_port = self._get_port_from_file( self._driver_command_port_file()) self._event_port = self._get_port_from_file( self._driver_event_port_file()) log.debug("-- command port: %s, event port: %s", self._command_port, self._event_port)
def _transform(self, obj): # Note: This check to detect an IonObject is a bit risky (only type_) if isinstance(obj, dict) and "type_" in obj: objc = obj otype = objc['type_'].encode('ascii') # Correct? # don't supply a dict - we want the object to initialize with all its defaults intact, # which preserves things like IonEnumObject and invokes the setattr behavior we want there. ion_obj = self._obj_registry.new(otype) # get outdated attributes in data that are not defined in the current schema extra_attributes = objc.viewkeys() - ion_obj._schema.viewkeys( ) - BUILT_IN_ATTRS for extra in extra_attributes: objc.pop(extra) log.info('discard %s not in current schema' % extra) for k, v in objc.iteritems(): # unicode translate to utf8 if isinstance(v, unicode): v = str(v.encode('utf8')) # CouchDB adds _attachments and puts metadata in it # in pyon metadata is in the document # so we discard _attachments while transforming between the two if k not in ("type_", "_attachments", "_conflicts"): setattr(ion_obj, k, v) if k == "_conflicts": log.warn( "CouchDB conflict detected for ID=%S (ignored): %s", obj.get('_id', None), v) return ion_obj return obj
def _get_params(self, params): result = self._ia_client.get_param(params) log.info('_get_params result: %s' % str(result)) self.assertTrue(isinstance(result, dict)) if params == DriverParameter.ALL: all_requested_params = TrhphParameter.list() else: all_requested_params = params # check all requested params are in the result for p in all_requested_params: self.assertTrue(p in result) if TrhphParameter.TIME_BETWEEN_BURSTS == p: seconds = result.get(p) self.assertTrue(isinstance(seconds, int)) elif TrhphParameter.VERBOSE_MODE == p: is_data_only = result.get(p) self.assertTrue(isinstance(is_data_only, bool)) return result
def stop(self): log.info('Stop port agent') # When calling stop, IMS grabs a new port agent process via PortAgentProcess.get_process # So self._pid is None and needs to be initialized pid_file = PID_FILE % (PROCESS_BASE_DIR, self._command_port) try: with open(pid_file, 'r') as f: pid = f.read().strip('\0\n\4') if pid: try: self._pid = int(pid) except ValueError: pass except IOError: log.exception('Port agent pid file not found!') command_line = [ self._binary_path ] command_line.append("-c") command_line.append(self._tmp_config.name); command_line.append("-k") command_line.append("-p") command_line.append("%s" % (self._command_port)); self.run_command(command_line); timeout = Timeout(5) timeout.start() try: while self.poll(): log.warn('WAITING HERE with pid %s' % self._pid) gevent.sleep(1) except Timeout, t: log.error('Timed out waiting for pagent to die. Going in for kill.') os.kill(self._pid, signal.SIGKILL)
def _launch(self): """ @brief Launch the port agent process. If the address isn't localhost then we don't start anything @retval return the command port the process is listening on. """ log.info("Startup Unix Port Agent") # Create port agent object. this_pid = os.getpid() if self._test_mode else None log.debug(" -- our pid: %s" % this_pid) log.debug(" -- command port: %s" % (self._command_port)) log.debug(" -- address: %s, port: %s" % (self._device_addr, self._device_port)) command_line = [self._binary_path] if (self._log_level > 0): for num in range(1, self._log_level): command_line.append("-v") if (self._test_mode): this_pid = os.getpid() command_line.append("--ppid") command_line.append(str(this_pid)) command_line.append("-p") command_line.append("%s" % (self._command_port)) command_line.append("-c") command_line.append(self._tmp_config.name) self.run_command(command_line) self._pid = self._read_pid() return self._command_port
def _server_greenlet(self): # the main DA server thread that listens for a tcp client connection and passes it to the # handler for processing. It expects the handler to raise an exception when it needs to quit, # but it will work correctly if it simply returns when done log.debug("TcpServer._server_greenlet(): started") # set socket to timeout on blocking calls so accept() method will timeout to let server # check the stop_server flag self.server_socket.settimeout(1.0) self.server_socket.listen(1) while True: try: self.connection_socket, address = self.server_socket.accept() log.info("TcpServer._server_greenlet(): connection accepted from <%s>" %str(address)) # set socket to non-blocking so recv() will not block to let server # check the stop_server flag self.connection_socket.setblocking(0) break except Exception as ex: log.debug("TcpServer._server_greenlet(): exception caught while listening for connection <%s>" %str(ex)) if self.server_socket.timeout: # check to see if server shut down is requested if self.stop_server: self._notify_parent() return else: # something is wrong with the tcp socket so shut down log.info("TcpServer._server_greenlet(): exception caught while listening for connection <%s>" %str(ex)) self._indicate_server_stopping(SessionCloseReasons.socket_error) self._notify_parent() return # got a client connection so call handler to process it try: self._handler() except Exception as ex: log.info("TcpServer._server_greenlet(): exception caught from handler <%s>" %str(ex)) finally: # handler has exited so indicate to parent the server is shutting down self._notify_parent()
def stop(self): log.info('Stop port agent') os.kill(self._pid, 2)
def consume_event(*args, **kwargs): log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) if self._no_events and self._no_events == len(self._event_received): self._async_event_result.set()
def consume_data(message, headers): log.info('Subscriber received data message: %s.', str(message)) self._samples_received.append(message) if self._no_samples and self._no_samples == len( self._samples_received): self._async_data_result.set()
def consume_data(message, headers): log.info('Subscriber received data message: %s.', str(message)) self._samples_received.append(message) if self._no_samples and self._no_samples == len(self._samples_received): self._async_data_result.set()