def _process_command(self):
"""
Build the process command line using the driver_config dict
@return a list containing spawn args for the _spawn method
1. check cache (CACHE_DIR = '/tmp')
2. download to cache if not present.
3. construct call command
"""
log.debug("cwd: %s", os.getcwd())
driver_package = self.config.get('dvr_egg')
ppid = os.getpid() if self.test_mode else None
python = PYTHON_PATH
if not driver_package:
raise DriverLaunchException(
"missing driver config: driver_package")
if not os.path.exists(python):
raise DriverLaunchException(
"could not find python executable: %s" % python)
path = self._get_egg(self.config.get('dvr_egg'))
log.debug("_process_command: %s", path)
cmd_port_fname = self._driver_command_port_file()
evt_port_fname = self._driver_event_port_file()
cmd_str = "import sys; sys.path.insert(0, '%s/%s'); from mi.main import run; sys.exit(run(command_port_file='%s', event_port_file='%s', ppid=%s))" % \
(CACHE_DIR, get_filename_from_uri(driver_package), cmd_port_fname, evt_port_fname, str(ppid))
return [python, '-c', cmd_str]
def get_process(cls, driver_config, test_mode=False):
"""
Factory method to get the correct driver process object based on the type in driver_config
@param driver_config a dict containing configuration information for the driver process. This will be different
for each driver process type, but each should minimally have 'process_type' which defines which object
to use for launching.
@param test_mode tell the driver you are running in test mode. Some drivers use this to add a poison pill to
the driver process.
"""
type = driver_config.get("process_type")[0]
driver_module = driver_config.get('dvr_mod')
if not type:
raise DriverLaunchException("missing driver config: process_type")
# For some reason the enum wasn't working with the instrument agent. I would see [emum] when called from
# the IA, but (enum,) when called from this module.
#
#
elif type == DriverProcessType.PYTHON_MODULE:
log.debug("DriverProcessType.PYTHON_MODULE")
return ZMQPyClassDriverProcess(driver_config, test_mode)
elif type == DriverProcessType.EGG:
log.debug("DriverProcessType.EGG")
return ZMQEggDriverProcess(driver_config, test_mode)
else:
raise DriverLaunchException("unknown driver process type: %s" %
type)
def _get_data(self, timeout=None):
# must be used by all servers that inherit this class to allow the server greenlet to detect that
# server shut down is requested
log.debug("TcpServer._get_data(): timeout = %s" %str(timeout))
start_time = time.time()
input_data = ''
self.connection_socket.setblocking(0) # just to be sure!
while True:
try:
# this call must be non-blocking to let server check the stop_server flag
input_data = self.connection_socket.recv(1024)
if len(input_data) == 0:
# one way that the socket can indicate that the client has closed the connection
self._exit_handler(SessionCloseReasons.client_closed)
self.activity_seen = True;
return input_data
except gevent.socket.error, error:
if error.errno == errno.EAGAIN or error.errno == errno.EWOULDBLOCK:
# exceptions that indicate that nothing is available to read
if self.stop_server:
self._exit_handler(self.close_reason)
if timeout:
# if a timeout was specified then check for if it has elapsed
if ((time.time() - start_time) > timeout):
return ''
gevent.sleep(.1)
else:
# some socket error condition other than 'nothing to read' so shut down server
log.debug("TcpServer._get_data(): exception caught <%s>" %str(error))
self._exit_handler(SessionCloseReasons.client_closed)
def _process_command(self):
"""
Build the process command line using the driver_config dict
@return a list containing spawn args for the _spawn method
1. check cache (CACHE_DIR = '/tmp')
2. download to cache if not present.
3. construct call command
"""
log.debug("cwd: %s", os.getcwd())
driver_package = self.config.get('dvr_egg')
ppid = os.getpid() if self.test_mode else None
python = PYTHON_PATH
if not driver_package:
raise DriverLaunchException("missing driver config: driver_package")
if not os.path.exists(python):
raise DriverLaunchException("could not find python executable: %s" % python)
path = self._get_egg(self.config.get('dvr_egg'))
log.debug("_process_command: %s", path)
cmd_port_fname = self._driver_command_port_file()
evt_port_fname = self._driver_event_port_file()
cmd_str = "import sys; sys.path.insert(0, '%s/%s'); from mi.main import run; sys.exit(run(command_port_file='%s', event_port_file='%s', ppid=%s))" % \
(CACHE_DIR, get_filename_from_uri(driver_package), cmd_port_fname, evt_port_fname, str(ppid))
return [python, '-c', cmd_str]
def get_process(cls, driver_config, test_mode=False):
"""
Factory method to get the correct driver process object based on the type in driver_config
@param driver_config a dict containing configuration information for the driver process. This will be different
for each driver process type, but each should minimally have 'process_type' which defines which object
to use for launching.
@param test_mode tell the driver you are running in test mode. Some drivers use this to add a poison pill to
the driver process.
"""
type = driver_config.get("process_type")[0]
driver_module = driver_config.get('dvr_mod')
if not type:
raise DriverLaunchException("missing driver config: process_type")
# For some reason the enum wasn't working with the instrument agent. I would see [emum] when called from
# the IA, but (enum,) when called from this module.
#
#
elif type == DriverProcessType.PYTHON_MODULE:
log.debug("DriverProcessType.PYTHON_MODULE")
return ZMQPyClassDriverProcess(driver_config, test_mode)
elif type == DriverProcessType.EGG:
log.debug("DriverProcessType.EGG")
return ZMQEggDriverProcess(driver_config, test_mode)
else:
raise DriverLaunchException("unknown driver process type: %s" % type)
def _process_command(self):
"""
Build the process command line using the driver_config dict
@return a list containing spawn args for the _spawn method
"""
log.debug("cwd: %s", os.getcwd())
mi_repo = self.config.get('mi_repo', None)
driver_module = self.config.get('dvr_mod')
driver_class = self.config.get('dvr_cls')
ppid = os.getpid() if self.test_mode else None
python = PYTHON_PATH
if not driver_module:
raise DriverLaunchException("missing driver config: driver_module")
if not driver_class:
raise DriverLaunchException("missing driver config: driver_class")
if not os.path.exists(python):
raise DriverLaunchException(
"could not find python executable: %s" % python)
cmd_port_fname = self._driver_command_port_file()
evt_port_fname = self._driver_event_port_file()
cmd_str = ''
if mi_repo:
cmd_str += 'import sys; sys.path.insert(0,"%s");' % mi_repo
cmd_str += 'from %s import %s; dp = %s("%s", "%s", "%s", "%s", %s);dp.run()'\
% ('mi.core.instrument.zmq_driver_process', 'ZmqDriverProcess', 'ZmqDriverProcess', driver_module,
driver_class, cmd_port_fname, evt_port_fname, str(ppid))
return [python, '-c', cmd_str]
def assertNoParticleRegression(self, filepath, data_handler):
"""
Compares particles with previous run. If no YAML file exists, creates one.
:param filepath: fully qualified name of the input file (that was parsed)
:param data_handler: ParticleDataHandler returned from parse()
:return:
"""
yaml_file = os.path.splitext(filepath)[0] + '.yml'
particles = data_handler._samples
if os.path.isfile(yaml_file):
with open(yaml_file, 'r') as stream:
prev_particles = yaml.load(stream)
# particle key names should match
self.assertListEqual(sorted(prev_particles.keys()),
sorted(particles.keys()))
# compare number of samples across one of the particle keys
for p in prev_particles.keys():
log.debug('%s: %d %d', p, len(prev_particles[p]),
len(particles[p]))
self.assertEqual(len(prev_particles[p]), len(particles[p]))
else:
with open(yaml_file, 'w') as stream:
log.warn(
'creating yaml output file for regression testing - commit %s',
yaml_file)
yaml.dump(particles, stream, default_flow_style=False)
def _process_command(self):
"""
Build the process command line using the driver_config dict
@return a list containing spawn args for the _spawn method
"""
log.debug("cwd: %s", os.getcwd())
mi_repo = self.config.get('mi_repo', None)
driver_module = self.config.get('dvr_mod')
driver_class = self.config.get('dvr_cls')
ppid = os.getpid() if self.test_mode else None
python = PYTHON_PATH
if not driver_module:
raise DriverLaunchException("missing driver config: driver_module")
if not driver_class:
raise DriverLaunchException("missing driver config: driver_class")
cmd_port_fname = self._driver_command_port_file()
evt_port_fname = self._driver_event_port_file()
cmd_str = ''
if mi_repo:
cmd_str += 'import sys; sys.path.insert(0,"%s");' % mi_repo
cmd_str += 'from %s import %s; dp = %s("%s", "%s", "%s", "%s", %s);dp.run()' \
% ('mi.core.instrument.zmq_driver_process', 'ZmqDriverProcess', 'ZmqDriverProcess', driver_module,
driver_class, cmd_port_fname, evt_port_fname, str(ppid))
return [python, '-c', cmd_str]
def _authorized(self, token):
# check for correct token to authorize this connection
if token == self.token:
return True
else:
log.debug("TcpServer._authorized: entered token =" + token + ", not equal to expected token =" + self.token)
return False
def _prepare_context(self, resource_id):
"""
Initializes the context object and loads associations for resource id.
"""
self.ctx = dict(by_subject={}, by_object={})
assocs = self._rr.find_associations(anyside=resource_id, id_only=False)
self._add_associations(assocs)
log.debug("Found %s associations for resource %s", len(assocs), resource_id)
def _egg_path(self, egg_name):
log.debug("_egg_path")
if not CACHE_DIR:
raise ServerError("CACHE_DIR is %s'%s'" % (type(CACHE_DIR), CACHE_DIR))
elif not egg_name:
raise ServerError("egg_name for path is %s'%s'" % (type(egg_name), egg_name))
return CACHE_DIR + "/" + egg_name
def _get_egg(self, egg_uri):
filename = get_filename_from_uri(egg_uri)
log.debug("_get_egg: %s", filename)
path = self._check_cache_for_egg(filename)
if None == path:
path = self._get_remote_egg(filename) # Will exception out if problem.
return path
def _egg_remotepath(self, egg_name):
log.debug("_egg_remotepath: %s", egg_name)
if not REPO_BASE:
raise ServerError("REPO_BASE is %s'%s'" % (type(REPO_BASE), REPO_BASE))
elif not egg_name:
raise ServerError("egg_name for remotepath is %s'%s'" % (type(egg_name), egg_name))
return REPO_BASE + "/" + egg_name
def _exit_handler (self, reason):
# used by handlers to indicate to the server greenlet that it should shut down
log.debug("TcpServer._exit_handler(): stopping, reason = %s" %SessionCloseReasons.string(reason))
if self.stop_server == False:
# not already stopping for another reason so indicate the handler's reason for shut down
self._indicate_server_stopping(reason)
# raise the server exit exception to the server greenlet
raise ServerExitException("TcpServer: exiting from server, reason = %s" %SessionCloseReasons.string(reason))
def _get_egg(self, egg_uri):
filename = get_filename_from_uri(egg_uri)
log.debug("_get_egg: %s", filename)
path = self._check_cache_for_egg(filename)
if None == path:
path = self._get_remote_egg(
filename) # Will exception out if problem.
return path
def setUp(self):
"""
Initialize test members.
Start port agent.
Start container and client.
Start streams and subscribers.
Start agent, client.
"""
TrhphTestCase.setUp(self)
# Start port agent, add stop to cleanup.
self._pagent = None
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
self._start_container()
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._no_samples = None
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# Start event subscribers, add stop to cleanup.
self._no_events = None
self._async_event_result = AsyncResult()
self._events_received = []
self._event_subscribers = []
self._start_event_subscribers()
self.addCleanup(self._stop_event_subscribers)
# Create agent config.
agent_config = {
'driver_config': DVR_CONFIG,
'stream_config': self._stream_config,
'agent': {
'resource_id': IA_RESOURCE_ID
},
'test_mode': True
}
# Start instrument agent.
self._ia_pid = None
log.debug("TestInstrumentAgentWithTrhph.setup(): starting IA.")
# make sure the driver is stopped
self.addCleanup(self._reset)
def _spawn(self, spawnargs):
"""
Launch a process using popen
@param spawnargs a list of arguments for the Popen command line. The first argument must be a path to a
program and arguments much be in additional list elements.
@returns subprocess.Popen object
"""
log.debug("run cmd: %s", " ".join(spawnargs))
return subprocess.Popen(spawnargs, close_fds=True)
def _spawn(self, spawnargs):
"""
Launch a process using popen
@param spawnargs a list of arguments for the Popen command line. The first argument must be a path to a
program and arguments much be in additional list elements.
@returns subprocess.Popen object
"""
log.debug("run cmd: %s", " ".join(spawnargs))
return subprocess.Popen(spawnargs, close_fds=True)
def _forward_data_to_parent(self, data):
if self.stop_server:
self._exit_handler(self.close_reason)
try:
self.parent_input_callback(data)
except Exception as ex:
log.debug("TcpServer._forward_data_to_parent(): exception caught while trying to forward data to IA, exception = <%s>"
%str(ex))
self._exit_handler(SessionCloseReasons.instrument_agent_exception)
def _egg_path(self, egg_name):
log.debug("_egg_path")
if not CACHE_DIR:
raise ServerError("CACHE_DIR is %s'%s'" %
(type(CACHE_DIR), CACHE_DIR))
elif not egg_name:
raise ServerError("egg_name for path is %s'%s'" %
(type(egg_name), egg_name))
return CACHE_DIR + "/" + egg_name
def stop(self, reason):
# called by DA server parent to shut down the server
if self.stop_server == True:
log.debug("TcpServer.stop(): already stopping")
return
self.stop_server = True
self.server_ready_to_send = False
# set close reason in case it's not 'parent closed' so server can inform parent via callback
self.close_reason = reason
log.debug("TcpServer.stop(): stopping TCP server - reason = %s", SessionCloseReasons.string(self.close_reason))
def _egg_remotepath(self, egg_name):
log.debug("_egg_remotepath: %s", egg_name)
if not REPO_BASE:
raise ServerError("REPO_BASE is %s'%s'" %
(type(REPO_BASE), REPO_BASE))
elif not egg_name:
raise ServerError("egg_name for remotepath is %s'%s'" %
(type(egg_name), egg_name))
return REPO_BASE + "/" + egg_name
def setUp(self):
"""
Initialize test members.
Start port agent.
Start container and client.
Start streams and subscribers.
Start agent, client.
"""
TrhphTestCase.setUp(self)
# Start port agent, add stop to cleanup.
self._pagent = None
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
self._start_container()
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._no_samples = None
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
# Start event subscribers, add stop to cleanup.
self._no_events = None
self._async_event_result = AsyncResult()
self._events_received = []
self._event_subscribers = []
self._start_event_subscribers()
self.addCleanup(self._stop_event_subscribers)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
self._ia_pid = None
log.debug("TestInstrumentAgentWithTrhph.setup(): starting IA.")
# make sure the driver is stopped
self.addCleanup(self._reset)
def test_13694(self):
# log.setLevel('DEBUG')
source_file_path = os.path.join(RESOURCE_PATH, '20181010-bad.syslog.log')
particle_data_handler = parse(None,
source_file_path=source_file_path,
particle_data_handler=ParticleDataHandler())
log.debug('SAMPLES: %s', particle_data_handler._samples)
log.debug('FAILURE: %s', particle_data_handler._failure)
self.assertEquals(particle_data_handler._failure, True)
self.assertNoParticleRegression(source_file_path, particle_data_handler)
def _write(self, data):
# write data to tcp client
log.debug("TcpServer._write(): data = " + str(data))
if self.connection_socket:
self.activity_seen = True;
MSGLEN = len(data)
total_sent = 0
while total_sent < MSGLEN:
sent = self.connection_socket.send(data[total_sent:])
if sent == 0:
raise RuntimeError("socket connection broken")
total_sent = total_sent + sent
else:
log.warning("TcpServer._write(): no connection yet, can not write data")
def test_base(self):
# log.setLevel('DEBUG')
source_file_path = os.path.join(RESOURCE_PATH, '20140915.syslog.log')
particle_data_handler = parse(None,
source_file_path=source_file_path,
particle_data_handler=ParticleDataHandler())
log.debug("SAMPLES: %s", particle_data_handler._samples)
log.debug("FAILURE: %s", particle_data_handler._failure)
self.assertEquals(particle_data_handler._failure, False)
self.assertNoParticleRegression(source_file_path, particle_data_handler)
def _timer_greenlet(self, session_timeout, inactivity_timeout):
# implements session and inactivity timeouts
# do NOT add parent callback to this greenlet
# ALL callbacks to the parent should be handled by the server greenlet
log.debug("DirectAccessServer._timer_greenlet(): started - sessionTO=%d, inactivityTO=%d"
%(session_timeout, inactivity_timeout))
session_start_time = inactivity_start_time = time.time()
try:
while True:
gevent.sleep(1)
timenow = time.time()
if ((timenow - session_start_time) > session_timeout):
log.debug("DirectAccessServer._timer_greenlet(): session exceeded session timeout of %d seconds"
%session_timeout)
# indicate to the server that it should shut down; the server will inform the parent of the shutdown
self._stop(SessionCloseReasons.session_timeout)
break
if self.server.any_activity():
# activity detected so reset the inactivity start time
inactivity_start_time = time.time()
elif ((timenow - inactivity_start_time) > inactivity_timeout):
log.debug("DirectAccessServer._timer_greenlet(): session exceeded inactivity timeout of %d seconds"
%inactivity_timeout)
# indicate to the server that it should shut down; the server will inform the parent of the shutdown
self._stop(reason=SessionCloseReasons.inactivity_timeout)
break
except:
# to detect a kill() from the DA server
pass
log.debug("DirectAccessServer._timer_greenlet(): stopped ")
def _get_remote_egg(self, egg_name):
"""
pull the egg from a remote server if present to the local cache dir.
@return: returns the path, throws exception if not found.
"""
log.debug("_get_remote_egg %s", egg_name)
try:
if egg_name.startswith("http://"):
response = urlopen(egg_name)
else:
response = urlopen(self._egg_remotepath(egg_name))
egg_yolk = response.read()
log.debug("_fetch_egg GOT YOLK")
except HTTPError, e:
raise DriverLaunchException('failed to download egg: ' + str(e))
def _get_remote_egg(self, egg_name):
"""
pull the egg from a remote server if present to the local cache dir.
@return: returns the path, throws exception if not found.
"""
log.debug("_get_remote_egg %s", egg_name)
try:
if egg_name.startswith("http://"):
response = urlopen(egg_name)
else:
response = urlopen(self._egg_remotepath(egg_name))
egg_yolk = response.read()
log.debug("_fetch_egg GOT YOLK")
except HTTPError, e:
raise DriverLaunchException('failed to download egg: ' + str(e))
def test_base(self):
# log.setLevel('DEBUG')
source_file_path = os.path.join(RESOURCE_PATH, '20140915.syslog.log')
particle_data_handler = parse(
None,
source_file_path=source_file_path,
particle_data_handler=ParticleDataHandler())
log.debug("SAMPLES: %s", particle_data_handler._samples)
log.debug("FAILURE: %s", particle_data_handler._failure)
self.assertEquals(particle_data_handler._failure, False)
self.assertNoParticleRegression(source_file_path,
particle_data_handler)
def test_13694(self):
# log.setLevel('DEBUG')
source_file_path = os.path.join(RESOURCE_PATH,
'20181010-bad.syslog.log')
particle_data_handler = parse(
None,
source_file_path=source_file_path,
particle_data_handler=ParticleDataHandler())
log.debug('SAMPLES: %s', particle_data_handler._samples)
log.debug('FAILURE: %s', particle_data_handler._failure)
self.assertEquals(particle_data_handler._failure, True)
self.assertNoParticleRegression(source_file_path,
particle_data_handler)
def run_command(self, command_line):
log.debug("run command: " + str(command_line))
process = subprocess.Popen(command_line, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
gevent.sleep(1)
process.poll()
# We have failed!
if process.returncode and process.pid:
output, error_message = process.communicate()
log.error("Failed to run command: STDERR: %s", error_message)
raise PortAgentLaunchException("failed to launch port agent")
log.debug("command successful. pid: %d", process.pid)
return process.pid
def _read_pid(self):
pid_file = PID_FILE % (PROCESS_BASE_DIR, self._command_port)
start_time = time.time()
boo = 0
log.debug("read pid file: " + pid_file)
while (start_time + DEFAULT_TIMEOUT > time.time()):
try:
file = open(pid_file)
pid = file.read().strip('\0\n\r')
if (pid):
int(pid)
log.info("port agent pid: [%s]" % (pid))
return int(pid)
except ValueError, e:
log.warn("Failed to convert %s to an int '%s" % (pid, e))
break
except:
def _read_pid(self):
pid_file = PID_FILE % (PROCESS_BASE_DIR, self._command_port)
start_time = time.time()
boo = 0;
log.debug("read pid file: " + pid_file)
while(start_time + DEFAULT_TIMEOUT > time.time()):
try:
file = open(pid_file)
pid = file.read().strip('\0\n\r')
if(pid):
int(pid)
log.info("port agent pid: [%s]" % (pid))
return int(pid)
except ValueError, e:
log.warn("Failed to convert %s to an int '%s" % (pid, e) )
break
except:
def run_command(self, command_line):
log.debug("run command: " + str(command_line))
process = subprocess.Popen(command_line,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
gevent.sleep(1)
process.poll()
# We have failed!
if (process.returncode and process.pid):
output, error_message = process.communicate()
log.error("Failed to run command: STDERR: %s" % (error_message))
raise PortAgentLaunchException("failed to launch port agent")
log.debug("command successful. pid: %d" % (process.pid))
return process.pid
def _get_port_from_file(self, filename):
"""
Read the driver port from a status file. The driver process writes two status files containing the port
number for events and commands.
@param filename path to the port file
@return port port number read from the file
@raise ServerError if file not read w/in 10sec
"""
maxWait=10 # try for up to 10sec
waitInterval=0.5 # repeating every 1/2 sec
log.debug("about to read port from file %s", filename)
for n in xrange(int(maxWait/waitInterval)):
try:
with open(filename, 'r') as f:
port = int(f.read().strip())
log.debug("read port %d from file %s", port, filename)
return port
except: pass
time.sleep(waitInterval)
raise ServerError('process PID file was not found: ' + filename)
def _stop(self, reason):
# can be called by timer greenlet or by parent interface stop() method
if self.already_stopping == True:
log.debug("DirectAccessServer.stop(): already stopping")
return
self.already_stopping = True
log.debug("DirectAccessServer.stop(): stopping DA server - reason = %s (%d)",
SessionCloseReasons.string(reason), reason)
if self.server:
log.debug("DirectAccessServer.stop(): stopping TCP server")
# pass in reason so server can tell parent via callback if necessary
self.server.stop(reason)
del self.server
if self.timer and reason == SessionCloseReasons.parent_closed:
# timer didn't initiate the stop, so kill it
log.debug("DirectAccessServer.stop(): stopping timer")
try:
self.timer.kill()
except Exception as ex:
# can happen if timer already closed session (race condition)
log.debug("DirectAccessServer.stop(): exception caught for timer kill: " + str(ex))
def launch(self):
"""
Launch the driver process. Once the process is launched read the two status files that contain the event port
and command port for the driver.
@raises DriverLaunchException
"""
log.info("Launch driver process")
cmd = self._process_command()
self._driver_process = self._spawn(cmd)
if not self._driver_process and not self.poll():
log.error("Failed to launch driver: %s", cmd)
raise DriverLaunchException('Error starting driver process')
log.debug("driver process started, pid: %s", self.getpid())
self._command_port = self._get_port_from_file(self._driver_command_port_file())
self._event_port = self._get_port_from_file(self._driver_event_port_file())
log.debug("-- command port: %s, event port: %s", self._command_port, self._event_port)
def _get_port_from_file(self, filename):
"""
Read the driver port from a status file. The driver process writes two status files containing the port
number for events and commands.
@param filename path to the port file
@return port port number read from the file
@raise ServerError if file not read w/in 10sec
"""
maxWait = 10 # try for up to 10sec
waitInterval = 0.5 # repeating every 1/2 sec
log.debug("about to read port from file %s", filename)
for n in xrange(int(maxWait / waitInterval)):
try:
with open(filename, 'r') as f:
port = int(f.read().strip())
log.debug("read port %d from file %s", port, filename)
return port
except:
pass
time.sleep(waitInterval)
raise ServerError('process PID file was not found: ' + filename)
def _handler(self):
"The actual telnet server to which the telnet client is connected."
log.debug("TelnetServer._handler(): starting")
username = None
token = None
# prompt user for username; the response is ignored and not used
self._write("Username: "******"token: ")
token = self._readline()
# check token for correctness
if not self._authorized(token):
log.debug("login failed")
self._writeline("login failed")
self._exit_handler(SessionCloseReasons.login_failed)
return
# setup the telnet session with the telnet client
if not self._setup_session():
return
self._writeline("connected") # let telnet client user know they are connected
self.server_ready_to_send = True
while True:
if self.TELNET_PROMPT:
self._write(self.TELNET_PROMPT)
# must use _get_data() method from the TcpServer class to ensure that requested shut downs are detected
input_data = self._get_data()
log.debug("rcvd: " + input_data)
log.debug("len=" + str(len(input_data)))
for i in range(len(input_data)):
log.debug("%d - %x", i, ord(input_data[i]))
# ship the data read from tcp client up to parent to forward on to instrument
self._forward_data_to_parent(input_data)
def __init__(self,
direct_access_type=None,
input_callback=None,
ip_address=None,
session_timeout=None,
inactivity_timeout=None):
log.debug("DirectAccessServer.__init__()")
if not direct_access_type:
log.warning("DirectAccessServer.__init__(): direct access type not specified")
raise ServerError("DirectAccessServer.__init__(): direct access type not specified")
if not input_callback:
log.warning("DirectAccessServer.__init__(): callback not specified")
raise ServerError("DirectAccessServer.__init__(): callback not specified")
if not ip_address:
log.warning("DirectAccessServer.__init__(): IP address not specified")
raise ServerError("DirectAccessServer.__init__(): IP address not specified")
if not session_timeout:
log.warning("DirectAccessServer.__init__(): session timeout not specified")
raise ServerError("DirectAccessServer.__init__(): session timeout not specified")
if not inactivity_timeout:
log.warning("DirectAccessServer.__init__(): inactivity timeout not specified")
raise ServerError("DirectAccessServer.__init__(): inactivity timeout not specified")
# start the correct server based on direct_access_type
if direct_access_type == DirectAccessTypes.telnet:
self.server = TelnetServer(input_callback, ip_address)
elif direct_access_type == DirectAccessTypes.vsp:
self.server = SerialServer(input_callback, ip_address)
else:
raise ServerError("DirectAccessServer.__init__(): Unsupported direct access type")
log.debug("DirectAccessServer.__init__(): starting timer greenlet")
self.timer = gevent.spawn(self._timer_greenlet,
session_timeout=session_timeout,
inactivity_timeout=inactivity_timeout)
def _launch(self):
"""
@brief Launch the port agent process. If the address isn't localhost
then we don't start anything
@retval return the command port the process is listening on.
"""
log.info("Startup Unix Port Agent")
# Create port agent object.
this_pid = os.getpid() if self._test_mode else None
log.debug(" -- our pid: %s" % this_pid)
log.debug(" -- command port: %s" % self._command_port)
log.debug(" -- address: %s, port: %s" % (self._device_addr, self._device_port))
command_line = [self._binary_path, self._type, self._data_port, self._command_port, self._device_addr]
if self._type == PortAgentType.ETHERNET:
command_line.extend([self._device_port])
elif self._type == PortAgentType.RSN:
command_line.extend([self._device_port, self._device_cmd_port])
elif self._type == PortAgentType.BOTPT:
command_line.extend([self._device_rx_port, self._device_tx_port])
if self._sniffer_port:
command_line.append('--sniff=%d' % self._sniffer_port)
command_line = [str(arg) for arg in command_line]
self._pid = self.run_command(command_line)
return self._command_port
def assertNoParticleRegression(self, filepath, data_handler):
"""
Compares particles with previous run. If no YAML file exists, creates one.
:param filepath: fully qualified name of the input file (that was parsed)
:param data_handler: ParticleDataHandler returned from parse()
:return:
"""
yaml_file = os.path.splitext(filepath)[0] + '.yml'
particles = data_handler._samples
if os.path.isfile(yaml_file):
with open(yaml_file, 'r') as stream:
prev_particles = yaml.load(stream)
# particle key names should match
self.assertListEqual(sorted(prev_particles.keys()), sorted(particles.keys()))
# compare number of samples across one of the particle keys
for p in prev_particles.keys():
log.debug('%s: %d %d', p, len(prev_particles[p]), len(particles[p]))
self.assertEqual(len(prev_particles[p]), len(particles[p]))
else:
with open(yaml_file, 'w') as stream:
log.warn('creating yaml output file for regression testing - commit %s', yaml_file)
yaml.dump(particles, stream, default_flow_style=False)
def _server_greenlet(self):
# the main DA server thread that listens for a tcp client connection and passes it to the
# handler for processing. It expects the handler to raise an exception when it needs to quit,
# but it will work correctly if it simply returns when done
log.debug("TcpServer._server_greenlet(): started")
# set socket to timeout on blocking calls so accept() method will timeout to let server
# check the stop_server flag
self.server_socket.settimeout(1.0)
self.server_socket.listen(1)
while True:
try:
self.connection_socket, address = self.server_socket.accept()
log.info("TcpServer._server_greenlet(): connection accepted from <%s>" %str(address))
# set socket to non-blocking so recv() will not block to let server
# check the stop_server flag
self.connection_socket.setblocking(0)
break
except Exception as ex:
log.debug("TcpServer._server_greenlet(): exception caught while listening for connection <%s>" %str(ex))
if self.server_socket.timeout:
# check to see if server shut down is requested
if self.stop_server:
self._notify_parent()
return
else:
# something is wrong with the tcp socket so shut down
log.info("TcpServer._server_greenlet(): exception caught while listening for connection <%s>" %str(ex))
self._indicate_server_stopping(SessionCloseReasons.socket_error)
self._notify_parent()
return
# got a client connection so call handler to process it
try:
self._handler()
except Exception as ex:
log.info("TcpServer._server_greenlet(): exception caught from handler <%s>" %str(ex))
finally:
# handler has exited so indicate to parent the server is shutting down
self._notify_parent()
def launch(self):
"""
Launch the driver process. Once the process is launched read the two status files that contain the event port
and command port for the driver.
@raises DriverLaunchException
"""
log.info("Launch driver process")
cmd = self._process_command()
self._driver_process = self._spawn(cmd)
if not self._driver_process and not self.poll():
log.error("Failed to launch driver: %s", cmd)
raise DriverLaunchException('Error starting driver process')
log.debug("driver process started, pid: %s", self.getpid())
self._command_port = self._get_port_from_file(
self._driver_command_port_file())
self._event_port = self._get_port_from_file(
self._driver_event_port_file())
log.debug("-- command port: %s, event port: %s", self._command_port,
self._event_port)
def launch(self):
"""
@brief Launch the driver process and driver client. This is used in the
integration and qualification tests. The port agent abstracts the physical
interface with the instrument.
@retval return the pid to the logger process
"""
log.info("Startup Port Agent")
# Create port agent object.
this_pid = os.getpid() if self._test_mode else None
log.debug( " -- our pid: %s" % this_pid)
log.debug( " -- address: %s, port: %s" % (self._device_addr, self._device_port))
# Working dir and delim are hard coded here because this launch process
# will change with the new port agent.
self.port_agent = EthernetDeviceLogger.launch_process(
self._device_addr,
self._device_port,
self._working_dir,
self._delimiter,
this_pid)
log.debug( " Port agent object created" )
start_time = time.time()
expire_time = start_time + int(self._timeout)
pid = self.port_agent.get_pid()
while not pid:
gevent.sleep(.1)
pid = self.port_agent.get_pid()
if time.time() > expire_time:
log.error("!!!! Failed to start Port Agent !!!!")
raise PortAgentTimeout('port agent could not be started')
self._pid = pid
port = self.port_agent.get_port()
start_time = time.time()
expire_time = start_time + int(self._timeout)
while not port:
gevent.sleep(.1)
port = self.port_agent.get_port()
if time.time() > expire_time:
log.error("!!!! Port Agent could not bind to port !!!!")
self.stop()
raise PortAgentTimeout('port agent could not bind to port')
self._data_port = port
log.info('Started port agent pid %s listening at port %s' % (pid, port))
return port
def _check_cache_for_egg(self, egg_name):
"""
Check if the egg is already cached, if so, return the path.
@return: egg path if cached, else None
"""
log.debug("_check_cache_for_egg: %s", egg_name)
path = self._egg_path(egg_name)
if os.path.exists(path):
log.debug("_check_cache_for_egg cache hit PATH = " + str(path))
return path
else:
log.debug("_check_cache_for_egg cache miss" + str(path))
return None
def _handler(self):
"The actual virtual serial port server to which the com0com client is connected."
log.debug("SerialServer._handler(): starting")
self.server_ready_to_send = True
while True:
# must use _get_data() method from the TcpServer class to ensure that requested shut downs are detected
input_data = self._get_data()
log.debug("SerialServer._handler: rcvd: [" + input_data + "]\nlen=" + str(len(input_data)))
for i in range(len(input_data)):
log.debug("SerialServer._handler: char @ %d = %x", i, ord(input_data[i]))
# ship the data read from virtual serial port client up to parent to forward on to instrument
self._forward_data_to_parent(input_data)
def launch(self):
"""
@brief Launch the driver process and driver client. This is used in the
integration and qualification tests. The port agent abstracts the physical
interface with the instrument.
@retval return the pid to the logger process
"""
log.info("Startup Port Agent")
# Create port agent object.
this_pid = os.getpid() if self._test_mode else None
log.debug(" -- our pid: %s" % this_pid)
log.debug(" -- address: %s, port: %s" %
(self._device_addr, self._device_port))
# Working dir and delim are hard coded here because this launch process
# will change with the new port agent.
self.port_agent = EthernetDeviceLogger.launch_process(
self._device_addr, self._device_port, self._working_dir,
self._delimiter, this_pid)
log.debug(" Port agent object created")
start_time = time.time()
expire_time = start_time + int(self._timeout)
pid = self.port_agent.get_pid()
while not pid:
gevent.sleep(.1)
pid = self.port_agent.get_pid()
if time.time() > expire_time:
log.error("!!!! Failed to start Port Agent !!!!")
raise PortAgentTimeout('port agent could not be started')
self._pid = pid
port = self.port_agent.get_port()
start_time = time.time()
expire_time = start_time + int(self._timeout)
while not port:
gevent.sleep(.1)
port = self.port_agent.get_port()
if time.time() > expire_time:
log.error("!!!! Port Agent could not bind to port !!!!")
self.stop()
raise PortAgentTimeout('port agent could not bind to port')
self._data_port = port
log.info('Started port agent pid %s listening at port %s' %
(pid, port))
return port
