def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self,
mock_get_python_lib):
self.assertFalse(is_pack_virtualenv_using_python3(pack='dummy_pack')[0])
# No inheritance
python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack',
inherit_from_parent=False,
inherit_parent_virtualenv=False)
self.assertEqual(python_path, ':')
# Inherit python path from current process
# Mock the current process python path
os.environ['PYTHONPATH'] = ':/data/test1:/data/test2'
python_path = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=False)
self.assertEqual(python_path, ':/data/test1:/data/test2')
# Inherit from current process and from virtualenv (not running inside virtualenv)
del sys.real_prefix
python_path = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=False)
self.assertEqual(python_path, ':/data/test1:/data/test2')
# Inherit from current process and from virtualenv (running inside virtualenv)
sys.real_prefix = '/usr'
mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest'
python_path = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=True)
self.assertEqual(python_path, ':/data/test1:/data/test2:%s/virtualenvtest' %
(sys.prefix))
def test_get_sandbox_python_path_for_python_action_python2_used_for_venv(self,
mock_get_python_lib):
self.assertFalse(is_pack_virtualenv_using_python3(pack='dummy_pack')[0])
# No inheritance
python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack',
inherit_from_parent=False,
inherit_parent_virtualenv=False)
self.assertEqual(python_path, ':')
# Inherit python path from current process
# Mock the current process python path
os.environ['PYTHONPATH'] = ':/data/test1:/data/test2'
python_path = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=False)
self.assertEqual(python_path, ':/data/test1:/data/test2')
# Inherit from current process and from virtualenv (not running inside virtualenv)
del sys.real_prefix
python_path = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=False)
self.assertEqual(python_path, ':/data/test1:/data/test2')
# Inherit from current process and from virtualenv (running inside virtualenv)
sys.real_prefix = '/usr'
mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest'
python_path = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=True)
self.assertEqual(python_path, ':/data/test1:/data/test2:%s/virtualenvtest' %
(sys.prefix))
def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(
self, mock_get_python_lib):
self.assertTrue(is_pack_virtualenv_using_python3(pack='dummy_pack')[0])
# No inheritance
python_path = get_sandbox_python_path_for_python_action(
pack='dummy_pack',
inherit_from_parent=False,
inherit_parent_virtualenv=False)
split = python_path.strip(':').split(':')
self.assertEqual(len(split), 3)
# First entry should be lib/python3 dir from venv
self.assertTrue('virtualenvs/dummy_pack/lib/python3.6' in split[0])
# Second entry should be python3 site-packages dir from venv
self.assertTrue(
'virtualenvs/dummy_pack/lib/python3.6/site-packages' in split[1])
# Third entry should be actions/lib dir from pack root directory
self.assertTrue('packs/dummy_pack/actions/lib/' in split[2])
# Inherit python path from current process
# Mock the current process python path
os.environ['PYTHONPATH'] = ':/data/test1:/data/test2'
python_path = get_sandbox_python_path_for_python_action(
pack='dummy_pack',
inherit_from_parent=True,
inherit_parent_virtualenv=False)
expected = (
'/tmp/virtualenvs/dummy_pack/lib/python3.6:'
'/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:'
'/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2')
self.assertEqual(python_path, expected)
# Inherit from current process and from virtualenv (not running inside virtualenv)
del sys.real_prefix
python_path = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=False)
self.assertEqual(python_path, ':/data/test1:/data/test2')
# Inherit from current process and from virtualenv (running inside virtualenv)
sys.real_prefix = '/usr'
mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest'
python_path = get_sandbox_python_path_for_python_action(
pack='dummy_pack',
inherit_from_parent=True,
inherit_parent_virtualenv=True)
expected = (
'/tmp/virtualenvs/dummy_pack/lib/python3.6:'
'/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:'
'/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2:'
'%s/virtualenvtest' % (sys.prefix))
self.assertEqual(python_path, expected)
def test_get_sandbox_python_path_for_python_action_python3_used_for_venv(self,
mock_get_python_lib):
self.assertTrue(is_pack_virtualenv_using_python3(pack='dummy_pack')[0])
# No inheritance
python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack',
inherit_from_parent=False,
inherit_parent_virtualenv=False)
split = python_path.strip(':').split(':')
self.assertEqual(len(split), 3)
# First entry should be lib/python3 dir from venv
self.assertTrue('virtualenvs/dummy_pack/lib/python3.6' in split[0])
# Second entry should be python3 site-packages dir from venv
self.assertTrue('virtualenvs/dummy_pack/lib/python3.6/site-packages' in split[1])
# Third entry should be actions/lib dir from pack root directory
self.assertTrue('packs/dummy_pack/actions/lib/' in split[2])
# Inherit python path from current process
# Mock the current process python path
os.environ['PYTHONPATH'] = ':/data/test1:/data/test2'
python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack',
inherit_from_parent=True,
inherit_parent_virtualenv=False)
expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6:'
'/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:'
'/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2')
self.assertEqual(python_path, expected)
# Inherit from current process and from virtualenv (not running inside virtualenv)
del sys.real_prefix
python_path = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=False)
self.assertEqual(python_path, ':/data/test1:/data/test2')
# Inherit from current process and from virtualenv (running inside virtualenv)
sys.real_prefix = '/usr'
mock_get_python_lib.return_value = sys.prefix + '/virtualenvtest'
python_path = get_sandbox_python_path_for_python_action(pack='dummy_pack',
inherit_from_parent=True,
inherit_parent_virtualenv=True)
expected = ('/tmp/virtualenvs/dummy_pack/lib/python3.6:'
'/tmp/virtualenvs/dummy_pack/lib/python3.6/site-packages:'
'/tmp/packs/dummy_pack/actions/lib/::/data/test1:/data/test2:'
'%s/virtualenvtest' % (sys.prefix))
self.assertEqual(python_path, expected)
def _spawn_sensor_process(self, sensor):
"""
Spawn a new process for the provided sensor.
New process uses isolated Python binary from a virtual environment
belonging to the sensor pack.
"""
sensor_id = self._get_sensor_id(sensor=sensor)
pack_ref = sensor['pack']
virtualenv_path = get_sandbox_virtualenv_path(pack=pack_ref)
python_path = get_sandbox_python_binary_path(pack=pack_ref)
if virtualenv_path and not os.path.isdir(virtualenv_path):
format_values = {
'pack': sensor['pack'],
'virtualenv_path': virtualenv_path
}
msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
raise Exception(msg)
# NOTE: Running sensors using Python 3 virtual environments is not supported
uses_python3, _ = is_pack_virtualenv_using_python3(pack=sensor['pack'])
if uses_python3 and not six.PY3:
format_values = {
'pack': sensor['pack'],
'virtualenv_path': virtualenv_path
}
msg = PACK_VIRTUALENV_USES_PYTHON3 % format_values
raise Exception(msg)
args = self._get_args_for_wrapper_script(python_binary=python_path,
sensor=sensor)
if self._enable_common_pack_libs:
pack_common_libs_path = get_pack_common_libs_path_for_pack_ref(
pack_ref=pack_ref)
else:
pack_common_libs_path = None
env = os.environ.copy()
sandbox_python_path = get_sandbox_python_path(
inherit_from_parent=True, inherit_parent_virtualenv=True)
if self._enable_common_pack_libs and pack_common_libs_path:
env['PYTHONPATH'] = pack_common_libs_path + ':' + sandbox_python_path
else:
env['PYTHONPATH'] = sandbox_python_path
if self._create_token:
# Include full api URL and API token specific to that sensor
LOG.debug('Creating temporary auth token for sensor %s' %
(sensor['class_name']))
ttl = cfg.CONF.auth.service_token_ttl
metadata = {
'service': 'sensors_container',
'sensor_path': sensor['file_path'],
'sensor_class': sensor['class_name']
}
temporary_token = create_token(username='******',
ttl=ttl,
metadata=metadata,
service=True)
env[API_URL_ENV_VARIABLE_NAME] = get_full_public_api_url()
env[AUTH_TOKEN_ENV_VARIABLE_NAME] = temporary_token.token
# TODO 1: Purge temporary token when service stops or sensor process dies
# TODO 2: Store metadata (wrapper process id) with the token and delete
# tokens for old, dead processes on startup
cmd = ' '.join(args)
LOG.debug('Running sensor subprocess (cmd="%s")', cmd)
# TODO: Intercept stdout and stderr for aggregated logging purposes
try:
process = subprocess.Popen(args=args,
stdin=None,
stdout=None,
stderr=None,
shell=False,
env=env,
preexec_fn=on_parent_exit('SIGTERM'))
except Exception as e:
cmd = ' '.join(args)
message = ('Failed to spawn process for sensor %s ("%s"): %s' %
(sensor_id, cmd, six.text_type(e)))
raise Exception(message)
self._processes[sensor_id] = process
self._sensors[sensor_id] = sensor
self._sensor_start_times[sensor_id] = int(time.time())
self._dispatch_trigger_for_sensor_spawn(sensor=sensor,
process=process,
cmd=cmd)
return process
def _spawn_sensor_process(self, sensor):
"""
Spawn a new process for the provided sensor.
New process uses isolated Python binary from a virtual environment
belonging to the sensor pack.
"""
sensor_id = self._get_sensor_id(sensor=sensor)
pack_ref = sensor['pack']
virtualenv_path = get_sandbox_virtualenv_path(pack=pack_ref)
python_path = get_sandbox_python_binary_path(pack=pack_ref)
if virtualenv_path and not os.path.isdir(virtualenv_path):
format_values = {'pack': sensor['pack'], 'virtualenv_path': virtualenv_path}
msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
raise Exception(msg)
# NOTE: Running sensors using Python 3 virtual environments is not supported
uses_python3, _ = is_pack_virtualenv_using_python3(pack=sensor['pack'])
if uses_python3 and not six.PY3:
format_values = {'pack': sensor['pack'], 'virtualenv_path': virtualenv_path}
msg = PACK_VIRTUALENV_USES_PYTHON3 % format_values
raise Exception(msg)
trigger_type_refs = sensor['trigger_types'] or []
trigger_type_refs = ','.join(trigger_type_refs)
parent_args = json.dumps(sys.argv[1:])
args = [
python_path,
WRAPPER_SCRIPT_PATH,
'--pack=%s' % (sensor['pack']),
'--file-path=%s' % (sensor['file_path']),
'--class-name=%s' % (sensor['class_name']),
'--trigger-type-refs=%s' % (trigger_type_refs),
'--parent-args=%s' % (parent_args)
]
if sensor['poll_interval']:
args.append('--poll-interval=%s' % (sensor['poll_interval']))
sandbox_python_path = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=True)
if self._enable_common_pack_libs:
pack_common_libs_path = get_pack_common_libs_path_for_pack_ref(pack_ref=pack_ref)
else:
pack_common_libs_path = None
env = os.environ.copy()
if self._enable_common_pack_libs and pack_common_libs_path:
env['PYTHONPATH'] = pack_common_libs_path + ':' + sandbox_python_path
else:
env['PYTHONPATH'] = sandbox_python_path
# Include full api URL and API token specific to that sensor
ttl = cfg.CONF.auth.service_token_ttl
metadata = {
'service': 'sensors_container',
'sensor_path': sensor['file_path'],
'sensor_class': sensor['class_name']
}
temporary_token = create_token(username='******', ttl=ttl, metadata=metadata,
service=True)
env[API_URL_ENV_VARIABLE_NAME] = get_full_public_api_url()
env[AUTH_TOKEN_ENV_VARIABLE_NAME] = temporary_token.token
# TODO 1: Purge temporary token when service stops or sensor process dies
# TODO 2: Store metadata (wrapper process id) with the token and delete
# tokens for old, dead processes on startup
cmd = ' '.join(args)
LOG.debug('Running sensor subprocess (cmd="%s")', cmd)
# TODO: Intercept stdout and stderr for aggregated logging purposes
try:
process = subprocess.Popen(args=args, stdin=None, stdout=None,
stderr=None, shell=False, env=env,
preexec_fn=on_parent_exit('SIGTERM'))
except Exception as e:
cmd = ' '.join(args)
message = ('Failed to spawn process for sensor %s ("%s"): %s' %
(sensor_id, cmd, six.text_type(e)))
raise Exception(message)
self._processes[sensor_id] = process
self._sensors[sensor_id] = sensor
self._sensor_start_times[sensor_id] = int(time.time())
self._dispatch_trigger_for_sensor_spawn(sensor=sensor, process=process, cmd=cmd)
return process
