def create_and_start_executor(self, config, parameter_values=None):
if parameter_values is None:
parameter_values = {}
self.executor = ScriptExecutor(config, parameter_values)
self.executor.start()
return self.executor
def start_script(self, config, values, user_id, all_audit_names):
audit_name = audit_utils.get_audit_name(all_audit_names)
executor = ScriptExecutor(config, values)
execution_id = self._id_generator.next_id()
audit_command = executor.get_secure_command()
LOGGER.info('Calling script #%s: %s', execution_id, audit_command)
executor.start()
self._executors[execution_id] = executor
self._execution_infos[execution_id] = _ExecutionInfo(
execution_id=execution_id,
owner=user_id,
audit_name=audit_name,
audit_command=audit_command,
config=config,
all_audit_names=all_audit_names)
self._active_executor_ids.add(execution_id)
self._add_post_finish_handling(execution_id, executor)
self._fire_execution_started(execution_id)
return execution_id
def build_command_args(self, param_values, config):
if config.script_command is None:
config.script_command = 'ping'
script_executor = ScriptExecutor(config, param_values)
args_string = executor.build_command_args(script_executor.get_script_parameter_values(), config)
return args_string
def start_script(self, config, values, user: User):
audit_name = user.get_audit_name()
config.set_all_param_values(values)
normalized_values = dict(config.parameter_values)
executor = ScriptExecutor(config, normalized_values)
execution_id = self._id_generator.next_id()
audit_command = executor.get_secure_command()
LOGGER.info('Calling script #%s: %s', execution_id, audit_command)
executor.start()
self._executors[execution_id] = executor
self._execution_infos[execution_id] = _ExecutionInfo(
execution_id=execution_id,
owner_user=user,
audit_name=audit_name,
audit_command=audit_command,
config=config)
self._active_executor_ids.add(execution_id)
self._add_post_finish_handling(execution_id, executor, user)
self._fire_execution_started(execution_id, user)
return execution_id
def start_script(self, config, values, user_id, all_audit_names):
audit_name = audit_utils.get_audit_name(all_audit_names)
executor = ScriptExecutor(config, values)
execution_id = self._id_generator.next_id()
audit_command = executor.get_secure_command()
LOGGER.info('Calling script #%s: %s', execution_id, audit_command)
executor.start()
self._executors[execution_id] = executor
self._execution_infos[execution_id] = _ExecutionInfo(
execution_id=execution_id,
owner=user_id,
audit_name=audit_name,
audit_command=audit_command,
config=config,
all_audit_names=all_audit_names)
self._active_executor_ids.add(execution_id)
self._add_post_finish_handling(execution_id, executor)
self._fire_execution_started(execution_id)
return execution_id
def create_and_start_executor(self, parameter_values=None):
if parameter_values is None:
parameter_values = {}
self.executor = ScriptExecutor(self.config, parameter_values,
'executor_test')
self.executor.start(_MockProcessWrapper)
return self.executor
def test_parameter_secure_some_value(self):
parameter = create_script_param_config('p1', param='-p1', secure=True)
config = create_config_model('config_x',
config={'script_path': 'ls'},
parameters=[parameter])
executor = ScriptExecutor(config, {'p1': 'value'})
secure_command = executor.get_secure_command()
self.assertEqual('ls -p1 ******', secure_command)
def test_parameter_secure_value_and_same_unsecure(self):
p1 = create_script_param_config('p1', param='-p1', secure=True)
p2 = create_script_param_config('p2', param='-p2')
config = create_config_model('config_x',
config={'script_path': 'ls'},
parameters=[p1, p2])
executor = ScriptExecutor(config, {'p1': 'value', 'p2': 'value'})
secure_command = executor.get_secure_command()
self.assertEqual('ls -p1 ****** -p2 value', secure_command)
def subscribe_fail_alerter(script_name, alerts_config, audit_name,
executor: ScriptExecutor,
output_stream: Observable):
class Alerter(object):
def finished(self):
return_code = executor.get_return_code()
if return_code != 0:
script = str(script_name)
title = script + ' FAILED'
body = 'The script "' + script + '", started by ' + audit_name + \
' exited with return code ' + str(return_code) + '.' + \
' Usually this means an error, occurred during the execution.' + \
' Please check the corresponding logs'
output_stream.wait_close()
script_output = ''.join(output_stream.get_old_data())
send_alerts(alerts_config, title, body, script_output)
executor.add_finish_listener(Alerter())
def test_parameter_secure_multiselect(self):
parameter = create_script_param_config('p1',
param='-p1',
secure=True,
type=PARAM_TYPE_MULTISELECT)
config = create_config_model('config_x',
config={'script_path': 'ls'},
parameters=[parameter])
executor = ScriptExecutor(config, {'p1': ['one', 'two', 'three']})
secure_command = executor.get_secure_command()
self.assertEqual('ls -p1 ******', secure_command)
def test_parameter_secure_some_value(self):
config = script_configs.Config()
config.script_command = 'ls'
parameter = script_configs.Parameter()
parameter.name = 'p1'
parameter.param = '-p1'
parameter.secure = True
config.add_parameter(parameter)
executor = ScriptExecutor(config, {'p1': 'value'})
secure_command = executor.get_secure_command()
self.assertEqual('ls -p1 ******', secure_command)
def test_parameter_secure_multiselect_as_multiarg(self):
parameter = create_script_param_config('p1',
param='-p1',
secure=True,
type='multiselect',
multiple_arguments=True)
config = create_config_model('config_x',
config={'script_path': 'ls'},
parameters=[parameter])
executor = ScriptExecutor(config, {'p1': ['one', 'two', 'three']})
secure_command = executor.get_secure_command()
self.assertEqual('ls -p1 ****** ****** ******', secure_command)
def test_parameter_secure_multiselect(self):
config = script_configs.Config()
config.script_command = 'ls'
parameter = script_configs.Parameter()
parameter.name = 'p1'
parameter.param = '-p1'
parameter.secure = True
parameter.type = 'multiselect'
config.add_parameter(parameter)
executor = ScriptExecutor(config, {'p1': ['one', 'two', 'three']})
secure_command = executor.get_secure_command()
self.assertEqual('ls -p1 ******,******,******', secure_command)
def test_parameter_secure_value_and_same_unsecure(self):
config = script_configs.Config()
config.script_command = 'ls'
parameter = script_configs.Parameter()
parameter.name = 'p1'
parameter.param = '-p1'
parameter.secure = True
config.add_parameter(parameter)
parameter = script_configs.Parameter()
parameter.name = 'p2'
parameter.param = '-p2'
config.add_parameter(parameter)
executor = ScriptExecutor(config, {
'p1': 'value',
'p2': 'value'
}, 'executor_test')
secure_command = executor.get_secure_command()
self.assertEqual('ls -p1 ****** -p2 value', secure_command)
def start_script(self, config, values, audit_name):
script_name = config.name
executor = ScriptExecutor(config, values, audit_name)
execution_id = self._id_generator.next_id()
audit_command = executor.get_secure_command()
LOGGER.info('Calling script #%s: %s', execution_id, audit_command)
executor.start()
self._running_scripts[execution_id] = executor
secure_output_stream = executor.get_secure_output_stream()
self._execution_logging_service.start_logging(execution_id, audit_name,
script_name,
audit_command,
secure_output_stream,
self)
self.subscribe_fail_alerter(script_name, self._alerts_service,
audit_name, executor, secure_output_stream)
return execution_id
class TestProcessOutput(unittest.TestCase):
def test_log_raw_single_line(self):
config = self._create_config()
self.create_and_start_executor(config)
observer = _StoringObserver()
self.executor.get_raw_output_stream().subscribe(observer)
self.write_process_output('some text')
wait_buffer_flush()
self.assertEqual(['some text'], observer.data)
def test_log_raw_single_buffer(self):
config = self._create_config()
self.create_and_start_executor(config)
observer = _StoringObserver()
self.executor.get_raw_output_stream().subscribe(observer)
self.write_process_output('some text')
self.write_process_output(' and continuation')
wait_buffer_flush()
self.assertEqual(['some text and continuation'], observer.data)
def test_log_raw_multiple_buffers(self):
config = self._create_config()
self.create_and_start_executor(config)
observer = _StoringObserver()
self.executor.get_raw_output_stream().subscribe(observer)
self.write_process_output('some text')
wait_buffer_flush()
self.write_process_output(' and continuation')
wait_buffer_flush()
self.assertEqual(['some text', ' and continuation'], observer.data)
def test_log_with_secure(self):
parameter = create_script_param_config('p1', secure=True)
config = self._create_config(parameters=[parameter])
self.create_and_start_executor(config, {'p1': 'a'})
self.write_process_output('a| some text')
self.write_process_output('\nand a new line')
self.write_process_output(' with some long long text |a')
self.finish_process()
output = self.get_finish_output()
self.assertEqual(output, '******| some text\nand ****** new line with some long long text |******')
def test_log_with_secure_ignore_whitespaces(self):
parameter = create_script_param_config('p1', secure=True)
config = self._create_config(parameters=[parameter])
self.create_and_start_executor(config, {'p1': ' '})
self.write_process_output('some text')
self.write_process_output('\nand a new line')
self.write_process_output(' with some long long text')
self.finish_process()
output = self.get_finish_output()
self.assertEqual(output, 'some text\nand a new line with some long long text')
def test_log_with_secure_ignore_inside_word(self):
parameter = create_script_param_config('p1', secure=True)
config = self._create_config(parameters=[parameter])
self.create_and_start_executor(config, {'p1': 'cat'})
self.write_process_output('cat\n-cat-\nbobcat\ncatty\n1cat\nmy cat is cute')
self.finish_process()
output = self.get_finish_output()
self.assertEqual(output, '******\n-******-\nbobcat\ncatty\n1cat\nmy ****** is cute')
def test_log_with_secure_when_multiselect(self):
parameter = create_script_param_config('p1', secure=True, type=PARAM_TYPE_MULTISELECT)
config = self._create_config(parameters=[parameter])
self.create_and_start_executor(config, {'p1': ['123', 'password']})
self.write_process_output('some text(123)')
self.write_process_output('\nand a new line')
self.write_process_output(' with my password')
self.finish_process()
output = self.get_finish_output()
self.assertEqual(output, 'some text(******)\nand a new line with my ******')
def test_log_with_secure_when_with_symbols(self):
parameter = create_script_param_config('p1', secure=True)
config = self._create_config(parameters=[parameter])
value = '/some.text?#&^and_=+chars\\'
self.create_and_start_executor(config, {'p1': value})
self.write_process_output('Writing ' + value + '\n')
self.write_process_output('...\n')
self.write_process_output(value + '-')
self.write_process_output('\nDone')
self.finish_process()
output = self.get_finish_output()
self.assertEqual(output, 'Writing ******\n...\n******-\nDone')
@staticmethod
def _create_config(parameters=None):
return create_config_model('config_x', parameters=parameters)
def setUp(self):
self.config = create_config_model('config_x')
self.config.script_command = 'ls'
executor._process_creator = _MockProcessWrapper
test_utils.setup()
super().setUp()
def tearDown(self):
super().tearDown()
test_utils.cleanup()
self.finish_process()
self.executor.cleanup()
def write_process_output(self, text):
wrapper = self.executor.process_wrapper
wrapper._write_script_output(text)
# noinspection PyUnresolvedReferences
def finish_process(self):
self.executor.process_wrapper.kill()
def get_finish_output(self):
data = read_until_closed(self.executor.get_anonymized_output_stream(), timeout=BUFFER_FLUSH_WAIT_TIME)
output = ''.join(data)
return output
def create_and_start_executor(self, config, parameter_values=None):
if parameter_values is None:
parameter_values = {}
self.executor = ScriptExecutor(config, parameter_values)
self.executor.start()
return self.executor
def create_executor(self, config, parameter_values):
self.executor = ScriptExecutor(config, parameter_values)
class TestProcessOutput(unittest.TestCase):
def test_log_unsecure_single_line(self):
self.create_and_start_executor()
observer = SimpleStoringObserver()
self.executor.get_unsecure_output_stream().subscribe(observer)
self.write_process_output('some text')
wait_buffer_flush()
self.assertEqual(['some text'], observer.data)
def test_log_unsecure_single_buffer(self):
self.create_and_start_executor()
observer = SimpleStoringObserver()
self.executor.get_unsecure_output_stream().subscribe(observer)
self.write_process_output('some text')
self.write_process_output(' and continuation')
wait_buffer_flush()
self.assertEqual(['some text and continuation'], observer.data)
def test_log_unsecure_multiple_buffers(self):
self.create_and_start_executor()
observer = SimpleStoringObserver()
self.executor.get_unsecure_output_stream().subscribe(observer)
self.write_process_output('some text')
wait_buffer_flush()
self.write_process_output(' and continuation')
wait_buffer_flush()
self.assertEqual(['some text', ' and continuation'], observer.data)
def test_log_with_secure(self):
parameter = script_configs.Parameter()
parameter.name = 'p1'
parameter.secure = True
self.config.add_parameter(parameter)
self.create_and_start_executor({'p1': 'a'})
self.write_process_output('a| some text')
self.write_process_output('\nand a new line')
self.write_process_output(' with some long long text |a')
self.finish_process()
output = self.get_finish_output()
self.assertEqual(
output,
'******| some text\nand ****** new line with some long long text |******'
)
def test_log_with_secure_ignore_whitespaces(self):
parameter = script_configs.Parameter()
parameter.name = 'p1'
parameter.secure = True
self.config.add_parameter(parameter)
self.create_and_start_executor({'p1': ' '})
self.write_process_output('some text')
self.write_process_output('\nand a new line')
self.write_process_output(' with some long long text')
self.finish_process()
output = self.get_finish_output()
self.assertEqual(output,
'some text\nand a new line with some long long text')
def setUp(self):
self.config = script_configs.Config()
self.config.script_command = 'ls'
super().setUp()
def tearDown(self):
super().tearDown()
self.finish_process()
self.executor.kill()
def write_process_output(self, text):
wrapper = self.executor.process_wrapper
wrapper._write_script_output(text)
# noinspection PyUnresolvedReferences
def finish_process(self):
self.executor.process_wrapper.set_finished()
def get_finish_output(self):
output_stream = self.executor.get_secure_output_stream()
output_stream.wait_close()
output = ''.join(output_stream.get_old_data())
return output
def create_and_start_executor(self, parameter_values=None):
if parameter_values is None:
parameter_values = {}
self.executor = ScriptExecutor(self.config, parameter_values,
'executor_test')
self.executor.start(_MockProcessWrapper)
return self.executor
class TestProcessOutput(unittest.TestCase):
def test_log_raw_single_line(self):
self.create_and_start_executor()
observer = _StoringObserver()
self.executor.get_raw_output_stream().subscribe(observer)
self.write_process_output('some text')
wait_buffer_flush()
self.assertEqual(['some text'], observer.data)
def test_log_raw_single_buffer(self):
self.create_and_start_executor()
observer = _StoringObserver()
self.executor.get_raw_output_stream().subscribe(observer)
self.write_process_output('some text')
self.write_process_output(' and continuation')
wait_buffer_flush()
self.assertEqual(['some text and continuation'], observer.data)
def test_log_raw_multiple_buffers(self):
self.create_and_start_executor()
observer = _StoringObserver()
self.executor.get_raw_output_stream().subscribe(observer)
self.write_process_output('some text')
wait_buffer_flush()
self.write_process_output(' and continuation')
wait_buffer_flush()
self.assertEqual(['some text', ' and continuation'], observer.data)
def test_log_with_secure(self):
parameter = script_configs.Parameter()
parameter.name = 'p1'
parameter.secure = True
self.config.add_parameter(parameter)
self.create_and_start_executor({'p1': 'a'})
self.write_process_output('a| some text')
self.write_process_output('\nand a new line')
self.write_process_output(' with some long long text |a')
self.finish_process()
output = self.get_finish_output()
self.assertEqual(
output,
'******| some text\nand ****** new line with some long long text |******'
)
def test_log_with_secure_ignore_whitespaces(self):
parameter = script_configs.Parameter()
parameter.name = 'p1'
parameter.secure = True
self.config.add_parameter(parameter)
self.create_and_start_executor({'p1': ' '})
self.write_process_output('some text')
self.write_process_output('\nand a new line')
self.write_process_output(' with some long long text')
self.finish_process()
output = self.get_finish_output()
self.assertEqual(output,
'some text\nand a new line with some long long text')
def test_log_with_secure_when_multiselect(self):
parameter = script_configs.Parameter()
parameter.name = 'p1'
parameter.secure = True
parameter.type = 'multiselect'
self.config.add_parameter(parameter)
self.create_and_start_executor({'p1': ['123', 'password']})
self.write_process_output('some text(123)')
self.write_process_output('\nand a new line')
self.write_process_output(' with my password')
self.finish_process()
output = self.get_finish_output()
self.assertEqual(output,
'some text(******)\nand a new line with my ******')
def setUp(self):
self.config = script_configs.Config()
self.config.script_command = 'ls'
executor._process_creator = _MockProcessWrapper
super().setUp()
def tearDown(self):
super().tearDown()
self.finish_process()
self.executor.cleanup()
def write_process_output(self, text):
wrapper = self.executor.process_wrapper
wrapper._write_script_output(text)
# noinspection PyUnresolvedReferences
def finish_process(self):
self.executor.process_wrapper.kill()
def get_finish_output(self):
data = read_until_closed(self.executor.get_anonymized_output_stream(),
timeout=BUFFER_FLUSH_WAIT_TIME)
output = ''.join(data)
return output
def create_and_start_executor(self, parameter_values=None):
if parameter_values is None:
parameter_values = {}
self.executor = ScriptExecutor(self.config, parameter_values)
self.executor.start()
return self.executor
def get_secure_command(self, parameters, values):
config = create_config_model('config_x', parameters=parameters)
executor = ScriptExecutor(config, values)
return executor.get_secure_command()
def post(self):
script_name = None
audit_name = get_audit_name(self)
try:
arguments = tornado_utils.get_form_arguments(self)
execution_info = external_model.to_execution_info(arguments)
script_name = execution_info.script
config = load_config(script_name)
if not config:
message = 'Script with name "' + str(
script_name) + '" not found'
LOGGER.error(message)
respond_error(self, 400, message)
return
file_upload_feature = self.application.file_upload_feature
if self.request.files:
for key, value in self.request.files.items():
file_info = value[0]
file_path = file_upload_feature.save_file(
file_info.filename, file_info.body, audit_name)
execution_info.param_values[key] = file_path
valid_parameters = model_helper.validate_parameters(
execution_info.param_values, config)
if not valid_parameters:
message = 'Received invalid parameters'
LOGGER.error(message)
respond_error(self, 400, message)
return
executor = ScriptExecutor(config, execution_info.param_values,
audit_name)
audit_command = executor.get_secure_command()
LOGGER.info('Calling script: ' + audit_command)
LOGGER.info('User info: ' + str(get_all_audit_names(self)))
process_id = executor.start()
running_scripts[process_id] = executor
self.write(str(process_id))
secure_output_stream = executor.get_secure_output_stream()
self.start_script_output_logger(audit_name, script_name,
secure_output_stream)
alerts_config = self.application.alerts_config
if alerts_config:
self.subscribe_fail_alerter(script_name, alerts_config,
audit_name, executor,
secure_output_stream)
except Exception as e:
LOGGER.exception("Error while calling the script")
if hasattr(e, "strerror") and e.strerror:
error_output = e.strerror
else:
error_output = "Unknown error occurred, contact the administrator"
result = " --- ERRORS --- \n"
result += error_output
if script_name:
script = str(script_name)
else:
script = "Some script"
audit_name = audit_name
send_alerts(
self.application.alerts_config, script + ' NOT STARTED',
"Couldn't start the script " + script + ' by ' + audit_name +
'.\n\n' + result)
respond_error(self, 500, result)
