def get(self, user, execution_id):
if is_empty(execution_id):
respond_error(self, 400, 'Execution id is not specified')
return
try:
history_entry = self.application.execution_logging_service.find_history_entry(
execution_id, user.user_id)
except AccessProhibitedException:
respond_error(
self, 403,
'Access to execution #' + str(execution_id) + ' is prohibited')
return
if history_entry is None:
respond_error(self, 400, 'No history found for id ' + execution_id)
return
log = self.application.execution_logging_service.find_log(execution_id)
if is_empty(log):
LOGGER.warning('No log found for execution ' + execution_id)
running = self.application.execution_service.is_running(
history_entry.id, user)
long_log = to_long_execution_log(history_entry, log, running)
self.write(json.dumps(long_log))
def validate_value(self, value, *, ignore_required=False):
if self.constant:
return None
if is_empty(value):
if self.required and not ignore_required:
return 'is not specified'
return None
value_string = self.value_to_str(value)
if self.no_value:
if value not in ['true', True, 'false', False]:
return 'should be boolean, but has value ' + value_string
return None
if self.type == 'text':
return None
if self.type == 'file_upload':
if not os.path.exists(value):
return 'Cannot find file ' + value
return None
if self.type == 'int':
if not (isinstance(value, int) or (isinstance(value, str) and string_utils.is_integer(value))):
return 'should be integer, but has value ' + value_string
int_value = int(value)
if (not is_empty(self.max)) and (int_value > int(self.max)):
return 'is greater than allowed value (' \
+ value_string + ' > ' + str(self.max) + ')'
if (not is_empty(self.min)) and (int_value < int(self.min)):
return 'is lower than allowed value (' \
+ value_string + ' < ' + str(self.min) + ')'
return None
allowed_values = self.values
if self.type == 'list':
if value not in allowed_values:
return 'has value ' + value_string \
+ ', but should be in [' + ','.join(allowed_values) + ']'
return None
if self.type == 'multiselect':
if not isinstance(value, list):
return 'should be a list, but was: ' + value_string + '(' + str(type(value)) + ')'
for value_element in value:
if value_element not in allowed_values:
element_str = self.value_to_str(value_element)
return 'has value ' + element_str \
+ ', but should be in [' + ','.join(allowed_values) + ']'
return None
return None
def _create_values_provider(self, values_config, type, constant):
if constant:
return NoneValuesProvider()
if self._is_plain_server_file():
return FilesProvider(self._list_files_dir, self.file_type,
self.file_extensions)
if (type != 'list') and (type != PARAM_TYPE_MULTISELECT):
return NoneValuesProvider()
if is_empty(values_config):
return EmptyValuesProvider()
if isinstance(values_config, list):
return ConstValuesProvider(values_config)
elif 'script' in values_config:
script = values_config['script']
if '${' not in script:
return ScriptValuesProvider(script)
return DependantScriptValuesProvider(script,
self._parameters_supplier)
else:
message = 'Unsupported "values" format for ' + self.name
raise Exception(message)
def _create_values_provider(self, values_config, type, constant):
if constant:
return NoneValuesProvider()
if self._is_plain_server_file():
return FilesProvider(self._list_files_dir, self.file_type, self.file_extensions)
if (type != 'list') and (type != PARAM_TYPE_MULTISELECT):
return NoneValuesProvider()
if is_empty(values_config):
return EmptyValuesProvider()
if isinstance(values_config, list):
return ConstValuesProvider(values_config)
elif 'script' in values_config:
script = values_config['script']
if '${' not in script:
return ScriptValuesProvider(script)
return DependantScriptValuesProvider(script, self._parameters_supplier)
else:
message = 'Unsupported "values" format for ' + self.name
raise Exception(message)
def list_files(self, path):
if not self._is_recursive_server_file():
raise WrongParameterUsageException(self.name, 'Can list files only for recursive file parameters')
validation_error = self._validate_recursive_path(path, intermediate=True)
if validation_error:
raise InvalidValueException(self.name, validation_error)
full_path = self._build_list_file_path(path)
result = []
if is_empty(self.file_type) or self.file_type == FILE_TYPE_FILE:
files = model_helper.list_files(full_path, FILE_TYPE_FILE, self.file_extensions)
for file in files:
result.append({'name': file, 'type': FILE_TYPE_FILE, 'readable': True})
dirs = model_helper.list_files(full_path, FILE_TYPE_DIR)
for dir in dirs:
dir_path = os.path.join(full_path, dir)
readable = os.access(dir_path, os.R_OK)
result.append({'name': dir, 'type': FILE_TYPE_DIR, 'readable': readable})
return result
def __init_secure_replacements(self):
word_replacements = {}
for parameter in self.config.parameters:
if not parameter.secure:
continue
value = self._parameter_values.get(parameter.name)
if value is None:
continue
mapped_value = value.mapped_script_value
if model_helper.is_empty(mapped_value):
continue
if isinstance(mapped_value, list):
elements = mapped_value
else:
elements = [mapped_value]
for value_element in elements:
element_string = str(value_element)
if not element_string.strip():
continue
value_pattern = '((?<!\w)|^)' + re.escape(
element_string) + '((?!\w)|$)'
word_replacements[value_pattern] = model_helper.SECURE_MASK
return word_replacements
def list_files(self, path):
if not self._is_recursive_server_file():
raise WrongParameterUsageException(self.name, 'Can list files only for recursive file parameters')
validation_error = self._validate_recursive_path(path, intermediate=True)
if validation_error:
raise InvalidValueException(self.name, validation_error)
full_path = self._build_list_file_path(path)
result = []
if is_empty(self.file_type) or self.file_type == FILE_TYPE_FILE:
files = model_helper.list_files(full_path, FILE_TYPE_FILE, self.file_extensions)
for file in files:
result.append({'name': file, 'type': FILE_TYPE_FILE, 'readable': True})
dirs = model_helper.list_files(full_path, FILE_TYPE_DIR)
for dir in dirs:
dir_path = os.path.join(full_path, dir)
readable = os.access(dir_path, os.R_OK)
result.append({'name': dir, 'type': FILE_TYPE_DIR, 'readable': readable})
return result
def __init_secure_replacements(self):
word_replacements = {}
for parameter in self.config.parameters:
if not parameter.secure:
continue
value = self._parameter_values.get(parameter.name)
if value is None:
continue
mapped_value = value.mapped_script_value
if model_helper.is_empty(mapped_value):
continue
if isinstance(mapped_value, list):
elements = mapped_value
else:
elements = [mapped_value]
for value_element in elements:
element_string = str(value_element)
if not element_string.strip():
continue
value_pattern = '((?<!\w)|^)' + re.escape(element_string) + '((?!\w)|$)'
word_replacements[value_pattern] = model_helper.SECURE_MASK
return word_replacements
def _create_values_provider(self, values_config, type, constant):
if constant:
return NoneValuesProvider()
if self._is_plain_server_file():
return FilesProvider(self._list_files_dir, self.file_type,
self.file_extensions)
if (type in [PARAM_TYPE_MULTISELECT, PARAM_TYPE_LIST]):
if is_empty(values_config):
return EmptyValuesProvider()
if isinstance(values_config, list):
return ConstValuesProvider(values_config)
if PARAM_TYPE_DEPENDANT_LIST in values_config:
return DependantValuesProvider(
self.name, values_config[PARAM_TYPE_DEPENDANT_LIST],
self._parameters_supplier)
if 'script' in values_config:
script = replace_auth_vars(values_config['script'],
self._username, self._audit_name)
if '${' not in script:
return ScriptValuesProvider(script)
return DependantScriptValuesProvider(script,
self._parameters_supplier)
elif (type in ["text"]):
if is_empty(values_config):
return NoneValuesProvider()
if 'script' in values_config:
script = values_config['script']
if '${' not in script:
return ScriptValuesProvider(script)
return DependantScriptValuesProvider(script,
self._parameters_supplier)
else:
return NoneValuesProvider()
message = 'Unsupported "values" format for ' + self.name
raise Exception(message)
def get(self, execution_id):
if is_empty(execution_id):
respond_error(self, 400, 'Execution id is not specified')
return
history_entry = self.application.execution_logging_service.find_history_entry(execution_id)
if history_entry is None:
respond_error(self, 400, 'No history found for id ' + execution_id)
return
log = self.application.execution_logging_service.find_log(execution_id)
if is_empty(log):
LOGGER.warning('No log found for execution ' + execution_id)
running = self.application.execution_service.is_running(history_entry.id)
long_log = to_long_execution_log(history_entry, log, running)
self.write(json.dumps(long_log))
def get(self, execution_id):
if is_empty(execution_id):
respond_error(self, 400, 'Execution id is not specified')
return
history_entry = self.application.execution_logging_service.find_history_entry(execution_id)
if history_entry is None:
respond_error(self, 400, 'No history found for id ' + execution_id)
return
log = self.application.execution_logging_service.find_log(execution_id)
if is_empty(log):
LOGGER.warning('No log found for execution ' + execution_id)
running = self.application.execution_service.is_running(history_entry.id)
long_log = to_long_execution_log(history_entry, log, running)
self.write(json.dumps(long_log))
def normalize_user_value(self, value):
if self.type == PARAM_TYPE_MULTISELECT or self._is_recursive_server_file():
if isinstance(value, list):
return value
if not is_empty(value):
return [value]
else:
return []
return value
def _resolve_parameter_file_type(config, key, file_extensions):
if file_extensions:
return FILE_TYPE_FILE
value = config.get(key)
if is_empty(value):
return value
return value.strip().lower()
def _resolve_parameter_file_type(config, key, file_extensions):
if file_extensions:
return FILE_TYPE_FILE
value = config.get(key)
if is_empty(value):
return value
return value.strip().lower()
def normalize_user_value(self, value):
if self.type == PARAM_TYPE_MULTISELECT or self._is_recursive_server_file():
if isinstance(value, list):
return value
if not is_empty(value):
return [value]
else:
return []
return value
def get_values(self, parameter_values):
for param_name in self._required_parameters:
value = parameter_values.get(param_name)
if is_empty(value):
return []
parameters = self._parameters_supplier()
script = fill_parameter_values(parameters, self._script_template,
parameter_values)
try:
script_output = process_utils.invoke(script)
except Exception as e:
LOGGER.warning('Failed to execute script. ' + str(e))
return []
script_output = script_output.rstrip('\n')
return [
line for line in script_output.split('\n') if not is_empty(line)
]
def validate_execution_id(self, execution_id, user, only_active=True, allow_when_history_access=False):
if is_empty(execution_id):
raise MissingArgumentException('Execution id is missing', 'execution_id')
if only_active and (not self.is_active(execution_id)):
raise NotFoundException('No (active) executor found for id ' + execution_id)
if not self.can_access(execution_id, user.user_id) \
and not (allow_when_history_access and self._has_full_history_rights(user.user_id)):
LOGGER.warning('Prohibited access to not owned execution #%s (user=%s)',
execution_id, str(user))
raise AccessProhibitedException('Prohibited access to not owned execution')
def _get_paths(self, execution_id, predicate):
config = self.config
if is_empty(config.output_files):
return []
paths = [_extract_path(f) for f in config.output_files if predicate(f)]
paths = [p for p in paths if p]
parameter_values = self.execution_service.get_user_parameter_values(
execution_id)
return substitute_parameter_values(config.parameters, paths,
parameter_values)
def get_values(self, parameter_values):
self._availible_values = set()
deny_values = set()
for e in self._values_conditions:
for key in self._values_conditions[e]:
pvalue = parameter_values.get(key)
if not is_empty(pvalue):
if not(pvalue in self._values_conditions[e][key] ):
deny_values.add(e)
self._availible_values = self._possible_values.difference(deny_values)
return list(self._availible_values)
def validate_execution_id(execution_id, request_handler, only_active=True):
if is_empty(execution_id):
raise tornado.web.HTTPError(400, reason='Execution id is missing')
execution_service = request_handler.application.execution_service
if only_active and (not execution_service.is_active(execution_id)):
raise tornado.web.HTTPError(400, reason='No (active) executor found for id ' + execution_id)
user_id = _identify_user(request_handler)
if not execution_service.can_access(execution_id, user_id):
LOGGER.warning('Prohibited access to not owned execution #%s (user=%s)',
execution_id, str(user_id))
raise tornado.web.HTTPError(403, reason='Prohibited access to not owned execution')
def validate_execution_id(execution_id, request_handler, only_active=True):
if is_empty(execution_id):
raise tornado.web.HTTPError(400, reason='Execution id is missing')
execution_service = request_handler.application.execution_service
if only_active and (not execution_service.is_active(execution_id)):
raise tornado.web.HTTPError(400, reason='No (active) executor found for id ' + execution_id)
user_id = _identify_user(request_handler)
if not execution_service.can_access(execution_id, user_id):
LOGGER.warning('Prohibited access to not owned execution #%s (user=%s)',
execution_id, str(user_id))
raise tornado.web.HTTPError(403, reason='Prohibited access to not owned execution')
def _reload(self):
values_filled = True
for param_name in self.required_parameters:
value = self._values.get(param_name)
if is_empty(value):
values_filled = False
break
if self._template is None:
self.value = None
elif values_filled:
self.value = fill_parameter_values(self._parameters, self._template, self._values)
else:
self.value = self._empty
self._value_property.set(self.value)
def _reload(self):
values_filled = True
for param_name in self.required_parameters:
value = self._values.get(param_name)
if is_empty(value):
values_filled = False
break
if self._template is None:
self.value = None
elif values_filled:
self.value = fill_parameter_values(self._parameters, self._template, self._values)
else:
self.value = self._empty
self._value_property.set(self.value)
def get_values(self, parameter_values):
for param_name in self._required_parameters:
value = parameter_values.get(param_name)
if is_empty(value):
return []
parameters = self._parameters_supplier()
script = fill_parameter_values(parameters, self._script_template, parameter_values)
try:
script_output = process_utils.invoke(script)
except Exception as e:
LOGGER.warn('Failed to execute script. ' + str(e))
return []
script_output = script_output.rstrip('\n')
return script_output.split('\n')
def get_values(self, parameter_values):
for param_name in self._required_parameters:
value = parameter_values.get(param_name)
if is_empty(value):
return []
script = fill_parameter_values(self._parameter_configs,
self._script_template, parameter_values)
try:
script_output = process_utils.invoke(script)
except:
LOGGER.exception('Failed to execute script')
return []
script_output = script_output.rstrip('\n')
return script_output.split('\n')
def _reload_values(self):
values_provider = self._values_provider
if not values_provider:
self.values = None
return
if (self.type in ('ip', 'ip4', 'ip6', "text", "int")):
if not isinstance(values_provider, NoneValuesProvider):
values = values_provider.get_values(self._parameter_values)
if (not is_empty(values) and (not self._changed_by_user)):
self.default = values[0]
self.values = values
else:
self.default = ""
self.values = []
else:
values = values_provider.get_values(self._parameter_values)
self.values = values
def _get_paths(self, execution_id, predicate):
config = self.config
if is_empty(config.output_files):
return []
paths = [_extract_path(f) for f in config.output_files if predicate(f)]
paths = [p for p in paths if p]
parameter_values = self.execution_service.get_user_parameter_values(
execution_id)
all_audit_names = self.execution_service.get_all_audit_names(
execution_id)
audit_name = audit_utils.get_audit_name(all_audit_names)
username = audit_utils.get_audit_username(all_audit_names)
return substitute_variable_values(config.parameters, paths,
parameter_values, audit_name,
username)
def map_to_script(self, user_value):
def map_single_value(user_value):
if self._values_provider:
return self._values_provider.map_value(user_value)
return user_value
if self.type == PARAM_TYPE_MULTISELECT:
return [map_single_value(v) for v in user_value]
elif self._is_recursive_server_file():
if user_value:
return os.path.join(self.file_dir, *user_value)
else:
return None
elif self._is_plain_server_file():
if not is_empty(user_value):
return os.path.join(self.file_dir, user_value)
else:
return None
return map_single_value(user_value)
def _create_values_provider(self, values_config, type, constant):
if constant or ((type != 'list') and (type != 'multiselect')):
return NoneValuesProvider()
if is_empty(values_config):
return EmptyValuesProvider()
if isinstance(values_config, list):
return ConstValuesProvider(values_config)
elif 'script' in values_config:
script = values_config['script']
if '${' not in script:
return ScriptValuesProvider(script)
return DependantScriptValuesProvider(script, self._parameters_supplier)
else:
message = 'Unsupported "values" format for ' + self.name
raise Exception(message)
def map_to_script(self, user_value):
def map_single_value(user_value):
if self._values_provider:
return self._values_provider.map_value(user_value)
return user_value
if self.type == PARAM_TYPE_MULTISELECT:
return [map_single_value(v) for v in user_value]
elif self._is_recursive_server_file():
if user_value:
return os.path.join(self.file_dir, *user_value)
else:
return None
elif self._is_plain_server_file():
if not is_empty(user_value):
return os.path.join(self.file_dir, user_value)
else:
return None
return map_single_value(user_value)
def _create_values_provider(self, values_config, type, constant):
if constant:
return NoneValuesProvider()
if self._is_plain_server_file():
return FilesProvider(self._list_files_dir, self.file_type,
self.file_extensions,
self.excluded_files_matcher)
if (type != 'list') and (type != PARAM_TYPE_MULTISELECT) and (
type != PARAM_TYPE_EDITABLE_LIST):
return NoneValuesProvider()
if is_empty(values_config):
return EmptyValuesProvider()
if isinstance(values_config, list):
return ConstValuesProvider(values_config)
elif 'script' in values_config:
original_script = values_config['script']
has_variables = ('${' in original_script)
script = replace_auth_vars(original_script, self._username,
self._audit_name)
shell = read_bool_from_config('shell',
values_config,
default=not has_variables)
if '${' not in script:
return ScriptValuesProvider(script, shell)
return DependantScriptValuesProvider(script,
self._parameters_supplier,
shell)
else:
message = 'Unsupported "values" format for ' + self.name
raise Exception(message)
def __init_secure_replacements(self):
word_replacements = {}
for parameter in self.config.parameters:
if not parameter.secure:
continue
value = self.parameter_values.get(parameter.name)
if model_helper.is_empty(value):
continue
if isinstance(value, list):
elements = value
else:
elements = [value]
for value_element in elements:
element_string = str(value_element)
if not element_string.strip():
continue
value_pattern = '\\b' + re.escape(element_string) + '\\b'
word_replacements[value_pattern] = model_helper.SECURE_MASK
return word_replacements
def test_next_id(self):
generator = IdGenerator([])
id = generator.next_id()
self.assertIsNotNone(id)
self.assertIsInstance(id, str)
self.assertFalse(is_empty(id))
def validate_value(self, value, *, ignore_required=False):
if self.constant:
return None
if is_empty(value):
if self.required and not ignore_required:
return 'is not specified'
return None
value_string = self.value_to_repr(value)
if self.no_value:
if value not in ['true', True, 'false', False]:
return 'should be boolean, but has value ' + value_string
return None
if self.type == 'text':
return None
if self.type == 'file_upload':
if not os.path.exists(value):
return 'Cannot find file ' + value
return None
if self.type == 'int':
if not (isinstance(value, int) or
(isinstance(value, str)
and string_utils.is_integer(value))):
return 'should be integer, but has value ' + value_string
int_value = int(value)
if (not is_empty(self.max)) and (int_value > int(self.max)):
return 'is greater than allowed value (' \
+ value_string + ' > ' + str(self.max) + ')'
if (not is_empty(self.min)) and (int_value < int(self.min)):
return 'is lower than allowed value (' \
+ value_string + ' < ' + str(self.min) + ')'
return None
if self.type in ('ip', 'ip4', 'ip6'):
try:
address = ip_address(value.strip())
if self.type == 'ip4':
if not isinstance(address, IPv4Address):
return value_string + ' is not an IPv4 address'
elif self.type == 'ip6':
if not isinstance(address, IPv6Address):
return value_string + ' is not an IPv6 address'
except ValueError:
return 'wrong IP address ' + value_string
allowed_values = self.values
if (self.type == 'list') or (self._is_plain_server_file()):
if value not in allowed_values:
return 'has value ' + value_string \
+ ', but should be in ' + repr(allowed_values)
return None
if self.type == PARAM_TYPE_MULTISELECT:
if not isinstance(value, list):
return 'should be a list, but was: ' + value_string + '(' + str(
type(value)) + ')'
for value_element in value:
if value_element not in allowed_values:
element_str = self.value_to_repr(value_element)
return 'has value ' + element_str \
+ ', but should be in ' + repr(allowed_values)
return None
if self._is_recursive_server_file():
return self._validate_recursive_path(value, intermediate=False)
return None
def validate_value(self, value, *, ignore_required=False):
if self.constant:
return None
if is_empty(value):
if self.required and not ignore_required:
return 'is not specified'
return None
value_string = self.value_to_str(value)
if self.no_value:
if value not in ['true', True, 'false', False]:
return 'should be boolean, but has value ' + value_string
return None
if self.type == 'text':
return None
if self.type == 'file_upload':
if not os.path.exists(value):
return 'Cannot find file ' + value
return None
if self.type == 'int':
if not (isinstance(value, int) or (isinstance(value, str) and string_utils.is_integer(value))):
return 'should be integer, but has value ' + value_string
int_value = int(value)
if (not is_empty(self.max)) and (int_value > int(self.max)):
return 'is greater than allowed value (' \
+ value_string + ' > ' + str(self.max) + ')'
if (not is_empty(self.min)) and (int_value < int(self.min)):
return 'is lower than allowed value (' \
+ value_string + ' < ' + str(self.min) + ')'
return None
if self.type in ('ip', 'ip4', 'ip6'):
try:
address = ip_address(value.strip())
if self.type == 'ip4':
if not isinstance(address, IPv4Address):
return value_string + ' is not an IPv4 address'
elif self.type == 'ip6':
if not isinstance(address, IPv6Address):
return value_string + ' is not an IPv6 address'
except ValueError:
return 'wrong IP address ' + value_string
allowed_values = self.values
if (self.type == 'list') or (self._is_plain_server_file()):
if value not in allowed_values:
return 'has value ' + value_string \
+ ', but should be in [' + ','.join(allowed_values) + ']'
return None
if self.type == PARAM_TYPE_MULTISELECT:
if not isinstance(value, list):
return 'should be a list, but was: ' + value_string + '(' + str(type(value)) + ')'
for value_element in value:
if value_element not in allowed_values:
element_str = self.value_to_str(value_element)
return 'has value ' + element_str \
+ ', but should be in [' + ','.join(allowed_values) + ']'
return None
if self._is_recursive_server_file():
return self._validate_recursive_path(value, intermediate=False)
return None
def get_request_body(request_handler):
raw_request_body = request_handler.request.body.decode('UTF-8')
if is_empty(raw_request_body):
return {}
return json.loads(raw_request_body)
def get_request_body(request_handler):
raw_request_body = request_handler.request.body.decode('UTF-8')
if is_empty(raw_request_body):
return {}
return json.loads(raw_request_body)
def _is_downloadable(config):
return not is_empty(config.output_files)
def __init__(self, script) -> None:
script_output = process_utils.invoke(script)
script_output = script_output.rstrip('\n')
self._values = [
line for line in script_output.split('\n') if not is_empty(line)
]
