def _preprocess_incoming_config(config):
name = config.get('name')
if is_blank(name):
raise InvalidConfigException('Script name is required')
config['name'] = name.strip()
script_path = config.get('script_path')
if is_blank(script_path):
raise InvalidConfigException('Script path is required')
config['script_path'] = script_path.strip()
def read_str_from_config(config_obj,
key,
*,
default=None,
blank_to_none=False):
"""
Reads string value from a config by the key
If the value is missing, returns specified default value
If the value is not string, InvalidValueTypeException is thrown
:param config_obj: where to read value from
:param key: key to read value from
:param default: default value, if config value is missing
:param blank_to_none: if value is blank, treat it as null
:return: config_obj[key] if non None, default otherwise
"""
value = config_obj.get(key)
if blank_to_none and isinstance(value, str) and is_blank(value):
value = None
if value is None:
return default
if isinstance(value, str):
return value
raise InvalidValueTypeException(
'Invalid %s value: string expected, but was: %s' % (key, repr(value)))
def update_config(self, user, config, filename):
self._check_admin_access(user)
_preprocess_incoming_config(config)
if is_blank(filename):
raise InvalidConfigException('Script filename should be specified')
original_file_path = os.path.join(self._script_configs_folder,
filename)
if not os.path.exists(original_file_path):
raise InvalidFileException(
original_file_path,
'Failed to find script path: ' + original_file_path)
name = config['name']
(short_config, found_config_path,
json_object) = self._find_config(name)
if (found_config_path is not None) and (
os.path.basename(found_config_path) != filename):
raise InvalidConfigException(
'Another script found with the same name: ' + name)
LOGGER.info('Updating script config "' + name + '" in ' +
original_file_path)
self._save_config(config, original_file_path)
def __migrate_output_files_parameters_substitution(context):
for (conf_file, json_object,
content) in _load_runner_files(context.conf_folder):
if ('output_files' not in json_object) or ('parameters'
not in json_object):
continue
output_files = json_object['output_files']
parameter_names = [
p['name'] for p in json_object['parameters']
if not is_blank(p.get('name'))
]
changed = False
for i in range(len(output_files)):
output_file = output_files[i]
if not isinstance(output_file, str):
continue
for param_name in parameter_names:
output_file = re.sub('\\$\\$\\$' + param_name,
'${' + param_name + '}', output_file)
if output_file != output_files[i]:
output_files[i] = output_file
changed = True
if changed:
_write_json(conf_file, json_object, content)
def __introduce_access_config(context):
conf_folder = os.path.join(context.conf_folder, 'runners')
if not os.path.exists(conf_folder):
return
conf_files = [os.path.join(conf_folder, file)
for file in os.listdir(conf_folder)
if file.lower().endswith('.json')]
for conf_file in conf_files:
content = file_utils.read_file(conf_file)
json_object = json.loads(content, object_pairs_hook=OrderedDict)
if ('output_files' not in json_object) or ('parameters' not in json_object):
continue
output_files = json_object['output_files']
parameter_names = [p['name'] for p in json_object['parameters'] if not is_blank(p.get('name'))]
changed = False
for i in range(len(output_files)):
output_file = output_files[i]
for param_name in parameter_names:
output_file = re.sub('\$\$\$' + param_name, '${' + param_name + '}', output_file)
if output_file != output_files[i]:
output_files[i] = output_file
changed = True
if changed:
_write_json(conf_file, json_object, content)
def _extract_path(output_file):
if isinstance(output_file, str):
return output_file
elif isinstance(output_file, dict):
path = output_file.get('path')
if not string_utils.is_blank(path):
return path.strip()
return None
def _read_repeat_unit(incoming_schedule_config):
repeat_unit = incoming_schedule_config.get('repeat_unit')
if is_blank(repeat_unit):
raise InvalidScheduleException('repeat_unit is required for repeatable schedule')
if repeat_unit.lower() not in ['minutes', 'hours', 'days', 'weeks', 'months']:
raise InvalidScheduleException('repeat_unit should be one of: minutes, hours, days, weeks, months')
return repeat_unit.lower()
def _parse_htpasswd(htpasswd_path):
with open(htpasswd_path, 'r') as f:
lines = [line.strip() for line in f.readlines() if not is_blank(line)]
result = {}
for line in lines:
(username, password) = line.split(':', 1)
result[username] = password
return result
def test_update_config_script_update_when_code_loading_problems(
self, mode, original_script_path, expected_exception,
expected_message):
copyfile(sys.executable, os.path.join(test_utils.temp_folder,
'python'))
body = None
if mode == 'new_code':
script = {
'mode':
'new_code',
'code':
'abcdef',
'path':
original_script_path
if not is_blank(original_script_path) else 'anything'
}
elif mode == 'upload_script':
script = {
'mode':
'upload_script',
'path':
original_script_path
if not is_blank(original_script_path) else 'anything'
}
body = HTTPFile(filename='whatever', body=b'xyz')
else:
script = None
_create_script_config_file('ConfA',
name='ConfA',
script_path=original_script_path)
config = _prepare_script_config_object('ConfA', script=script)
if expected_exception is not None:
self.assertRaisesRegex(expected_exception, expected_message,
self.config_service.update_config,
self.admin_user, config, 'ConfA.json', body)
else:
self.config_service.update_config(self.admin_user, config,
'ConfA.json', body)
def __introduce_access_config(context):
conf_folder = os.path.join(context.conf_folder, 'runners')
if not os.path.exists(conf_folder):
return
conf_files = [
os.path.join(conf_folder, file) for file in os.listdir(conf_folder)
if file.lower().endswith('.json')
]
for conf_file in conf_files:
content = file_utils.read_file(conf_file)
json_object = json.loads(content, object_pairs_hook=OrderedDict)
if ('output_files' not in json_object) or ('parameters'
not in json_object):
continue
output_files = json_object['output_files']
parameter_names = [
p['name'] for p in json_object['parameters']
if not is_blank(p.get('name'))
]
changed = False
for i in range(len(output_files)):
output_file = output_files[i]
if not isinstance(output_file, str):
continue
for param_name in parameter_names:
output_file = re.sub('\\$\\$\\$' + param_name,
'${' + param_name + '}', output_file)
if output_file != output_files[i]:
output_files[i] = output_file
changed = True
if changed:
_write_json(conf_file, json_object, content)
def _load_script_code_by_config(self, plain_config):
script_path = plain_config.get(SCRIPT_PATH_FIELD)
if is_blank(script_path):
raise InvalidFileException('', 'Script path is not specified')
command = process_utils.split_command(
script_path, plain_config.get(WORKING_DIR_FIELD))
binary_files = []
for argument in command:
if file_utils.exists(argument):
if file_utils.is_binary(argument):
binary_files.append(argument)
continue
return {
'code': file_utils.read_file(argument),
'file_path': argument
}
if binary_files:
if len(binary_files) == 1:
return {
'code': None,
'file_path': binary_files[0],
'code_edit_error': 'Cannot edit binary file'
}
raise InvalidFileException(
'command', 'Cannot choose which binary file to edit: ' +
str(binary_files))
if len(command) == 1:
return {
'code': None,
'file_path': command[0],
'code_edit_error': 'Script path does not exist'
}
raise InvalidFileException(
'command',
'Failed to find script path in command "' + script_path + '"')
def update_config(self, user, config, filename, uploaded_script):
self._check_admin_access(user)
_preprocess_incoming_config(config)
if is_blank(filename):
raise InvalidConfigException('Script filename should be specified')
original_file_path = os.path.join(self._script_configs_folder,
filename)
if not os.path.exists(original_file_path):
raise InvalidFileException(
original_file_path,
'Failed to find script path: ' + original_file_path)
with open(original_file_path, 'r') as f:
original_config_json = json.load(f)
short_original_config = script_config.read_short(
original_file_path, original_config_json)
name = config['name']
search_result = self._find_config(name)
if (search_result is not None) and (os.path.basename(
search_result.path) != filename):
raise InvalidConfigException(
'Another script found with the same name: ' + name)
if not self._can_edit_script(user, short_original_config):
raise ConfigNotAllowedException(
str(user) + ' is not allowed to modify ' +
short_original_config.name)
self._preprocess_script_fields(config, original_config_json,
uploaded_script, user)
LOGGER.info('Updating script config "' + name + '" in ' +
original_file_path)
self._save_config(config, original_file_path)
def assert_error(self, error):
self.assertFalse(is_blank(error), 'Expected validation error, but validation passed')
def _preprocess_script_fields(self, config, original_config_json,
uploaded_script, user):
script_config = config.get('script')
if not script_config:
raise InvalidConfigException('script option is required')
if SCRIPT_PATH_FIELD in config:
del config[SCRIPT_PATH_FIELD]
del config['script']
new_path = strip(script_config.get('path'))
if is_blank(new_path):
raise InvalidConfigException('script.path option is required')
config[SCRIPT_PATH_FIELD] = new_path
mode = script_config.get('mode')
if is_blank(mode) or mode == SCRIPT_EDIT_PATH_MODE:
pass
elif mode in (SCRIPT_EDIT_UPLOAD_MODE, SCRIPT_EDIT_CODE_MODE):
if not self._authorizer.can_edit_code(user.user_id):
raise InvalidAccessException('User ' + str(user) +
' is not allowed to edit code')
if mode == SCRIPT_EDIT_UPLOAD_MODE:
if uploaded_script is None:
raise InvalidConfigException(
'Uploaded script should be specified')
if original_config_json is None: # new config
if mode == SCRIPT_EDIT_UPLOAD_MODE:
# escaped name is needed, when uploaded file and server has different OSes,
# thus different special characters
escaped_name = to_filename(uploaded_script.filename)
target_path = os.path.join(self._scripts_folder,
escaped_name)
else:
filename = os.path.basename(new_path)
target_path = os.path.join(
self._scripts_folder,
_escape_characters_in_filename(filename))
script_path = file_utils.create_unique_filename(
target_path, 100)
config[SCRIPT_PATH_FIELD] = script_path
else:
existing_code = self._load_script_code_by_config(
original_config_json)
script_path = existing_code['file_path']
if (mode == SCRIPT_EDIT_CODE_MODE
) and existing_code.get('code_edit_error') is not None:
raise InvalidConfigException(
'Failed to edit code: ' +
existing_code.get('code_edit_error'))
if new_path != original_config_json.get(SCRIPT_PATH_FIELD):
raise InvalidConfigException(
'script.path override is not allowed for ' + mode +
' mode')
if mode == SCRIPT_EDIT_UPLOAD_MODE:
file_utils.write_file(script_path,
uploaded_script.body,
byte_content=True)
else:
code = script_config.get('code')
if code is None:
raise InvalidConfigException(
'script.code should be specified')
file_utils.write_file(script_path, code)
file_utils.make_executable(script_path)
else:
raise InvalidConfigException('Unsupported mode: ' + mode)
def assert_error(self, error):
self.assertFalse(is_blank(error), 'Expected validation error, but validation passed')
