def _preprocess_incoming_config(config): name = config.get('name') if is_blank(name): raise InvalidConfigException('Script name is required') config['name'] = name.strip() script_path = config.get('script_path') if is_blank(script_path): raise InvalidConfigException('Script path is required') config['script_path'] = script_path.strip()
def read_str_from_config(config_obj, key, *, default=None, blank_to_none=False): """ Reads string value from a config by the key If the value is missing, returns specified default value If the value is not string, InvalidValueTypeException is thrown :param config_obj: where to read value from :param key: key to read value from :param default: default value, if config value is missing :param blank_to_none: if value is blank, treat it as null :return: config_obj[key] if non None, default otherwise """ value = config_obj.get(key) if blank_to_none and isinstance(value, str) and is_blank(value): value = None if value is None: return default if isinstance(value, str): return value raise InvalidValueTypeException( 'Invalid %s value: string expected, but was: %s' % (key, repr(value)))
def update_config(self, user, config, filename): self._check_admin_access(user) _preprocess_incoming_config(config) if is_blank(filename): raise InvalidConfigException('Script filename should be specified') original_file_path = os.path.join(self._script_configs_folder, filename) if not os.path.exists(original_file_path): raise InvalidFileException( original_file_path, 'Failed to find script path: ' + original_file_path) name = config['name'] (short_config, found_config_path, json_object) = self._find_config(name) if (found_config_path is not None) and ( os.path.basename(found_config_path) != filename): raise InvalidConfigException( 'Another script found with the same name: ' + name) LOGGER.info('Updating script config "' + name + '" in ' + original_file_path) self._save_config(config, original_file_path)
def __migrate_output_files_parameters_substitution(context): for (conf_file, json_object, content) in _load_runner_files(context.conf_folder): if ('output_files' not in json_object) or ('parameters' not in json_object): continue output_files = json_object['output_files'] parameter_names = [ p['name'] for p in json_object['parameters'] if not is_blank(p.get('name')) ] changed = False for i in range(len(output_files)): output_file = output_files[i] if not isinstance(output_file, str): continue for param_name in parameter_names: output_file = re.sub('\\$\\$\\$' + param_name, '${' + param_name + '}', output_file) if output_file != output_files[i]: output_files[i] = output_file changed = True if changed: _write_json(conf_file, json_object, content)
def __introduce_access_config(context): conf_folder = os.path.join(context.conf_folder, 'runners') if not os.path.exists(conf_folder): return conf_files = [os.path.join(conf_folder, file) for file in os.listdir(conf_folder) if file.lower().endswith('.json')] for conf_file in conf_files: content = file_utils.read_file(conf_file) json_object = json.loads(content, object_pairs_hook=OrderedDict) if ('output_files' not in json_object) or ('parameters' not in json_object): continue output_files = json_object['output_files'] parameter_names = [p['name'] for p in json_object['parameters'] if not is_blank(p.get('name'))] changed = False for i in range(len(output_files)): output_file = output_files[i] for param_name in parameter_names: output_file = re.sub('\$\$\$' + param_name, '${' + param_name + '}', output_file) if output_file != output_files[i]: output_files[i] = output_file changed = True if changed: _write_json(conf_file, json_object, content)
def _extract_path(output_file): if isinstance(output_file, str): return output_file elif isinstance(output_file, dict): path = output_file.get('path') if not string_utils.is_blank(path): return path.strip() return None
def _read_repeat_unit(incoming_schedule_config): repeat_unit = incoming_schedule_config.get('repeat_unit') if is_blank(repeat_unit): raise InvalidScheduleException('repeat_unit is required for repeatable schedule') if repeat_unit.lower() not in ['minutes', 'hours', 'days', 'weeks', 'months']: raise InvalidScheduleException('repeat_unit should be one of: minutes, hours, days, weeks, months') return repeat_unit.lower()
def _parse_htpasswd(htpasswd_path): with open(htpasswd_path, 'r') as f: lines = [line.strip() for line in f.readlines() if not is_blank(line)] result = {} for line in lines: (username, password) = line.split(':', 1) result[username] = password return result
def test_update_config_script_update_when_code_loading_problems( self, mode, original_script_path, expected_exception, expected_message): copyfile(sys.executable, os.path.join(test_utils.temp_folder, 'python')) body = None if mode == 'new_code': script = { 'mode': 'new_code', 'code': 'abcdef', 'path': original_script_path if not is_blank(original_script_path) else 'anything' } elif mode == 'upload_script': script = { 'mode': 'upload_script', 'path': original_script_path if not is_blank(original_script_path) else 'anything' } body = HTTPFile(filename='whatever', body=b'xyz') else: script = None _create_script_config_file('ConfA', name='ConfA', script_path=original_script_path) config = _prepare_script_config_object('ConfA', script=script) if expected_exception is not None: self.assertRaisesRegex(expected_exception, expected_message, self.config_service.update_config, self.admin_user, config, 'ConfA.json', body) else: self.config_service.update_config(self.admin_user, config, 'ConfA.json', body)
def __introduce_access_config(context): conf_folder = os.path.join(context.conf_folder, 'runners') if not os.path.exists(conf_folder): return conf_files = [ os.path.join(conf_folder, file) for file in os.listdir(conf_folder) if file.lower().endswith('.json') ] for conf_file in conf_files: content = file_utils.read_file(conf_file) json_object = json.loads(content, object_pairs_hook=OrderedDict) if ('output_files' not in json_object) or ('parameters' not in json_object): continue output_files = json_object['output_files'] parameter_names = [ p['name'] for p in json_object['parameters'] if not is_blank(p.get('name')) ] changed = False for i in range(len(output_files)): output_file = output_files[i] if not isinstance(output_file, str): continue for param_name in parameter_names: output_file = re.sub('\\$\\$\\$' + param_name, '${' + param_name + '}', output_file) if output_file != output_files[i]: output_files[i] = output_file changed = True if changed: _write_json(conf_file, json_object, content)
def _load_script_code_by_config(self, plain_config): script_path = plain_config.get(SCRIPT_PATH_FIELD) if is_blank(script_path): raise InvalidFileException('', 'Script path is not specified') command = process_utils.split_command( script_path, plain_config.get(WORKING_DIR_FIELD)) binary_files = [] for argument in command: if file_utils.exists(argument): if file_utils.is_binary(argument): binary_files.append(argument) continue return { 'code': file_utils.read_file(argument), 'file_path': argument } if binary_files: if len(binary_files) == 1: return { 'code': None, 'file_path': binary_files[0], 'code_edit_error': 'Cannot edit binary file' } raise InvalidFileException( 'command', 'Cannot choose which binary file to edit: ' + str(binary_files)) if len(command) == 1: return { 'code': None, 'file_path': command[0], 'code_edit_error': 'Script path does not exist' } raise InvalidFileException( 'command', 'Failed to find script path in command "' + script_path + '"')
def update_config(self, user, config, filename, uploaded_script): self._check_admin_access(user) _preprocess_incoming_config(config) if is_blank(filename): raise InvalidConfigException('Script filename should be specified') original_file_path = os.path.join(self._script_configs_folder, filename) if not os.path.exists(original_file_path): raise InvalidFileException( original_file_path, 'Failed to find script path: ' + original_file_path) with open(original_file_path, 'r') as f: original_config_json = json.load(f) short_original_config = script_config.read_short( original_file_path, original_config_json) name = config['name'] search_result = self._find_config(name) if (search_result is not None) and (os.path.basename( search_result.path) != filename): raise InvalidConfigException( 'Another script found with the same name: ' + name) if not self._can_edit_script(user, short_original_config): raise ConfigNotAllowedException( str(user) + ' is not allowed to modify ' + short_original_config.name) self._preprocess_script_fields(config, original_config_json, uploaded_script, user) LOGGER.info('Updating script config "' + name + '" in ' + original_file_path) self._save_config(config, original_file_path)
def assert_error(self, error): self.assertFalse(is_blank(error), 'Expected validation error, but validation passed')
def _preprocess_script_fields(self, config, original_config_json, uploaded_script, user): script_config = config.get('script') if not script_config: raise InvalidConfigException('script option is required') if SCRIPT_PATH_FIELD in config: del config[SCRIPT_PATH_FIELD] del config['script'] new_path = strip(script_config.get('path')) if is_blank(new_path): raise InvalidConfigException('script.path option is required') config[SCRIPT_PATH_FIELD] = new_path mode = script_config.get('mode') if is_blank(mode) or mode == SCRIPT_EDIT_PATH_MODE: pass elif mode in (SCRIPT_EDIT_UPLOAD_MODE, SCRIPT_EDIT_CODE_MODE): if not self._authorizer.can_edit_code(user.user_id): raise InvalidAccessException('User ' + str(user) + ' is not allowed to edit code') if mode == SCRIPT_EDIT_UPLOAD_MODE: if uploaded_script is None: raise InvalidConfigException( 'Uploaded script should be specified') if original_config_json is None: # new config if mode == SCRIPT_EDIT_UPLOAD_MODE: # escaped name is needed, when uploaded file and server has different OSes, # thus different special characters escaped_name = to_filename(uploaded_script.filename) target_path = os.path.join(self._scripts_folder, escaped_name) else: filename = os.path.basename(new_path) target_path = os.path.join( self._scripts_folder, _escape_characters_in_filename(filename)) script_path = file_utils.create_unique_filename( target_path, 100) config[SCRIPT_PATH_FIELD] = script_path else: existing_code = self._load_script_code_by_config( original_config_json) script_path = existing_code['file_path'] if (mode == SCRIPT_EDIT_CODE_MODE ) and existing_code.get('code_edit_error') is not None: raise InvalidConfigException( 'Failed to edit code: ' + existing_code.get('code_edit_error')) if new_path != original_config_json.get(SCRIPT_PATH_FIELD): raise InvalidConfigException( 'script.path override is not allowed for ' + mode + ' mode') if mode == SCRIPT_EDIT_UPLOAD_MODE: file_utils.write_file(script_path, uploaded_script.body, byte_content=True) else: code = script_config.get('code') if code is None: raise InvalidConfigException( 'script.code should be specified') file_utils.write_file(script_path, code) file_utils.make_executable(script_path) else: raise InvalidConfigException('Unsupported mode: ' + mode)