def start_logging(self, execution_id, user_name, user_id, script_name, command, output_stream, all_audit_names, output_format, start_time_millis=None): if start_time_millis is None: start_time_millis = get_current_millis() log_filename = self._log_name_creator.create_filename( execution_id, all_audit_names, script_name, start_time_millis) log_file_path = os.path.join(self._output_folder, log_filename) log_file_path = file_utils.create_unique_filename(log_file_path) output_logger = ScriptOutputLogger(log_file_path, output_stream) output_logger.write_line('id:' + execution_id) output_logger.write_line('user_name:' + user_name) output_logger.write_line('user_id:' + user_id) output_logger.write_line('script:' + script_name) output_logger.write_line('start_time:' + str(start_time_millis)) output_logger.write_line('command:' + command) output_logger.write_line('output_format:' + output_format) output_logger.write_line(OUTPUT_STARTED_MARKER) output_logger.start() log_filename = os.path.basename(log_file_path) self._visited_files.add(log_filename) self._ids_to_file_map[execution_id] = log_filename self._output_loggers[execution_id] = output_logger
def start_logging(self, execution_id, user_name, user_id, script_name, command, output_stream, all_audit_names, start_time_millis=None): if start_time_millis is None: start_time_millis = get_current_millis() log_filename = self._log_name_creator.create_filename( execution_id, all_audit_names, script_name, start_time_millis) log_file_path = os.path.join(self._output_folder, log_filename) log_file_path = file_utils.create_unique_filename(log_file_path) output_logger = ScriptOutputLogger(log_file_path, output_stream) output_logger.write_line('id:' + execution_id) output_logger.write_line('user_name:' + user_name) output_logger.write_line('user_id:' + user_id) output_logger.write_line('script:' + script_name) output_logger.write_line('start_time:' + str(start_time_millis)) output_logger.write_line('command:' + command) output_logger.write_line(OUTPUT_STARTED_MARKER) output_logger.start() log_filename = os.path.basename(log_file_path) self._visited_files.add(log_filename) self._ids_to_file_map[execution_id] = log_filename self._output_loggers[execution_id] = output_logger
def start_logging(self, execution_id, username, script_name, command, output_stream, post_execution_info_provider, start_time_millis=None): if start_time_millis is None: start_time_millis = get_current_millis() log_identifier = self._create_log_identifier(username, script_name, start_time_millis) log_file_path = os.path.join(self._output_folder, log_identifier + '.log') log_file_path = file_utils.create_unique_filename(log_file_path) def write_post_execution_info(): self._write_post_execution_info(execution_id, log_file_path, post_execution_info_provider) output_logger = ScriptOutputLogger(log_file_path, output_stream, write_post_execution_info) output_logger.write_line('id:' + execution_id) output_logger.write_line('user:'******'script:' + script_name) output_logger.write_line('start_time:' + str(start_time_millis)) output_logger.write_line('command:' + command) output_logger.write_line(OUTPUT_STARTED_MARKER) output_logger.start() log_filename = os.path.basename(log_file_path) self._visited_files.add(log_filename) self._ids_to_file_map[execution_id] = log_filename
def save_file(self, filename, body, username) -> str: upload_folder = self.user_file_storage.prepare_new_folder( username, self.folder) pref_result_path = os.path.join(upload_folder, filename) result_path = file_utils.create_unique_filename(pref_result_path) file_utils.write_file(result_path, body, True) return file_utils.normalize_path(result_path)
def prepare_downloadable_files(self, config, script_output, script_param_values, audit_name): output_files = config.output_files if not output_files: return [] output_files = substitute_parameter_values(config.parameters, config.output_files, script_param_values) correct_files = [] for output_file in output_files: files = find_matching_files(output_file, script_output) if files: for file in files: file_path = file_utils.normalize_path( file, config.get_working_directory()) if not os.path.exists(file_path): LOGGER.warning('file ' + file + ' (full path = ' + file_path + ') not found') elif os.path.isdir(file_path): LOGGER.warning('file ' + file + ' is a directory. Not allowed') elif file_path not in correct_files: correct_files.append(file_path) else: LOGGER.warning("Couldn't find file for " + output_file) if not correct_files: return [] download_folder = self.user_file_storage.prepare_new_folder( audit_name, self.result_folder) LOGGER.info('Created download folder for ' + audit_name + ': ' + download_folder) result = [] for file in correct_files: preferred_download_file = os.path.join(download_folder, os.path.basename(file)) try: download_file = create_unique_filename(preferred_download_file) except file_utils.FileExistsException: LOGGER.exception('Cannot get unique name') continue copyfile(file, download_file) result.append(download_file) return result
def _prepare_downloadable_files(self, output_files, config, script_output, *, should_exist=True): found_files = {} for output_file in output_files: files = find_matching_files(output_file, script_output) if files: for file in files: file_path = file_utils.normalize_path( file, config.working_directory) if not os.path.exists(file_path): if should_exist: LOGGER.warning('file ' + file + ' (full path = ' + file_path + ') not found') elif os.path.isdir(file_path): LOGGER.warning('file ' + file + ' is a directory. Not allowed') elif file_path not in found_files: found_files[file] = file_path elif should_exist: LOGGER.warning("Couldn't find file for " + output_file) if not found_files: return {} result = {} for original_file_path, normalized_path in found_files.items(): if original_file_path in self.prepared_files: result[original_file_path] = self.prepared_files[ original_file_path] continue preferred_download_file = os.path.join( self.download_folder, os.path.basename(normalized_path)) try: download_file = create_unique_filename(preferred_download_file) except file_utils.FileExistsException: LOGGER.exception('Cannot get unique name') continue copyfile(normalized_path, download_file) result[original_file_path] = download_file self.prepared_files[original_file_path] = download_file return result
def _prepare_downloadable_files(self, config, script_output, script_param_values, execution_owner): output_files = config.output_files if not output_files: return [] output_files = substitute_parameter_values( config.parameters, config.output_files, script_param_values) correct_files = [] for output_file in output_files: files = find_matching_files(output_file, script_output) if files: for file in files: file_path = file_utils.normalize_path(file, config.working_directory) if not os.path.exists(file_path): LOGGER.warning('file ' + file + ' (full path = ' + file_path + ') not found') elif os.path.isdir(file_path): LOGGER.warning('file ' + file + ' is a directory. Not allowed') elif file_path not in correct_files: correct_files.append(file_path) else: LOGGER.warning("Couldn't find file for " + output_file) if not correct_files: return [] download_folder = self.user_file_storage.prepare_new_folder(execution_owner, self.result_folder) LOGGER.info('Created download folder for ' + execution_owner + ': ' + download_folder) result = [] for file in correct_files: preferred_download_file = os.path.join(download_folder, os.path.basename(file)) try: download_file = create_unique_filename(preferred_download_file) except file_utils.FileExistsException: LOGGER.exception('Cannot get unique name') continue copyfile(file, download_file) result.append(download_file) return result
def create_config(self, user, config): self._check_admin_access(user) _preprocess_incoming_config(config) name = config['name'] (short_config, path, json_object) = self._find_config(name) if path is not None: raise InvalidConfigException('Another config with the same name already exists') path = os.path.join(self._script_configs_folder, _script_name_to_file_name(name)) unique_path = file_utils.create_unique_filename(path, 100) LOGGER.info('Creating new script config "' + name + '" in ' + unique_path) self._save_config(config, unique_path)
def __init__(self, header, files_path): header_str = header.decode('utf-8') if '\r\n' in header_str: header_str = header_str[:header_str.index('\r\n')] (value, sub_headers) = parse_header(header_str) self.name = sub_headers['name'] self.filename = sub_headers.get('filename') if self.filename: self.path = os.path.join(files_path, self.filename) self.path = file_utils.create_unique_filename(self.path) # touch file open(self.path, 'w').close() else: self.value = ''
def create_config(self, user, config, uploaded_script): self._check_admin_access(user) _preprocess_incoming_config(config) name = config['name'] search_result = self._find_config(name) if search_result is not None: raise InvalidConfigException( 'Another config with the same name already exists') self._preprocess_script_fields(config, None, uploaded_script, user) path = os.path.join(self._script_configs_folder, _script_name_to_file_name(name)) unique_path = file_utils.create_unique_filename(path, 100) LOGGER.info('Creating new script config "' + name + '" in ' + unique_path) self._save_config(config, unique_path)
def _preprocess_script_fields(self, config, original_config_json, uploaded_script, user): script_config = config.get('script') if not script_config: raise InvalidConfigException('script option is required') if SCRIPT_PATH_FIELD in config: del config[SCRIPT_PATH_FIELD] del config['script'] new_path = strip(script_config.get('path')) if is_blank(new_path): raise InvalidConfigException('script.path option is required') config[SCRIPT_PATH_FIELD] = new_path mode = script_config.get('mode') if is_blank(mode) or mode == SCRIPT_EDIT_PATH_MODE: pass elif mode in (SCRIPT_EDIT_UPLOAD_MODE, SCRIPT_EDIT_CODE_MODE): if not self._authorizer.can_edit_code(user.user_id): raise InvalidAccessException('User ' + str(user) + ' is not allowed to edit code') if mode == SCRIPT_EDIT_UPLOAD_MODE: if uploaded_script is None: raise InvalidConfigException( 'Uploaded script should be specified') if original_config_json is None: # new config if mode == SCRIPT_EDIT_UPLOAD_MODE: # escaped name is needed, when uploaded file and server has different OSes, # thus different special characters escaped_name = to_filename(uploaded_script.filename) target_path = os.path.join(self._scripts_folder, escaped_name) else: filename = os.path.basename(new_path) target_path = os.path.join( self._scripts_folder, _escape_characters_in_filename(filename)) script_path = file_utils.create_unique_filename( target_path, 100) config[SCRIPT_PATH_FIELD] = script_path else: existing_code = self._load_script_code_by_config( original_config_json) script_path = existing_code['file_path'] if (mode == SCRIPT_EDIT_CODE_MODE ) and existing_code.get('code_edit_error') is not None: raise InvalidConfigException( 'Failed to edit code: ' + existing_code.get('code_edit_error')) if new_path != original_config_json.get(SCRIPT_PATH_FIELD): raise InvalidConfigException( 'script.path override is not allowed for ' + mode + ' mode') if mode == SCRIPT_EDIT_UPLOAD_MODE: file_utils.write_file(script_path, uploaded_script.body, byte_content=True) else: code = script_config.get('code') if code is None: raise InvalidConfigException( 'script.code should be specified') file_utils.write_file(script_path, code) file_utils.make_executable(script_path) else: raise InvalidConfigException('Unsupported mode: ' + mode)