def start_logging(self,
execution_id,
user_name,
user_id,
script_name,
command,
output_stream,
all_audit_names,
output_format,
start_time_millis=None):
if start_time_millis is None:
start_time_millis = get_current_millis()
log_filename = self._log_name_creator.create_filename(
execution_id, all_audit_names, script_name, start_time_millis)
log_file_path = os.path.join(self._output_folder, log_filename)
log_file_path = file_utils.create_unique_filename(log_file_path)
output_logger = ScriptOutputLogger(log_file_path, output_stream)
output_logger.write_line('id:' + execution_id)
output_logger.write_line('user_name:' + user_name)
output_logger.write_line('user_id:' + user_id)
output_logger.write_line('script:' + script_name)
output_logger.write_line('start_time:' + str(start_time_millis))
output_logger.write_line('command:' + command)
output_logger.write_line('output_format:' + output_format)
output_logger.write_line(OUTPUT_STARTED_MARKER)
output_logger.start()
log_filename = os.path.basename(log_file_path)
self._visited_files.add(log_filename)
self._ids_to_file_map[execution_id] = log_filename
self._output_loggers[execution_id] = output_logger
def start_logging(self, execution_id,
user_name,
user_id,
script_name,
command,
output_stream,
all_audit_names,
start_time_millis=None):
if start_time_millis is None:
start_time_millis = get_current_millis()
log_filename = self._log_name_creator.create_filename(
execution_id, all_audit_names, script_name, start_time_millis)
log_file_path = os.path.join(self._output_folder, log_filename)
log_file_path = file_utils.create_unique_filename(log_file_path)
output_logger = ScriptOutputLogger(log_file_path, output_stream)
output_logger.write_line('id:' + execution_id)
output_logger.write_line('user_name:' + user_name)
output_logger.write_line('user_id:' + user_id)
output_logger.write_line('script:' + script_name)
output_logger.write_line('start_time:' + str(start_time_millis))
output_logger.write_line('command:' + command)
output_logger.write_line(OUTPUT_STARTED_MARKER)
output_logger.start()
log_filename = os.path.basename(log_file_path)
self._visited_files.add(log_filename)
self._ids_to_file_map[execution_id] = log_filename
self._output_loggers[execution_id] = output_logger
def start_logging(self,
execution_id,
username,
script_name,
command,
output_stream,
post_execution_info_provider,
start_time_millis=None):
if start_time_millis is None:
start_time_millis = get_current_millis()
log_identifier = self._create_log_identifier(username, script_name,
start_time_millis)
log_file_path = os.path.join(self._output_folder,
log_identifier + '.log')
log_file_path = file_utils.create_unique_filename(log_file_path)
def write_post_execution_info():
self._write_post_execution_info(execution_id, log_file_path,
post_execution_info_provider)
output_logger = ScriptOutputLogger(log_file_path, output_stream,
write_post_execution_info)
output_logger.write_line('id:' + execution_id)
output_logger.write_line('user:'******'script:' + script_name)
output_logger.write_line('start_time:' + str(start_time_millis))
output_logger.write_line('command:' + command)
output_logger.write_line(OUTPUT_STARTED_MARKER)
output_logger.start()
log_filename = os.path.basename(log_file_path)
self._visited_files.add(log_filename)
self._ids_to_file_map[execution_id] = log_filename
def save_file(self, filename, body, username) -> str:
upload_folder = self.user_file_storage.prepare_new_folder(
username, self.folder)
pref_result_path = os.path.join(upload_folder, filename)
result_path = file_utils.create_unique_filename(pref_result_path)
file_utils.write_file(result_path, body, True)
return file_utils.normalize_path(result_path)
def prepare_downloadable_files(self, config, script_output,
script_param_values, audit_name):
output_files = config.output_files
if not output_files:
return []
output_files = substitute_parameter_values(config.parameters,
config.output_files,
script_param_values)
correct_files = []
for output_file in output_files:
files = find_matching_files(output_file, script_output)
if files:
for file in files:
file_path = file_utils.normalize_path(
file, config.get_working_directory())
if not os.path.exists(file_path):
LOGGER.warning('file ' + file + ' (full path = ' +
file_path + ') not found')
elif os.path.isdir(file_path):
LOGGER.warning('file ' + file +
' is a directory. Not allowed')
elif file_path not in correct_files:
correct_files.append(file_path)
else:
LOGGER.warning("Couldn't find file for " + output_file)
if not correct_files:
return []
download_folder = self.user_file_storage.prepare_new_folder(
audit_name, self.result_folder)
LOGGER.info('Created download folder for ' + audit_name + ': ' +
download_folder)
result = []
for file in correct_files:
preferred_download_file = os.path.join(download_folder,
os.path.basename(file))
try:
download_file = create_unique_filename(preferred_download_file)
except file_utils.FileExistsException:
LOGGER.exception('Cannot get unique name')
continue
copyfile(file, download_file)
result.append(download_file)
return result
def _prepare_downloadable_files(self,
output_files,
config,
script_output,
*,
should_exist=True):
found_files = {}
for output_file in output_files:
files = find_matching_files(output_file, script_output)
if files:
for file in files:
file_path = file_utils.normalize_path(
file, config.working_directory)
if not os.path.exists(file_path):
if should_exist:
LOGGER.warning('file ' + file + ' (full path = ' +
file_path + ') not found')
elif os.path.isdir(file_path):
LOGGER.warning('file ' + file +
' is a directory. Not allowed')
elif file_path not in found_files:
found_files[file] = file_path
elif should_exist:
LOGGER.warning("Couldn't find file for " + output_file)
if not found_files:
return {}
result = {}
for original_file_path, normalized_path in found_files.items():
if original_file_path in self.prepared_files:
result[original_file_path] = self.prepared_files[
original_file_path]
continue
preferred_download_file = os.path.join(
self.download_folder, os.path.basename(normalized_path))
try:
download_file = create_unique_filename(preferred_download_file)
except file_utils.FileExistsException:
LOGGER.exception('Cannot get unique name')
continue
copyfile(normalized_path, download_file)
result[original_file_path] = download_file
self.prepared_files[original_file_path] = download_file
return result
def _prepare_downloadable_files(self, config, script_output, script_param_values, execution_owner):
output_files = config.output_files
if not output_files:
return []
output_files = substitute_parameter_values(
config.parameters,
config.output_files,
script_param_values)
correct_files = []
for output_file in output_files:
files = find_matching_files(output_file, script_output)
if files:
for file in files:
file_path = file_utils.normalize_path(file, config.working_directory)
if not os.path.exists(file_path):
LOGGER.warning('file ' + file + ' (full path = ' + file_path + ') not found')
elif os.path.isdir(file_path):
LOGGER.warning('file ' + file + ' is a directory. Not allowed')
elif file_path not in correct_files:
correct_files.append(file_path)
else:
LOGGER.warning("Couldn't find file for " + output_file)
if not correct_files:
return []
download_folder = self.user_file_storage.prepare_new_folder(execution_owner, self.result_folder)
LOGGER.info('Created download folder for ' + execution_owner + ': ' + download_folder)
result = []
for file in correct_files:
preferred_download_file = os.path.join(download_folder, os.path.basename(file))
try:
download_file = create_unique_filename(preferred_download_file)
except file_utils.FileExistsException:
LOGGER.exception('Cannot get unique name')
continue
copyfile(file, download_file)
result.append(download_file)
return result
def create_config(self, user, config):
self._check_admin_access(user)
_preprocess_incoming_config(config)
name = config['name']
(short_config, path, json_object) = self._find_config(name)
if path is not None:
raise InvalidConfigException('Another config with the same name already exists')
path = os.path.join(self._script_configs_folder, _script_name_to_file_name(name))
unique_path = file_utils.create_unique_filename(path, 100)
LOGGER.info('Creating new script config "' + name + '" in ' + unique_path)
self._save_config(config, unique_path)
def __init__(self, header, files_path):
header_str = header.decode('utf-8')
if '\r\n' in header_str:
header_str = header_str[:header_str.index('\r\n')]
(value, sub_headers) = parse_header(header_str)
self.name = sub_headers['name']
self.filename = sub_headers.get('filename')
if self.filename:
self.path = os.path.join(files_path, self.filename)
self.path = file_utils.create_unique_filename(self.path)
# touch file
open(self.path, 'w').close()
else:
self.value = ''
def __init__(self, header, files_path):
header_str = header.decode('utf-8')
if '\r\n' in header_str:
header_str = header_str[:header_str.index('\r\n')]
(value, sub_headers) = parse_header(header_str)
self.name = sub_headers['name']
self.filename = sub_headers.get('filename')
if self.filename:
self.path = os.path.join(files_path, self.filename)
self.path = file_utils.create_unique_filename(self.path)
# touch file
open(self.path, 'w').close()
else:
self.value = ''
def create_config(self, user, config, uploaded_script):
self._check_admin_access(user)
_preprocess_incoming_config(config)
name = config['name']
search_result = self._find_config(name)
if search_result is not None:
raise InvalidConfigException(
'Another config with the same name already exists')
self._preprocess_script_fields(config, None, uploaded_script, user)
path = os.path.join(self._script_configs_folder,
_script_name_to_file_name(name))
unique_path = file_utils.create_unique_filename(path, 100)
LOGGER.info('Creating new script config "' + name + '" in ' +
unique_path)
self._save_config(config, unique_path)
def _preprocess_script_fields(self, config, original_config_json,
uploaded_script, user):
script_config = config.get('script')
if not script_config:
raise InvalidConfigException('script option is required')
if SCRIPT_PATH_FIELD in config:
del config[SCRIPT_PATH_FIELD]
del config['script']
new_path = strip(script_config.get('path'))
if is_blank(new_path):
raise InvalidConfigException('script.path option is required')
config[SCRIPT_PATH_FIELD] = new_path
mode = script_config.get('mode')
if is_blank(mode) or mode == SCRIPT_EDIT_PATH_MODE:
pass
elif mode in (SCRIPT_EDIT_UPLOAD_MODE, SCRIPT_EDIT_CODE_MODE):
if not self._authorizer.can_edit_code(user.user_id):
raise InvalidAccessException('User ' + str(user) +
' is not allowed to edit code')
if mode == SCRIPT_EDIT_UPLOAD_MODE:
if uploaded_script is None:
raise InvalidConfigException(
'Uploaded script should be specified')
if original_config_json is None: # new config
if mode == SCRIPT_EDIT_UPLOAD_MODE:
# escaped name is needed, when uploaded file and server has different OSes,
# thus different special characters
escaped_name = to_filename(uploaded_script.filename)
target_path = os.path.join(self._scripts_folder,
escaped_name)
else:
filename = os.path.basename(new_path)
target_path = os.path.join(
self._scripts_folder,
_escape_characters_in_filename(filename))
script_path = file_utils.create_unique_filename(
target_path, 100)
config[SCRIPT_PATH_FIELD] = script_path
else:
existing_code = self._load_script_code_by_config(
original_config_json)
script_path = existing_code['file_path']
if (mode == SCRIPT_EDIT_CODE_MODE
) and existing_code.get('code_edit_error') is not None:
raise InvalidConfigException(
'Failed to edit code: ' +
existing_code.get('code_edit_error'))
if new_path != original_config_json.get(SCRIPT_PATH_FIELD):
raise InvalidConfigException(
'script.path override is not allowed for ' + mode +
' mode')
if mode == SCRIPT_EDIT_UPLOAD_MODE:
file_utils.write_file(script_path,
uploaded_script.body,
byte_content=True)
else:
code = script_config.get('code')
if code is None:
raise InvalidConfigException(
'script.code should be specified')
file_utils.write_file(script_path, code)
file_utils.make_executable(script_path)
else:
raise InvalidConfigException('Unsupported mode: ' + mode)
