def setUp(self) -> None:
super().setUp()
test_utils.setup()
self.requires_explicit_ioloop_factory = os_utils.is_win(
) and env_utils.is_min_version('3.8')
self.windows = os_utils.is_win()
def setUp(self) -> None:
super().setUp()
test_utils.setup()
self.requires_explicit_ioloop_factory = os_utils.is_win(
) and env_utils.is_min_version('3.8')
self.windows = os_utils.is_win()
self.conf_folder = test_utils.create_dir(os.path.join('conf'))
self.runners_folder = os.path.join(self.conf_folder, 'runners')
def verify(self, username, password):
if username not in self.user_passwords:
LOGGER.warning('User ' + username + ' does not exist')
return False
existing_password = self.user_passwords.get(username)
# Selects encryption algorithm depending on the password format
# https://httpd.apache.org/docs/2.4/misc/password_encryptions.html
if existing_password.startswith('$2y$'):
import bcrypt
return bcrypt.checkpw(password.encode('utf8'),
existing_password.encode('utf8'))
elif existing_password.startswith('$apr1$'):
salt = existing_password[6:existing_password.find('$', 6)]
hashed_password = encryption_utils.md5_apr1(salt, password)
return hashed_password == existing_password
elif existing_password.startswith('{SHA}'):
expected = existing_password[5:]
hashed_password = encryption_utils.sha1(password)
return hashed_password == expected
elif not os_utils.is_win():
import crypt
hashed_password = crypt.crypt(password, existing_password[:2])
return hashed_password == existing_password
else:
return password == existing_password
def create_log_identifier(audit_name, script_name):
if os_utils.is_win():
audit_name = audit_name.replace(":", "-")
date_string = datetime.today().strftime("%y%m%d_%H%M%S")
script_name = script_name.replace(" ", "_")
log_identifier = script_name + "_" + audit_name + "_" + date_string
return log_identifier
def kill(self):
if not self.is_finished():
if not os_utils.is_win():
group_id = os.getpgid(self.get_process_id())
os.killpg(group_id, signal.SIGKILL)
self._write_script_output('\n>> KILLED\n')
else:
subprocess.Popen("taskkill /F /T /PID " + self.get_process_id())
def kill(self):
if not self.is_finished():
if not os_utils.is_win():
group_id = os.getpgid(self.get_process_id())
os.killpg(group_id, signal.SIGKILL)
self._write_script_output('\n>> KILLED\n')
else:
subprocess.Popen("taskkill /F /T /PID " + self.get_process_id())
def normalize_hostname(hostname):
if hostname == 'ip6-localhost':
return 'localhost'
if os_utils.is_win():
import platform
if hostname == platform.node():
return 'localhost'
return hostname
def find_matching_files(file_pattern, script_output):
files = []
separator = re.escape(os_utils.path_sep())
output_patterns = [file_pattern]
while len(output_patterns) > 0:
output_pattern = output_patterns.pop(0)
if '#' in output_pattern:
regex_start = output_pattern.find('#')
group_number_matches = re.findall('^#\d+#',
output_pattern[regex_start:])
if group_number_matches:
first_match = group_number_matches[0]
group_number = int(first_match[1:-1])
pattern_start = regex_start + len(first_match) - 1
else:
group_number = 0
pattern_start = regex_start
regex_end = output_pattern.find('#', pattern_start + 1)
while (regex_end >=
0) and output_pattern[regex_end:].startswith('#any_path'):
regex_end = output_pattern.find('#', regex_end + 1)
if regex_end >= 0:
regex_pattern = output_pattern[pattern_start + 1:regex_end]
if regex_pattern.startswith('#any_path') and (regex_start
== 0):
if os_utils.is_linux() or os_utils.is_mac():
regex_pattern = '~?' + regex_pattern
elif os_utils.is_win():
regex_pattern = '(([^\W\d_]:)|~)' + regex_pattern
regex_pattern = regex_pattern.replace(
'#any_path', '(' + separator + '([\w.\-]|(\\\ ))+)+')
found_matches = re.finditer(regex_pattern, script_output)
for match in found_matches:
matched_group = match.group(group_number)
new_output_pattern = string_utils.replace(
output_pattern, matched_group, regex_start, regex_end)
output_patterns.append(new_output_pattern)
continue
if '*' not in output_pattern:
files.append(output_pattern)
else:
recursive = '**' in output_pattern
matching_files = file_utils.search_glob(output_pattern,
recursive=recursive)
files.extend(matching_files)
return files
def normalize_hostname(hostname):
if hostname == 'ip6-localhost':
return 'localhost'
if os_utils.is_win():
import platform
if hostname == platform.node():
return 'localhost'
return hostname
def test_authenticate_failure_when_crypt_with_plain_password(self):
if self.verifier == 'htpasswd' and os_utils.is_win():
return
os_utils.set_linux()
authenticator = self._create_authenticator(
{'htpasswd_path': self.file_path})
for username in crypt_users.keys():
password = crypt_users[username]
self._assert_rejected(username, password, authenticator)
def test_authenticate_success_when_crypt(self):
if self.verifier == 'htpasswd' and os_utils.is_win():
return
os_utils.set_linux()
authenticator = self._create_authenticator(
{'htpasswd_path': self.file_path})
for username in crypt_users.keys():
password = username_passwords[username]
self._assert_authenticated(username, password, authenticator)
def find_matching_files(file_pattern, script_output):
files = []
separator = re.escape(os_utils.path_sep())
output_patterns = [file_pattern]
while len(output_patterns) > 0:
output_pattern = output_patterns.pop(0)
if '#' in output_pattern:
regex_start = output_pattern.find('#')
group_number_matches = re.findall('^#\d+#', output_pattern[regex_start:])
if group_number_matches:
first_match = group_number_matches[0]
group_number = int(first_match[1:-1])
pattern_start = regex_start + len(first_match) - 1
else:
group_number = 0
pattern_start = regex_start
regex_end = output_pattern.find('#', pattern_start + 1)
while (regex_end >= 0) and output_pattern[regex_end:].startswith('#any_path'):
regex_end = output_pattern.find('#', regex_end + 1)
if regex_end >= 0:
regex_pattern = output_pattern[pattern_start + 1:regex_end]
if regex_pattern.startswith('#any_path') and (regex_start == 0):
if os_utils.is_linux() or os_utils.is_mac():
regex_pattern = '~?' + regex_pattern
elif os_utils.is_win():
regex_pattern = '(([^\W\d_]:)|~)' + regex_pattern
regex_pattern = regex_pattern.replace('#any_path', '(' + separator + '([\w.\-]|(\\\ ))+)+')
found_matches = re.finditer(regex_pattern, script_output)
for match in found_matches:
matched_group = match.group(group_number)
new_output_pattern = string_utils.replace(output_pattern, matched_group, regex_start, regex_end)
output_patterns.append(new_output_pattern)
continue
if '*' not in output_pattern:
files.append(output_pattern)
else:
recursive = '**' in output_pattern
matching_files = file_utils.search_glob(output_pattern, recursive=recursive)
files.extend(matching_files)
return files
def start_execution(self, command, working_directory):
shell = False
if os_utils.is_win():
(command, shell) = prepare_cmd_for_win(command)
self.process = subprocess.Popen(command,
cwd=working_directory,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
start_new_session=True,
universal_newlines=True,
shell=shell)
def relative_path(path, parent_path):
path = normalize_path(path)
parent_path = normalize_path(parent_path)
if os_utils.is_win():
path = path.capitalize()
parent_path = parent_path.capitalize()
if not path.startswith(parent_path):
raise ValueError(path + ' is not subpath of ' + parent_path)
relative_path = path[len(parent_path):]
if relative_path.startswith(os.path.sep):
return relative_path[1:]
return relative_path
def split_command(script_command, working_directory=None):
if ' ' in script_command:
posix = not os_utils.is_win()
args = shlex.split(script_command, posix=posix)
if not posix:
args = [string_utils.unwrap_quotes(arg) for arg in args]
else:
args = [script_command]
script_path = file_utils.normalize_path(args[0], working_directory)
script_args = args[1:]
for i, body_arg in enumerate(script_args):
expanded = os.path.expanduser(body_arg)
if expanded != body_arg:
script_args[i] = expanded
return [script_path] + script_args
def stop(self):
if not self.is_finished():
if not os_utils.is_win():
group_id = os.getpgid(self.get_process_id())
os.killpg(group_id, signal.SIGTERM)
class KillChildren(object):
def finished(self):
try:
os.killpg(group_id, signal.SIGKILL)
except ProcessLookupError:
# probably there are no children left
pass
self.add_finish_listener(KillChildren())
else:
self.process.terminate()
self._write_script_output('\n>> STOPPED BY USER\n')
def stop(self):
if not self.is_finished():
if not os_utils.is_win():
group_id = os.getpgid(self._get_process_id())
os.killpg(group_id, signal.SIGTERM)
class KillChildren(object):
def finished(self):
try:
os.killpg(group_id, signal.SIGKILL)
except ProcessLookupError:
# probably there are no children left
pass
self.add_finish_listener(KillChildren())
else:
self.process.terminate()
self._write_script_output('\n>> STOPPED BY USER\n')
def split_command(script_command, working_directory=None):
if ' ' in script_command:
if _is_file_path(script_command, working_directory):
args = [script_command]
else:
posix = not os_utils.is_win()
args = shlex.split(script_command, posix=posix)
if not posix:
args = [string_utils.unwrap_quotes(arg) for arg in args]
else:
args = [script_command]
script_path = file_utils.normalize_path(args[0], working_directory)
if (not os.path.isabs(script_path)) or (not os.path.exists(script_path)):
script_path = args[0]
script_args = args[1:]
for i, body_arg in enumerate(script_args):
expanded = os.path.expanduser(body_arg)
if expanded != body_arg:
script_args[i] = expanded
return [script_path] + script_args
def to_filename(txt):
if os_utils.is_win():
return txt.replace(':', '-')
return txt
def init(server_config: ServerConfig,
authenticator,
authorizer,
execution_service: ExecutionService,
schedule_service: ScheduleService,
execution_logging_service: ExecutionLoggingService,
config_service: ConfigService,
alerts_service: AlertsService,
file_upload_feature: FileUploadFeature,
file_download_feature: FileDownloadFeature,
secret,
server_version,
conf_folder,
*,
start_server=True):
ssl_context = None
if server_config.is_ssl():
ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
ssl_context.load_cert_chain(server_config.get_ssl_cert_path(),
server_config.get_ssl_key_path())
auth = TornadoAuth(authenticator)
if auth.is_enabled():
identification = AuthBasedIdentification(auth)
else:
identification = IpBasedIdentification(server_config.ip_validator,
server_config.user_header_name)
downloads_folder = file_download_feature.get_result_files_folder()
handlers = [
(r'/conf', GetServerConf), (r'/scripts', GetScripts),
(r'/scripts/([^/]*)', ScriptConfigSocket),
(r'/scripts/([^/]*)/([^/]*)/list-files', ScriptParameterListFiles),
(r'/executions/start', ScriptExecute),
(r'/executions/stop/(.*)', ScriptStop),
(r'/executions/kill/(.*)', ScriptKill),
(r'/executions/io/(.*)', ScriptStreamSocket),
(r'/executions/active', GetActiveExecutionIds),
(r'/executions/config/(.*)', GetExecutingScriptConfig),
(r'/executions/cleanup/(.*)', CleanupExecutingScript),
(r'/executions/status/(.*)', GetExecutionStatus),
(r'/history/execution_log/short', GetShortHistoryEntriesHandler),
(r'/history/execution_log/long/(.*)', GetLongHistoryEntryHandler),
(r'/schedule', AddSchedule), (r'/auth/info', AuthInfoHandler),
(r'/result_files/(.*)', DownloadResultFile, {
'path': downloads_folder
}), (r'/admin/scripts', AdminUpdateScriptEndpoint),
(r'/admin/scripts/(.*)', AdminGetScriptEndpoint),
(r"/", ProxiedRedirectHandler, {
"url": "/index.html"
})
]
if auth.is_enabled():
handlers.append((r'/login', LoginHandler))
handlers.append((r'/auth/config', AuthConfigHandler))
handlers.append((r'/logout', LogoutHandler))
handlers.append((r'/theme/(.*)', ThemeStaticFileHandler, {
'path': os.path.join(conf_folder, 'theme')
}))
handlers.append((r"/(.*)", AuthorizedStaticFileHandler, {"path": "web"}))
settings = {
"cookie_secret": secret,
"login_url": "/login.html",
'websocket_ping_interval': 30,
'websocket_ping_timeout': 300,
'compress_response': True
}
application = tornado.web.Application(handlers, **settings)
application.auth = auth
application.server_config = server_config
application.server_version = server_version
application.authorizer = authorizer
application.downloads_folder = downloads_folder
application.file_download_feature = file_download_feature
application.file_upload_feature = file_upload_feature
application.execution_service = execution_service
application.schedule_service = schedule_service
application.execution_logging_service = execution_logging_service
application.config_service = config_service
application.alerts_service = alerts_service
application.identification = identification
application.max_request_size_mb = server_config.max_request_size_mb
if os_utils.is_win() and env_utils.is_min_version('3.8'):
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
io_loop = tornado.ioloop.IOLoop.current()
global _http_server
_http_server = httpserver.HTTPServer(application,
ssl_options=ssl_context,
max_buffer_size=10 * BYTES_IN_MB)
_http_server.listen(server_config.port, address=server_config.address)
intercept_stop_when_running_scripts(io_loop, execution_service)
http_protocol = 'https' if server_config.ssl else 'http'
print('Server is running on: %s://%s:%s' %
(http_protocol, server_config.address, server_config.port))
if start_server:
io_loop.start()
def to_filename(txt):
if os_utils.is_win():
return txt.replace(':', '-')
return txt
def to_filename(txt):
if os_utils.is_win():
return re.sub('[<>:"/\\\\|?*]', '_', txt)
return txt.replace('/', '_')
