def start_server(self,
port,
address,
*,
xsrf_protection=XSRF_PROTECTION_TOKEN):
file_download_feature = FileDownloadFeature(
UserFileStorage(b'some_secret'), test_utils.temp_folder)
config = ServerConfig()
config.port = port
config.address = address
config.xsrf_protection = xsrf_protection
config.max_request_size_mb = 1
authorizer = Authorizer(ANY_USER, [], [], EmptyGroupProvider())
execution_service = MagicMock()
execution_service.start_script.return_value = 3
server.init(config,
None,
authorizer,
execution_service,
MagicMock(),
MagicMock(),
ConfigService(authorizer, self.conf_folder),
MagicMock(),
FileUploadFeature(UserFileStorage(b'cookie_secret'),
test_utils.temp_folder),
file_download_feature,
'cookie_secret',
None,
self.conf_folder,
start_server=False)
self.start_loop()
class TestUserFileStorage(unittest.TestCase):
def setUp(self):
test_utils.setup()
self.__storage = UserFileStorage(b'12345678')
self.upload_feature = FileUploadFeature(self.__storage,
test_utils.temp_folder)
def tearDown(self):
test_utils.cleanup()
self.__storage._stop_autoclean()
def test_create_file(self):
file_path = self.upload_feature.save_file('my_file.txt', b'test text',
'userX')
self.assertTrue(os.path.exists(file_path))
def test_content_in_created_file(self):
file_path = self.upload_feature.save_file('my_file.txt', b'My text',
'userX')
content = file_utils.read_file(file_path)
self.assertEqual('My text', content)
def test_same_filename(self):
file_path1 = self.upload_feature.save_file('my_file.txt', b'some text',
'userX')
file_path2 = self.upload_feature.save_file('my_file.txt', b'some text',
'userX')
self.assertNotEqual(file_path1, file_path2)
def setUp(self):
super().setUp()
test_utils.setup()
executor._process_creator = _MockProcessWrapper
self.executor_service = ExecutionService(AnyUserAuthorizer(), _IdGeneratorMock())
self.feature = FileDownloadFeature(UserFileStorage(b'123456'), test_utils.temp_folder)
self.feature.subscribe(self.executor_service)
def start_server(self,
port,
address,
*,
xsrf_protection=XSRF_PROTECTION_TOKEN):
file_download_feature = FileDownloadFeature(
UserFileStorage(b'some_secret'), test_utils.temp_folder)
config = ServerConfig()
config.port = port
config.address = address
config.xsrf_protection = xsrf_protection
config.max_request_size_mb = 1
authorizer = Authorizer(ANY_USER, ['admin_user'], [], ['admin_user'],
EmptyGroupProvider())
execution_service = MagicMock()
execution_service.start_script.return_value = 3
cookie_secret = b'cookie_secret'
server.init(config,
MockAuthenticator(),
authorizer,
execution_service,
MagicMock(),
MagicMock(),
ConfigService(authorizer, self.conf_folder),
MagicMock(),
FileUploadFeature(UserFileStorage(cookie_secret),
test_utils.temp_folder),
file_download_feature,
'cookie_secret',
None,
self.conf_folder,
start_server=False)
self.start_loop()
self._user_session = requests.Session()
self._user_session.cookies['username'] = create_signed_value(cookie_secret, 'username', 'normal_user') \
.decode('utf8')
self._admin_session = requests.Session()
self._admin_session.cookies['username'] = create_signed_value(cookie_secret, 'username', 'admin_user') \
.decode('utf8')
def setUp(self) -> None:
test_utils.setup()
executor._process_creator = _MockProcessWrapper
self.executor_service = ExecutionService(_IdGeneratorMock())
self.file_download_feature = file_download_feature.FileDownloadFeature(
UserFileStorage(b'123456'), test_utils.temp_folder)
self.file_download_feature.subscribe(self.executor_service)
self.images = []
def main():
tool_utils.validate_web_imports_exist(os.getcwd())
logging_conf_file = os.path.join(CONFIG_FOLDER, 'logging.json')
with open(logging_conf_file, 'rt') as f:
log_config = json.load(f)
file_utils.prepare_folder(os.path.join('logs'))
logging.config.dictConfig(log_config)
file_utils.prepare_folder(CONFIG_FOLDER)
file_utils.prepare_folder(TEMP_FOLDER)
migrations.migrate.migrate(TEMP_FOLDER, CONFIG_FOLDER)
server_config = server_conf.from_json(SERVER_CONF_PATH)
secret = get_secret(TEMP_FOLDER)
config_service = ConfigService(CONFIG_FOLDER)
alerts_service = AlertsService(server_config.get_alerts_config())
alerts_service = alerts_service
execution_logs_path = os.path.join('logs', 'processes')
log_name_creator = LogNameCreator(
server_config.logging_config.filename_pattern,
server_config.logging_config.date_format)
execution_logging_service = ExecutionLoggingService(
execution_logs_path, log_name_creator)
existing_ids = [
entry.id for entry in execution_logging_service.get_history_entries()
]
id_generator = IdGenerator(existing_ids)
execution_service = ExecutionService(id_generator)
execution_logging_initiator = ExecutionLoggingInitiator(
execution_service, execution_logging_service)
execution_logging_initiator.start()
user_file_storage = UserFileStorage(secret)
file_download_feature = FileDownloadFeature(user_file_storage, TEMP_FOLDER)
file_download_feature.subscribe(execution_service)
file_upload_feature = FileUploadFeature(user_file_storage, TEMP_FOLDER)
alerter_feature = FailAlerterFeature(execution_service, alerts_service)
alerter_feature.start()
server.init(server_config, execution_service, execution_logging_service,
config_service, alerts_service, file_upload_feature,
file_download_feature, secret)
class TestUserFileStorage(unittest.TestCase):
def setUp(self):
test_utils.setup()
self.__storage = UserFileStorage(b'12345678')
self.upload_feature = FileUploadFeature(self.__storage, test_utils.temp_folder)
def tearDown(self):
test_utils.cleanup()
self.__storage._stop_autoclean()
def test_prepare_new_folder(self):
file_path = self.upload_feature.prepare_new_folder('userX')
self.assertTrue(os.path.exists(file_path))
def test_prepare_new_folder_different_users(self):
path1 = self.upload_feature.prepare_new_folder('userX')
path2 = self.upload_feature.prepare_new_folder('userY')
self.assertNotEqual(path1, path2)
def test_prepare_new_folder_twice(self):
file_path1 = self.upload_feature.prepare_new_folder('userX')
time.sleep(0.1)
file_path2 = self.upload_feature.prepare_new_folder('userX')
self.assertNotEqual(file_path1, file_path2)
def start_server(self, port, address):
file_download_feature = FileDownloadFeature(UserFileStorage(b'123456'),
test_utils.temp_folder)
config = ServerConfig()
config.port = port
config.address = address
server.init(config,
None,
None,
None,
None,
None,
None,
None,
file_download_feature,
None,
None,
start_server=False)
self.start_loop()
def start_server(self, port, address):
file_download_feature = FileDownloadFeature(
UserFileStorage(b'some_secret'), test_utils.temp_folder)
config = ServerConfig()
config.port = port
config.address = address
authorizer = Authorizer(ANY_USER, [], [], EmptyGroupProvider())
server.init(config,
None,
authorizer,
None,
None,
ConfigService(authorizer, self.conf_folder),
None,
None,
file_download_feature,
'cookie_secret',
None,
start_server=False)
self.start_loop()
class TestUserFileStorage(unittest.TestCase):
def setUp(self):
test_utils.setup()
self.storage = UserFileStorage(b'12345678')
def tearDown(self):
test_utils.cleanup()
self.storage._stop_autoclean()
def test_create_folder(self):
user_folder = self.storage.prepare_new_folder('me', test_utils.temp_folder)
self.assertTrue(os.path.exists(user_folder))
def test_create_folder_for_different_users(self):
user1_folder = self.storage.prepare_new_folder('user1', test_utils.temp_folder)
user2_folder = self.storage.prepare_new_folder('user2', test_utils.temp_folder)
self.assertNotEqual(user1_folder, user2_folder)
def test_create_different_folders_for_a_user(self):
folder1 = self.storage.prepare_new_folder('me', test_utils.temp_folder)
time.sleep(0.002)
folder2 = self.storage.prepare_new_folder('me', test_utils.temp_folder)
self.assertNotEqual(folder1, folder2)
def test_autoclean_folder(self):
folder = self.storage.prepare_new_folder('me', test_utils.temp_folder)
self.storage.start_autoclean(test_utils.temp_folder, 2)
self.assertTrue(os.path.exists(folder))
time.sleep(0.005)
self.assertFalse(os.path.exists(folder))
def test_allow_to_access_own_folder(self):
user1_folder = self.storage.prepare_new_folder('user1', test_utils.temp_folder)
relative_folder = file_utils.relative_path(user1_folder, test_utils.temp_folder)
self.assertTrue(self.storage.allowed_to_access(relative_folder, 'user1'))
def test_prohibit_access_to_different_user(self):
user1_folder = self.storage.prepare_new_folder('user1', test_utils.temp_folder)
relative_folder = file_utils.relative_path(user1_folder, test_utils.temp_folder)
self.assertFalse(self.storage.allowed_to_access(relative_folder, 'user2'))
def setUp(self):
test_utils.setup()
self.storage = UserFileStorage(b'12345678')
def main():
tool_utils.validate_web_imports_exist(os.getcwd())
logging_conf_file = os.path.join(CONFIG_FOLDER, 'logging.json')
with open(logging_conf_file, "rt") as f:
log_config = json.load(f)
file_utils.prepare_folder(os.path.join("logs", "processes"))
logging.config.dictConfig(log_config)
file_utils.prepare_folder(CONFIG_FOLDER)
file_utils.prepare_folder(SCRIPT_CONFIGS_FOLDER)
server_config = server_conf.from_json(SERVER_CONF_PATH)
ssl_context = None
if server_config.is_ssl():
ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
ssl_context.load_cert_chain(server_config.get_ssl_cert_path(),
server_config.get_ssl_key_path())
file_utils.prepare_folder(TEMP_FOLDER)
settings = {
"cookie_secret": get_tornado_secret(),
"login_url": "/login.html"
}
auth = TornadoAuth(server_config.authenticator, server_config.authorizer)
user_file_storage = UserFileStorage(get_tornado_secret())
file_download_feature = FileDownloadFeature(user_file_storage, TEMP_FOLDER)
file_upload_feature = FileUploadFeature(user_file_storage, TEMP_FOLDER)
result_files_folder = file_download_feature.get_result_files_folder()
handlers = [(r"/conf/title", GetServerTitle),
(r"/scripts/list", GetScripts),
(r"/scripts/info", GetScriptInfo),
(r"/scripts/execute", ScriptExecute),
(r"/scripts/execute/stop", ScriptStop),
(r"/scripts/execute/io/(.*)", ScriptStreamSocket),
(r'/' + os.path.basename(result_files_folder) + '/(.*)',
DownloadResultFile, {
'path': result_files_folder
}), (r"/", ProxiedRedirectHandler, {
"url": "/index.html"
})]
if auth.is_enabled():
handlers.append((r'/login', LoginHandler))
handlers.append((r'/auth/config', AuthConfigHandler))
handlers.append((r'/logout', LogoutHandler))
handlers.append((r"/username", GetUsernameHandler))
handlers.append((r"/(.*)", AuthorizedStaticFileHandler, {"path": "web"}))
application = tornado.web.Application(handlers, **settings)
application.auth = auth
application.alerts_config = server_config.get_alerts_config()
application.server_title = server_config.title
application.file_download_feature = file_download_feature
application.file_upload_feature = file_upload_feature
http_server = httpserver.HTTPServer(application, ssl_options=ssl_context)
http_server.listen(server_config.port, address=server_config.address)
http_protocol = 'https' if server_config.ssl else 'http'
print('Server is running on: %s://%s:%s' %
(http_protocol, server_config.address, server_config.port))
tornado.ioloop.IOLoop.current().start()
def main():
try:
tool_utils.validate_web_build_exists(os.getcwd())
except InvalidWebBuildException as e:
print(str(e))
sys.exit(-1)
logging_conf_file = os.path.join(CONFIG_FOLDER, 'logging.json')
with open(logging_conf_file, 'rt') as f:
log_config = json.load(f)
file_utils.prepare_folder(LOG_FOLDER)
logging.config.dictConfig(log_config)
file_utils.prepare_folder(CONFIG_FOLDER)
file_utils.prepare_folder(TEMP_FOLDER)
migrations.migrate.migrate(TEMP_FOLDER, CONFIG_FOLDER, SERVER_CONF_PATH,
LOG_FOLDER)
server_config = server_conf.from_json(SERVER_CONF_PATH, TEMP_FOLDER)
secret = get_secret(TEMP_FOLDER)
tornado_client_config.initialize()
group_provider = create_group_provider(server_config.user_groups,
server_config.authenticator,
server_config.admin_users)
authorizer = Authorizer(server_config.allowed_users,
server_config.admin_users, group_provider)
config_service = ConfigService(authorizer, CONFIG_FOLDER)
alerts_service = AlertsService(server_config.get_alerts_config())
alerts_service = alerts_service
execution_logs_path = os.path.join(LOG_FOLDER, 'processes')
log_name_creator = LogNameCreator(
server_config.logging_config.filename_pattern,
server_config.logging_config.date_format)
execution_logging_service = ExecutionLoggingService(
execution_logs_path, log_name_creator)
existing_ids = [
entry.id for entry in execution_logging_service.get_history_entries()
]
id_generator = IdGenerator(existing_ids)
execution_service = ExecutionService(id_generator)
execution_logging_initiator = ExecutionLoggingInitiator(
execution_service, execution_logging_service)
execution_logging_initiator.start()
user_file_storage = UserFileStorage(secret)
file_download_feature = FileDownloadFeature(user_file_storage, TEMP_FOLDER)
file_download_feature.subscribe(execution_service)
file_upload_feature = FileUploadFeature(user_file_storage, TEMP_FOLDER)
alerter_feature = FailAlerterFeature(execution_service, alerts_service)
alerter_feature.start()
server.init(server_config, server_config.authenticator, authorizer,
execution_service, execution_logging_service, config_service,
alerts_service, file_upload_feature, file_download_feature,
secret)
def setUp(self):
test_utils.setup()
self.__storage = UserFileStorage(b'12345678')
self.upload_feature = FileUploadFeature(self.__storage, test_utils.temp_folder)
def __init__(self, user_file_storage: UserFileStorage,
temp_folder) -> None:
self.user_file_storage = user_file_storage
self.folder = os.path.join(temp_folder, RESULT_FILES_FOLDER)
user_file_storage.start_autoclean(self.folder, 1000 * 60 * 60 * 24)
def main():
project_path = os.getcwd()
try:
tool_utils.validate_web_build_exists(project_path)
except InvalidWebBuildException as e:
print(str(e))
sys.exit(-1)
logging_conf_file = os.path.join(CONFIG_FOLDER, 'logging.json')
with open(logging_conf_file, 'rt') as f:
log_config = json.load(f)
file_utils.prepare_folder(LOG_FOLDER)
logging.config.dictConfig(log_config)
server_version = tool_utils.get_server_version(project_path)
logging.info('Starting Script Server' +
(', v' +
server_version if server_version else ' (custom version)'))
file_utils.prepare_folder(CONFIG_FOLDER)
file_utils.prepare_folder(TEMP_FOLDER)
migrations.migrate.migrate(TEMP_FOLDER, CONFIG_FOLDER, SERVER_CONF_PATH,
LOG_FOLDER)
server_config = server_conf.from_json(SERVER_CONF_PATH, TEMP_FOLDER)
secret = get_secret(server_config.secret_storage_file)
tornado_client_config.initialize()
group_provider = create_group_provider(server_config.user_groups,
server_config.authenticator,
server_config.admin_users)
authorizer = Authorizer(server_config.allowed_users,
server_config.admin_users,
server_config.full_history_users,
server_config.code_editor_users, group_provider)
config_service = ConfigService(authorizer, CONFIG_FOLDER)
alerts_service = AlertsService(server_config.alerts_config)
alerts_service = alerts_service
execution_logs_path = os.path.join(LOG_FOLDER, 'processes')
log_name_creator = LogNameCreator(
server_config.logging_config.filename_pattern,
server_config.logging_config.date_format)
execution_logging_service = ExecutionLoggingService(
execution_logs_path, log_name_creator, authorizer)
existing_ids = [
entry.id for entry in execution_logging_service.get_history_entries(
None, system_call=True)
]
id_generator = IdGenerator(existing_ids)
execution_service = ExecutionService(authorizer, id_generator)
execution_logging_controller = ExecutionLoggingController(
execution_service, execution_logging_service)
execution_logging_controller.start()
user_file_storage = UserFileStorage(secret)
file_download_feature = FileDownloadFeature(user_file_storage, TEMP_FOLDER)
file_download_feature.subscribe(execution_service)
file_upload_feature = FileUploadFeature(user_file_storage, TEMP_FOLDER)
alerter_feature = FailAlerterFeature(execution_service, alerts_service)
alerter_feature.start()
executions_callback_feature = ExecutionsCallbackFeature(
execution_service, server_config.callbacks_config)
executions_callback_feature.start()
schedule_service = ScheduleService(config_service, execution_service,
CONFIG_FOLDER)
server.init(server_config, server_config.authenticator, authorizer,
execution_service, schedule_service, execution_logging_service,
config_service, alerts_service, file_upload_feature,
file_download_feature, secret, server_version, CONFIG_FOLDER)
def setUp(self):
test_utils.setup()
self.storage = UserFileStorage(b'12345678')
class TestUserFileStorage(unittest.TestCase):
def setUp(self):
test_utils.setup()
self.storage = UserFileStorage(b'12345678')
def tearDown(self):
test_utils.cleanup()
self.storage._stop_autoclean()
def test_create_folder(self):
user_folder = self.storage.prepare_new_folder('me', test_utils.temp_folder)
self.assertTrue(os.path.exists(user_folder))
def test_create_folder_for_different_users(self):
user1_folder = self.storage.prepare_new_folder('user1', test_utils.temp_folder)
user2_folder = self.storage.prepare_new_folder('user2', test_utils.temp_folder)
self.assertNotEqual(user1_folder, user2_folder)
def test_create_different_folders_for_a_user(self):
folder1 = self.storage.prepare_new_folder('me', test_utils.temp_folder)
time.sleep(0.002)
folder2 = self.storage.prepare_new_folder('me', test_utils.temp_folder)
self.assertNotEqual(folder1, folder2)
def test_autoclean_folder(self):
folder = self.storage.prepare_new_folder('me', test_utils.temp_folder)
self.storage.start_autoclean(test_utils.temp_folder, 2)
self.assertTrue(os.path.exists(folder))
time.sleep(0.020)
self.assertFalse(os.path.exists(folder))
def test_allow_to_access_own_folder(self):
user1_folder = self.storage.prepare_new_folder('user1', test_utils.temp_folder)
relative_folder = file_utils.relative_path(user1_folder, test_utils.temp_folder)
self.assertTrue(self.storage.allowed_to_access(relative_folder, 'user1'))
def test_prohibit_access_to_different_user(self):
user1_folder = self.storage.prepare_new_folder('user1', test_utils.temp_folder)
relative_folder = file_utils.relative_path(user1_folder, test_utils.temp_folder)
self.assertFalse(self.storage.allowed_to_access(relative_folder, 'user2'))
