def main():
tool_utils.validate_web_imports_exist(os.getcwd())
logging_conf_file = os.path.join(CONFIG_FOLDER, 'logging.json')
with open(logging_conf_file, 'rt') as f:
log_config = json.load(f)
file_utils.prepare_folder(os.path.join('logs'))
logging.config.dictConfig(log_config)
file_utils.prepare_folder(CONFIG_FOLDER)
file_utils.prepare_folder(TEMP_FOLDER)
migrations.migrate.migrate(TEMP_FOLDER, CONFIG_FOLDER)
server_config = server_conf.from_json(SERVER_CONF_PATH)
secret = get_secret(TEMP_FOLDER)
config_service = ConfigService(CONFIG_FOLDER)
alerts_service = AlertsService(server_config.get_alerts_config())
alerts_service = alerts_service
execution_logs_path = os.path.join('logs', 'processes')
log_name_creator = LogNameCreator(
server_config.logging_config.filename_pattern,
server_config.logging_config.date_format)
execution_logging_service = ExecutionLoggingService(
execution_logs_path, log_name_creator)
existing_ids = [
entry.id for entry in execution_logging_service.get_history_entries()
]
id_generator = IdGenerator(existing_ids)
execution_service = ExecutionService(id_generator)
execution_logging_initiator = ExecutionLoggingInitiator(
execution_service, execution_logging_service)
execution_logging_initiator.start()
user_file_storage = UserFileStorage(secret)
file_download_feature = FileDownloadFeature(user_file_storage, TEMP_FOLDER)
file_download_feature.subscribe(execution_service)
file_upload_feature = FileUploadFeature(user_file_storage, TEMP_FOLDER)
alerter_feature = FailAlerterFeature(execution_service, alerts_service)
alerter_feature.start()
server.init(server_config, execution_service, execution_logging_service,
config_service, alerts_service, file_upload_feature,
file_download_feature, secret)
class FileDownloadFeatureTest(unittest.TestCase):
def test_prepare_file_on_finish(self):
file1_path = test_utils.create_file('file1.txt')
downloadable_files = self.perform_execution([file1_path])
self.assert_downloadable_files(downloadable_files, [file1_path])
def test_no_output_files_in_config(self):
test_utils.create_file('file1.txt')
downloadable_files = self.perform_execution(None)
self.assertEqual([], downloadable_files)
def test_output_files_with_parameter_substitution(self):
file1 = test_utils.create_file('file1.txt', text='hello world')
file2 = test_utils.create_file(os.path.join('sub', 'child',
'admin.log'),
text='password=123')
downloadable_files = self.perform_execution([
os.path.join(test_utils.temp_folder, '${p1}.txt'),
os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')
],
parameter_values={
'p1': 'file1',
'p2': 'child'
})
self.assert_downloadable_files(downloadable_files, [file1, file2])
def test_output_files_with_secure_parameters(self):
test_utils.create_file('file1.txt', text='hello world')
file2 = test_utils.create_file(os.path.join('sub', 'child',
'admin.log'),
text='password=123')
param1 = create_script_param_config('p1', secure=True)
param2 = create_script_param_config('p2')
downloadable_files = self.perform_execution([
os.path.join(test_utils.temp_folder, '${p1}.txt'),
os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')
],
parameter_values={
'p1': 'file1',
'p2': 'child'
},
parameters=[
param1, param2
])
self.assert_downloadable_files(downloadable_files, [file2])
def perform_execution(self,
output_files,
parameter_values=None,
parameters=None):
if parameter_values is None:
parameter_values = {}
if parameters is None:
parameters = [
create_script_param_config(key)
for key in parameter_values.keys()
]
config_model = create_config_model('my_script',
output_files=output_files,
parameters=parameters)
execution_id = self.executor_service.start_script(
config_model, parameter_values, 'userX',
create_audit_names(ip='127.0.0.1'))
self.executor_service.stop_script(execution_id)
finish_condition = threading.Event()
self.executor_service.add_finish_listener(
lambda: finish_condition.set(), execution_id)
finish_condition.wait(2)
downloadable_files = self.feature.get_downloadable_files(execution_id)
return downloadable_files
def setUp(self):
super().setUp()
test_utils.setup()
executor._process_creator = _MockProcessWrapper
self.executor_service = ExecutionService(_IdGeneratorMock())
self.feature = FileDownloadFeature(UserFileStorage(b'123456'),
test_utils.temp_folder)
self.feature.subscribe(self.executor_service)
def tearDown(self):
super().tearDown()
test_utils.cleanup()
def assert_downloadable_files(self, prepared_files, original_files):
prepared_files_dict = {os.path.basename(f): f for f in prepared_files}
original_files_dict = {os.path.basename(f): f for f in original_files}
self.assertEqual(original_files_dict.keys(),
prepared_files_dict.keys())
for filename in original_files_dict.keys():
prepared_file = prepared_files_dict[filename]
original_file = original_files_dict[filename]
prepared_content = file_utils.read_file(prepared_file)
original_content = file_utils.read_file(original_file)
self.assertEqual(original_content, prepared_content,
'Different content for file ' + filename)
class FileDownloadFeatureTest(unittest.TestCase):
def test_prepare_file_on_finish(self):
file1_path = test_utils.create_file('file1.txt')
downloadable_files = self.perform_execution([file1_path])
self.assert_downloadable_files(downloadable_files, [file1_path])
def test_no_output_files_in_config(self):
test_utils.create_file('file1.txt')
downloadable_files = self.perform_execution(None)
self.assertEqual([], downloadable_files)
def test_output_files_with_parameter_substitution(self):
file1 = test_utils.create_file('file1.txt', text='hello world')
file2 = test_utils.create_file(os.path.join('sub', 'child', 'admin.log'), text='password=123')
downloadable_files = self.perform_execution([
os.path.join(test_utils.temp_folder, '${p1}.txt'),
os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')],
parameter_values={'p1': 'file1', 'p2': 'child'})
self.assert_downloadable_files(downloadable_files, [file1, file2])
def test_output_files_with_secure_parameters(self):
test_utils.create_file('file1.txt', text='hello world')
file2 = test_utils.create_file(os.path.join('sub', 'child', 'admin.log'), text='password=123')
param1 = create_script_param_config('p1', secure=True)
param2 = create_script_param_config('p2')
downloadable_files = self.perform_execution([
os.path.join(test_utils.temp_folder, '${p1}.txt'),
os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')],
parameter_values={'p1': 'file1', 'p2': 'child'},
parameters=[param1, param2])
self.assert_downloadable_files(downloadable_files, [file2])
def perform_execution(self, output_files, parameter_values=None, parameters=None):
if parameter_values is None:
parameter_values = {}
if parameters is None:
parameters = [create_script_param_config(key) for key in parameter_values.keys()]
config_model = create_config_model('my_script', output_files=output_files, parameters=parameters)
execution_id = self.executor_service.start_script(
config_model, parameter_values, 'userX', create_audit_names(ip='127.0.0.1'))
self.executor_service.stop_script(execution_id)
finish_condition = threading.Event()
self.executor_service.add_finish_listener(lambda: finish_condition.set(), execution_id)
finish_condition.wait(2)
downloadable_files = self.feature.get_downloadable_files(execution_id)
return downloadable_files
def setUp(self):
super().setUp()
test_utils.setup()
executor._process_creator = _MockProcessWrapper
self.executor_service = ExecutionService(_IdGeneratorMock())
self.feature = FileDownloadFeature(UserFileStorage(b'123456'), test_utils.temp_folder)
self.feature.subscribe(self.executor_service)
def tearDown(self):
super().tearDown()
test_utils.cleanup()
def assert_downloadable_files(self, prepared_files, original_files):
prepared_files_dict = {os.path.basename(f): f for f in prepared_files}
original_files_dict = {os.path.basename(f): f for f in original_files}
self.assertEqual(original_files_dict.keys(), prepared_files_dict.keys())
for filename in original_files_dict.keys():
prepared_file = prepared_files_dict[filename]
original_file = original_files_dict[filename]
prepared_content = file_utils.read_file(prepared_file)
original_content = file_utils.read_file(original_file)
self.assertEqual(original_content, prepared_content, 'Different content for file ' + filename)
def main():
try:
tool_utils.validate_web_build_exists(os.getcwd())
except InvalidWebBuildException as e:
print(str(e))
sys.exit(-1)
logging_conf_file = os.path.join(CONFIG_FOLDER, 'logging.json')
with open(logging_conf_file, 'rt') as f:
log_config = json.load(f)
file_utils.prepare_folder(LOG_FOLDER)
logging.config.dictConfig(log_config)
file_utils.prepare_folder(CONFIG_FOLDER)
file_utils.prepare_folder(TEMP_FOLDER)
migrations.migrate.migrate(TEMP_FOLDER, CONFIG_FOLDER, SERVER_CONF_PATH,
LOG_FOLDER)
server_config = server_conf.from_json(SERVER_CONF_PATH, TEMP_FOLDER)
secret = get_secret(TEMP_FOLDER)
tornado_client_config.initialize()
group_provider = create_group_provider(server_config.user_groups,
server_config.authenticator,
server_config.admin_users)
authorizer = Authorizer(server_config.allowed_users,
server_config.admin_users, group_provider)
config_service = ConfigService(authorizer, CONFIG_FOLDER)
alerts_service = AlertsService(server_config.get_alerts_config())
alerts_service = alerts_service
execution_logs_path = os.path.join(LOG_FOLDER, 'processes')
log_name_creator = LogNameCreator(
server_config.logging_config.filename_pattern,
server_config.logging_config.date_format)
execution_logging_service = ExecutionLoggingService(
execution_logs_path, log_name_creator)
existing_ids = [
entry.id for entry in execution_logging_service.get_history_entries()
]
id_generator = IdGenerator(existing_ids)
execution_service = ExecutionService(id_generator)
execution_logging_initiator = ExecutionLoggingInitiator(
execution_service, execution_logging_service)
execution_logging_initiator.start()
user_file_storage = UserFileStorage(secret)
file_download_feature = FileDownloadFeature(user_file_storage, TEMP_FOLDER)
file_download_feature.subscribe(execution_service)
file_upload_feature = FileUploadFeature(user_file_storage, TEMP_FOLDER)
alerter_feature = FailAlerterFeature(execution_service, alerts_service)
alerter_feature.start()
server.init(server_config, server_config.authenticator, authorizer,
execution_service, execution_logging_service, config_service,
alerts_service, file_upload_feature, file_download_feature,
secret)
def main():
project_path = os.getcwd()
try:
tool_utils.validate_web_build_exists(project_path)
except InvalidWebBuildException as e:
print(str(e))
sys.exit(-1)
logging_conf_file = os.path.join(CONFIG_FOLDER, 'logging.json')
with open(logging_conf_file, 'rt') as f:
log_config = json.load(f)
file_utils.prepare_folder(LOG_FOLDER)
logging.config.dictConfig(log_config)
server_version = tool_utils.get_server_version(project_path)
logging.info('Starting Script Server' +
(', v' +
server_version if server_version else ' (custom version)'))
file_utils.prepare_folder(CONFIG_FOLDER)
file_utils.prepare_folder(TEMP_FOLDER)
migrations.migrate.migrate(TEMP_FOLDER, CONFIG_FOLDER, SERVER_CONF_PATH,
LOG_FOLDER)
server_config = server_conf.from_json(SERVER_CONF_PATH, TEMP_FOLDER)
secret = get_secret(server_config.secret_storage_file)
tornado_client_config.initialize()
group_provider = create_group_provider(server_config.user_groups,
server_config.authenticator,
server_config.admin_users)
authorizer = Authorizer(server_config.allowed_users,
server_config.admin_users,
server_config.full_history_users,
server_config.code_editor_users, group_provider)
config_service = ConfigService(authorizer, CONFIG_FOLDER)
alerts_service = AlertsService(server_config.alerts_config)
alerts_service = alerts_service
execution_logs_path = os.path.join(LOG_FOLDER, 'processes')
log_name_creator = LogNameCreator(
server_config.logging_config.filename_pattern,
server_config.logging_config.date_format)
execution_logging_service = ExecutionLoggingService(
execution_logs_path, log_name_creator, authorizer)
existing_ids = [
entry.id for entry in execution_logging_service.get_history_entries(
None, system_call=True)
]
id_generator = IdGenerator(existing_ids)
execution_service = ExecutionService(authorizer, id_generator)
execution_logging_controller = ExecutionLoggingController(
execution_service, execution_logging_service)
execution_logging_controller.start()
user_file_storage = UserFileStorage(secret)
file_download_feature = FileDownloadFeature(user_file_storage, TEMP_FOLDER)
file_download_feature.subscribe(execution_service)
file_upload_feature = FileUploadFeature(user_file_storage, TEMP_FOLDER)
alerter_feature = FailAlerterFeature(execution_service, alerts_service)
alerter_feature.start()
executions_callback_feature = ExecutionsCallbackFeature(
execution_service, server_config.callbacks_config)
executions_callback_feature.start()
schedule_service = ScheduleService(config_service, execution_service,
CONFIG_FOLDER)
server.init(server_config, server_config.authenticator, authorizer,
execution_service, schedule_service, execution_logging_service,
config_service, alerts_service, file_upload_feature,
file_download_feature, secret, server_version, CONFIG_FOLDER)