def test_find_history_entry_after_restart(self): self.simulate_logging(execution_id='id1') new_service = ExecutionLoggingService(test_utils.temp_folder, LogNameCreator()) entry = new_service.find_history_entry('id1') self.assertIsNotNone(entry) self.validate_history_entry(entry, id='id1')
def test_find_history_entry_after_restart(self): self.simulate_logging(execution_id='id1') new_service = ExecutionLoggingService(test_utils.temp_folder) entry = new_service.find_history_entry('id1') self.assertIsNotNone(entry) self.validate_history_entry(entry, id='id1')
def test_history_entries_after_restart(self): self.simulate_logging(execution_id='id1') new_service = ExecutionLoggingService(test_utils.temp_folder, LogNameCreator()) entry = new_service.get_history_entries()[0] self.validate_history_entry(entry, id='id1')
def main(): tool_utils.validate_web_imports_exist(os.getcwd()) logging_conf_file = os.path.join(CONFIG_FOLDER, 'logging.json') with open(logging_conf_file, 'rt') as f: log_config = json.load(f) file_utils.prepare_folder(os.path.join('logs')) logging.config.dictConfig(log_config) file_utils.prepare_folder(CONFIG_FOLDER) file_utils.prepare_folder(TEMP_FOLDER) migrations.migrate.migrate(TEMP_FOLDER, CONFIG_FOLDER) server_config = server_conf.from_json(SERVER_CONF_PATH) secret = get_secret(TEMP_FOLDER) config_service = ConfigService(CONFIG_FOLDER) alerts_service = AlertsService(server_config.get_alerts_config()) alerts_service = alerts_service execution_logs_path = os.path.join('logs', 'processes') log_name_creator = LogNameCreator( server_config.logging_config.filename_pattern, server_config.logging_config.date_format) execution_logging_service = ExecutionLoggingService( execution_logs_path, log_name_creator) existing_ids = [ entry.id for entry in execution_logging_service.get_history_entries() ] id_generator = IdGenerator(existing_ids) execution_service = ExecutionService(id_generator) execution_logging_initiator = ExecutionLoggingInitiator( execution_service, execution_logging_service) execution_logging_initiator.start() user_file_storage = UserFileStorage(secret) file_download_feature = FileDownloadFeature(user_file_storage, TEMP_FOLDER) file_download_feature.subscribe(execution_service) file_upload_feature = FileUploadFeature(user_file_storage, TEMP_FOLDER) alerter_feature = FailAlerterFeature(execution_service, alerts_service) alerter_feature.start() server.init(server_config, execution_service, execution_logging_service, config_service, alerts_service, file_upload_feature, file_download_feature, secret)
def setUp(self): test_utils.setup() self.exit_codes = {} self.post_info_provider = _MapBasedPostExecInfo(self.exit_codes) self.logging_service = ExecutionLoggingService(test_utils.temp_folder)
def setUp(self): test_utils.setup() self.authorizer = Authorizer([], [], ['power_user'], [], EmptyGroupProvider()) self.logging_service = ExecutionLoggingService(test_utils.temp_folder, LogNameCreator(), self.authorizer)
def __migrate_user_id(context): output_folder = os.path.join(context.log_folder, 'processes') if not os.path.exists(output_folder): return log_files = [ os.path.join(output_folder, file) for file in os.listdir(output_folder) if file.lower().endswith('.log') ] for log_file in log_files: (correct, parameters_text ) = ExecutionLoggingService._read_parameters_text(log_file) if not correct: continue parameters = ExecutionLoggingService._parse_history_parameters( parameters_text) if not parameters or ('user' not in parameters): continue if ('user_id' in parameters) and ('user_name' in parameters): continue file_content = file_utils.read_file(log_file, keep_newlines=True) file_parts = file_content.split( execution.logging.OUTPUT_STARTED_MARKER + os.linesep, 1) parameters_text = file_parts[0] user = parameters['user'] if 'user_id' not in parameters: parameters_text += 'user_id:' + user + os.linesep if 'user_name' not in parameters: parameters_text += 'user_name:' + user + os.linesep new_content = parameters_text + execution.logging.OUTPUT_STARTED_MARKER + os.linesep + file_parts[ 1] file_utils.write_file(log_file, new_content.encode(execution.logging.ENCODING), byte_content=True)
def setUp(self): test_utils.setup() executor._process_creator = _MockProcessWrapper authorizer = Authorizer([], [], [], EmptyGroupProvider()) self.logging_service = ExecutionLoggingService(test_utils.temp_folder, LogNameCreator(), authorizer) self.executor_service = ExecutionService(_IdGeneratorMock()) self.controller = ExecutionLoggingController(self.executor_service, self.logging_service) self.controller.start()
def __migrate_user_id(context): output_folder = os.path.join(context.log_folder, 'processes') if not os.path.exists(output_folder): return log_files = [os.path.join(output_folder, file) for file in os.listdir(output_folder) if file.lower().endswith('.log')] for log_file in log_files: (correct, parameters_text) = ExecutionLoggingService._read_parameters_text(log_file) if not correct: continue parameters = ExecutionLoggingService._parse_history_parameters(parameters_text) if not parameters or ('user' not in parameters): continue if ('user_id' in parameters) and ('user_name' in parameters): continue file_content = file_utils.read_file(log_file, keep_newlines=True) file_parts = file_content.split(execution.logging.OUTPUT_STARTED_MARKER + os.linesep, 1) parameters_text = file_parts[0] user = parameters['user'] if 'user_id' not in parameters: parameters_text += 'user_id:' + user + os.linesep if 'user_name' not in parameters: parameters_text += 'user_name:' + user + os.linesep new_content = parameters_text + execution.logging.OUTPUT_STARTED_MARKER + os.linesep + file_parts[1] file_utils.write_file(log_file, new_content.encode(execution.logging.ENCODING), byte_content=True)
def __migrate_old_files(context): output_folder = os.path.join(context.log_folder, 'processes') if not os.path.exists(output_folder): return log_files = [ os.path.join(output_folder, file) for file in os.listdir(output_folder) if file.lower().endswith('.log') ] def is_new_format(log_file): with open(log_file, 'r') as f: first_line = f.readline().strip() if not first_line.startswith('id:'): return False for line in f: if line.strip() == execution.logging.OUTPUT_STARTED_MARKER: return True return False old_files = [ log_file for log_file in log_files if not is_new_format(log_file) ] if not old_files: return existing_ids = set() for file in log_files: correct, parameters_text = ExecutionLoggingService._read_parameters_text( file) if not correct: continue parameters = ExecutionLoggingService._parse_history_parameters( parameters_text) if not parameters or 'id' not in parameters: continue existing_ids.add(parameters['id']) id_generator = (str(id) for id in itertools.count()) id_generator = filter(lambda id: id not in existing_ids, id_generator) for old_file in old_files: log_basename = os.path.basename(old_file) filename = os.path.splitext(log_basename)[0] match = re.fullmatch( '(.+)_([^_]+)_((\d\d)(\d\d)(\d\d)_(\d\d)(\d\d)(\d\d))', filename) if match: script_name = match.group(1) username = match.group(2) start_time = datetime.strptime(match.group(3), '%y%m%d_%H%M%S') id = next(id_generator) else: script_name = 'unknown' username = '******' start_time = sec_to_datetime(os.path.getctime(old_file)) id = next(id_generator) new_begin = '' new_begin += 'id:' + id + '\n' new_begin += 'user_name:' + username + '\n' new_begin += 'user_id:' + username + '\n' new_begin += 'script:' + script_name + '\n' new_begin += 'start_time:' + str(to_millis(start_time)) + '\n' new_begin += 'command:unknown' + '\n' new_begin += execution.logging.OUTPUT_STARTED_MARKER + '\n' file_content = file_utils.read_file(old_file) file_content = new_begin + file_content file_utils.write_file(old_file, file_content)
def main(): try: tool_utils.validate_web_build_exists(os.getcwd()) except InvalidWebBuildException as e: print(str(e)) sys.exit(-1) logging_conf_file = os.path.join(CONFIG_FOLDER, 'logging.json') with open(logging_conf_file, 'rt') as f: log_config = json.load(f) file_utils.prepare_folder(LOG_FOLDER) logging.config.dictConfig(log_config) file_utils.prepare_folder(CONFIG_FOLDER) file_utils.prepare_folder(TEMP_FOLDER) migrations.migrate.migrate(TEMP_FOLDER, CONFIG_FOLDER, SERVER_CONF_PATH, LOG_FOLDER) server_config = server_conf.from_json(SERVER_CONF_PATH, TEMP_FOLDER) secret = get_secret(TEMP_FOLDER) tornado_client_config.initialize() group_provider = create_group_provider(server_config.user_groups, server_config.authenticator, server_config.admin_users) authorizer = Authorizer(server_config.allowed_users, server_config.admin_users, group_provider) config_service = ConfigService(authorizer, CONFIG_FOLDER) alerts_service = AlertsService(server_config.get_alerts_config()) alerts_service = alerts_service execution_logs_path = os.path.join(LOG_FOLDER, 'processes') log_name_creator = LogNameCreator( server_config.logging_config.filename_pattern, server_config.logging_config.date_format) execution_logging_service = ExecutionLoggingService( execution_logs_path, log_name_creator) existing_ids = [ entry.id for entry in execution_logging_service.get_history_entries() ] id_generator = IdGenerator(existing_ids) execution_service = ExecutionService(id_generator) execution_logging_initiator = ExecutionLoggingInitiator( execution_service, execution_logging_service) execution_logging_initiator.start() user_file_storage = UserFileStorage(secret) file_download_feature = FileDownloadFeature(user_file_storage, TEMP_FOLDER) file_download_feature.subscribe(execution_service) file_upload_feature = FileUploadFeature(user_file_storage, TEMP_FOLDER) alerter_feature = FailAlerterFeature(execution_service, alerts_service) alerter_feature.start() server.init(server_config, server_config.authenticator, authorizer, execution_service, execution_logging_service, config_service, alerts_service, file_upload_feature, file_download_feature, secret)
def main(): tool_utils.validate_web_imports_exist(os.getcwd()) logging_conf_file = os.path.join(CONFIG_FOLDER, 'logging.json') with open(logging_conf_file, "rt") as f: log_config = json.load(f) file_utils.prepare_folder(os.path.join('logs')) logging.config.dictConfig(log_config) file_utils.prepare_folder(CONFIG_FOLDER) file_utils.prepare_folder(SCRIPT_CONFIGS_FOLDER) server_config = server_conf.from_json(SERVER_CONF_PATH) ssl_context = None if server_config.is_ssl(): ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) ssl_context.load_cert_chain(server_config.get_ssl_cert_path(), server_config.get_ssl_key_path()) file_utils.prepare_folder(TEMP_FOLDER) settings = { "cookie_secret": get_tornado_secret(), "login_url": "/login.html" } auth = TornadoAuth(server_config.authenticator, server_config.authorizer) user_file_storage = UserFileStorage(get_tornado_secret()) file_download_feature = FileDownloadFeature(user_file_storage, TEMP_FOLDER) result_files_folder = file_download_feature.get_result_files_folder() handlers = [(r"/conf/title", GetServerTitle), (r"/scripts/list", GetScripts), (r"/scripts/info", GetScriptInfo), (r"/scripts/execute", ScriptExecute), (r"/scripts/execute/stop", ScriptStop), (r"/scripts/execute/io/(.*)", ScriptStreamSocket), (r'/admin/execution_log/short', GetShortHistoryEntriesHandler), (r'/admin/execution_log/long/(.*)', GetLongHistoryEntryHandler), (r'/' + os.path.basename(result_files_folder) + '/(.*)', DownloadResultFile, { 'path': result_files_folder }), (r"/", ProxiedRedirectHandler, { "url": "/index.html" })] if auth.is_enabled(): handlers.append((r'/login', LoginHandler)) handlers.append((r'/auth/config', AuthConfigHandler)) handlers.append((r'/logout', LogoutHandler)) handlers.append((r"/username", GetUsernameHandler)) handlers.append((r"/(.*)", AuthorizedStaticFileHandler, {"path": "web"})) application = tornado.web.Application(handlers, **settings) application.auth = auth application.server_title = server_config.title application.authorizer = server_config.authorizer application.file_download_feature = file_download_feature application.file_upload_feature = FileUploadFeature( user_file_storage, TEMP_FOLDER) alerts_service = AlertsService(server_config.get_alerts_config()) application.alerts_service = alerts_service execution_logs_path = os.path.join('logs', 'processes') log_name_creator = LogNameCreator( server_config.logging_config.filename_pattern, server_config.logging_config.date_format) execution_logging_service = ExecutionLoggingService( execution_logs_path, log_name_creator) application.execution_logging_service = execution_logging_service existing_ids = [ entry.id for entry in execution_logging_service.get_history_entries() ] id_generator = IdGenerator(existing_ids) execution_service = ExecutionService(execution_logging_service, alerts_service, id_generator) application.execution_service = execution_service http_server = httpserver.HTTPServer(application, ssl_options=ssl_context) http_server.listen(server_config.port, address=server_config.address) io_loop = tornado.ioloop.IOLoop.current() intercept_stop_when_running_scripts(io_loop, execution_service) http_protocol = 'https' if server_config.ssl else 'http' print('Server is running on: %s://%s:%s' % (http_protocol, server_config.address, server_config.port)) io_loop.start()
def main(): project_path = os.getcwd() try: tool_utils.validate_web_build_exists(project_path) except InvalidWebBuildException as e: print(str(e)) sys.exit(-1) logging_conf_file = os.path.join(CONFIG_FOLDER, 'logging.json') with open(logging_conf_file, 'rt') as f: log_config = json.load(f) file_utils.prepare_folder(LOG_FOLDER) logging.config.dictConfig(log_config) server_version = tool_utils.get_server_version(project_path) logging.info('Starting Script Server' + (', v' + server_version if server_version else ' (custom version)')) file_utils.prepare_folder(CONFIG_FOLDER) file_utils.prepare_folder(TEMP_FOLDER) migrations.migrate.migrate(TEMP_FOLDER, CONFIG_FOLDER, SERVER_CONF_PATH, LOG_FOLDER) server_config = server_conf.from_json(SERVER_CONF_PATH, TEMP_FOLDER) secret = get_secret(server_config.secret_storage_file) tornado_client_config.initialize() group_provider = create_group_provider(server_config.user_groups, server_config.authenticator, server_config.admin_users) authorizer = Authorizer(server_config.allowed_users, server_config.admin_users, server_config.full_history_users, server_config.code_editor_users, group_provider) config_service = ConfigService(authorizer, CONFIG_FOLDER) alerts_service = AlertsService(server_config.alerts_config) alerts_service = alerts_service execution_logs_path = os.path.join(LOG_FOLDER, 'processes') log_name_creator = LogNameCreator( server_config.logging_config.filename_pattern, server_config.logging_config.date_format) execution_logging_service = ExecutionLoggingService( execution_logs_path, log_name_creator, authorizer) existing_ids = [ entry.id for entry in execution_logging_service.get_history_entries( None, system_call=True) ] id_generator = IdGenerator(existing_ids) execution_service = ExecutionService(authorizer, id_generator) execution_logging_controller = ExecutionLoggingController( execution_service, execution_logging_service) execution_logging_controller.start() user_file_storage = UserFileStorage(secret) file_download_feature = FileDownloadFeature(user_file_storage, TEMP_FOLDER) file_download_feature.subscribe(execution_service) file_upload_feature = FileUploadFeature(user_file_storage, TEMP_FOLDER) alerter_feature = FailAlerterFeature(execution_service, alerts_service) alerter_feature.start() executions_callback_feature = ExecutionsCallbackFeature( execution_service, server_config.callbacks_config) executions_callback_feature.start() schedule_service = ScheduleService(config_service, execution_service, CONFIG_FOLDER) server.init(server_config, server_config.authenticator, authorizer, execution_service, schedule_service, execution_logging_service, config_service, alerts_service, file_upload_feature, file_download_feature, secret, server_version, CONFIG_FOLDER)
def setUp(self): test_utils.setup() self.logging_service = ExecutionLoggingService(test_utils.temp_folder, LogNameCreator())
def __migrate_old_files(context): output_folder = os.path.join(context.log_folder, 'processes') if not os.path.exists(output_folder): return log_files = [os.path.join(output_folder, file) for file in os.listdir(output_folder) if file.lower().endswith('.log')] def is_new_format(log_file): with open(log_file, 'r') as f: first_line = f.readline().strip() if not first_line.startswith('id:'): return False for line in f: if line.strip() == execution.logging.OUTPUT_STARTED_MARKER: return True return False old_files = [log_file for log_file in log_files if not is_new_format(log_file)] if not old_files: return existing_ids = set() for file in log_files: correct, parameters_text = ExecutionLoggingService._read_parameters_text(file) if not correct: continue parameters = ExecutionLoggingService._parse_history_parameters(parameters_text) if not parameters or 'id' not in parameters: continue existing_ids.add(parameters['id']) id_generator = (str(id) for id in itertools.count()) id_generator = filter(lambda id: id not in existing_ids, id_generator) for old_file in old_files: log_basename = os.path.basename(old_file) filename = os.path.splitext(log_basename)[0] match = re.fullmatch('(.+)_([^_]+)_((\d\d)(\d\d)(\d\d)_(\d\d)(\d\d)(\d\d))', filename) if match: script_name = match.group(1) username = match.group(2) start_time = datetime.strptime(match.group(3), '%y%m%d_%H%M%S') id = next(id_generator) else: script_name = 'unknown' username = '******' start_time = sec_to_datetime(os.path.getctime(old_file)) id = next(id_generator) new_begin = '' new_begin += 'id:' + id + '\n' new_begin += 'user_name:' + username + '\n' new_begin += 'user_id:' + username + '\n' new_begin += 'script:' + script_name + '\n' new_begin += 'start_time:' + str(to_millis(start_time)) + '\n' new_begin += 'command:unknown' + '\n' new_begin += execution.logging.OUTPUT_STARTED_MARKER + '\n' file_content = file_utils.read_file(old_file) file_content = new_begin + file_content file_utils.write_file(old_file, file_content)