def main(): args = vars(parse_args()) debug = args["debug"] parser_config(args["zone_id"], args["config"]) init_logging(debug) r = Replicator() r.start()
def test_init_logging_no_cw(self, caplog): """test_init_logging_no_cw""" with caplog.at_level(logging.INFO): init_logging(num_servers=4) assert caplog.records[ 0].msg == "CloudWatch logging disabled due to missing access key" caplog.clear()
def main(): """Sets up and run whole application""" # Set up endpoint for prometheus monitoring init_logging() LOGGER.info("Opening port [%s] for prometheus", prometheus_port) start_http_server(int(prometheus_port)) run_evaluator()
def main(filename): """ Main loop.""" init_logging() init_db() db_instance = DatabaseHandler.get_connection() data = JsonPkgTree(db_instance, filename) data.dump()
def main(): args = vars(parse_args()) debug = args["debug"] parser_config(args["zone_id"], args["config"]) init_logging(debug) r = Apply() asyncio.run(r.start())
def run_task(*args, **kwargs): """Function to import all repositories from input list to the DB.""" try: products = kwargs.get("products", None) repos = kwargs.get("repos", None) init_logging() init_db() if products: product_store = ProductStore() product_store.store(products) if repos: repository_controller = RepositoryController() # Sync repos from input for repo_url, content_set, basearch, releasever, cert_name, ca_cert, cert, key in repos: repository_controller.add_repository(repo_url, content_set, basearch, releasever, cert_name=cert_name, ca_cert=ca_cert, cert=cert, key=key) repository_controller.import_repositories() except Exception as err: # pylint: disable=broad-except msg = "Internal server error <%s>" % err.__hash__() LOGGER.exception(msg) DatabaseHandler.rollback() return "ERROR" return "OK"
def main(): # pylint: disable=too-many-statements """Main kafka listener entrypoint.""" init_logging() loop = asyncio.get_event_loop() status_app = create_status_app(LOGGER) _, status_site = create_status_runner(status_app, int(PROMETHEUS_PORT), LOGGER, loop) loop.run_until_complete(status_site.start()) loop.run_until_complete(a_ensure_minimal_schema_version()) LOGGER.info("Starting upload listener.") signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: loop.add_signal_handler( sig, lambda sig=sig: loop.create_task(terminate(sig, loop))) ListenerCtx.set_listener_ctx() with DatabasePool(WORKER_THREADS): # prepare repo name to id cache db_init_repo_cache() LISTENER_QUEUE.listen(process_message) # wait until loop is stopped from terminate callback loop.run_forever() LOGGER.info("Shutting down.") ListenerCtx.executor.shutdown()
def main(): """Main kafka listener entrypoint.""" start_http_server(int(PROMETHEUS_PORT)) init_logging() init_db() LOGGER.info("Starting upload listener.") # get DB connection conn = DatabaseHandler.get_connection() loop = asyncio.get_event_loop() signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: loop.add_signal_handler( sig, lambda sig=sig: loop.create_task(terminate(sig, loop))) executor = BoundedExecutor(MAX_QUEUE_SIZE, max_workers=WORKER_THREADS) def process_message(msg): """Message processing logic""" PROCESS_MESSAGES.inc() LOGGER.info('Received message from topic %s: %s', msg.topic, msg.value) try: msg_dict = json.loads(msg.value.decode("utf8")) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception("Unable to parse message: ") return if msg.topic == mqueue.UPLOAD_TOPIC: process_func = process_upload elif msg.topic == mqueue.EVENTS_TOPIC: if msg_dict['type'] == 'delete': process_func = process_delete else: UNKNOWN_EVENT_TYPE.inc() LOGGER.error("Received unknown event type: %s", msg_dict['type']) return else: UNKNOWN_TOPIC.inc() LOGGER.error("Received message on unsupported topic: %s", msg.topic) return if 'id' not in msg_dict or msg_dict["id"] is None: MISSING_ID.inc() LOGGER.warning( "Unable to process message, inventory ID is missing.") return future = executor.submit(process_func, msg_dict, conn, loop=loop) future.add_done_callback(on_thread_done) LISTENER_QUEUE.listen(process_message) # wait until loop is stopped from terminate callback loop.run_forever() LOGGER.info("Shutting down.") executor.shutdown()
def main(): """Creates scheduler, fills it up with tasks and runs it""" init_logging() LOGGER.info("Opening port [%s] for prometheus", PROMETHEUS_PORT) loop = asyncio.get_event_loop() status_app = create_status_app(LOGGER) _, status_site = create_status_runner(status_app, int(PROMETHEUS_PORT), LOGGER, loop) loop.run_until_complete(status_site.start()) loop.run_until_complete(a_ensure_minimal_schema_version()) signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sin in signals: signal.signal(sin, terminate) for job_info in JOBS: job_name, interval = job_info.split(':') job = import_job(job_name) if job: run_on_startup = job_name in JOBS_STARTUP if run_on_startup: next_run_time = datetime.now() else: next_run_time = undefined LOGGER.info('Adding job: %s, cadence each %s minutes, run on startup: %s', job_name, interval, run_on_startup) SCHEDULER.add_job(job.run, IntervalTrigger(minutes=int(interval)), id=job_name, next_run_time=next_run_time) else: LOGGER.error('Couldn\'t find job data for job: %s', job_name) SCHEDULER.start() MAIN_LOOP.start() LOGGER.info("Stopped.")
def main(): """Main VMaaS listener entrypoint.""" init_logging() loop = asyncio.get_event_loop() status_app = create_status_app(LOGGER) _, status_site = create_status_runner( status_app, int(CFG.prometheus_port or CFG.vmaas_sync_prometheus_port), LOGGER, loop, ) loop.run_until_complete(status_site.start()) loop.run_until_complete(a_ensure_minimal_schema_version()) LOGGER.info("Starting VMaaS sync service.") with DatabasePool(1): app_cont = VmaasSyncContext() def terminate(*_): """Trigger shutdown.""" LOGGER.info("Signal received, stopping application.") loop.add_callback_from_signal(app_cont.app.shutdown) signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: signal.signal(sig, terminate) web.run_app(app_cont.app, port=CFG.private_port) LOGGER.info("Shutting down.")
def main(filename): """ Main loop.""" init_logging() init_db() db_instance = DatabaseHandler.get_connection() #data = DataDump(db.cursor(), filename) data = DataDump(db_instance, filename) data.dump()
def main(): """Main platform mock entrypoint.""" init_logging() if not os.path.exists(STORAGE_PATH): os.makedirs(STORAGE_PATH) LOGGER.info("Starting platform mock.") app = ServerApplication() app.listen(8000) app.start()
def main(): """Main VMaaS listener entrypoint.""" start_http_server(int(PROMETHEUS_PORT)) init_logging() init_db() LOGGER.info("Starting VMaaS sync service.") app = ServerApplication() app.listen(8000) app.start()
def main(): """Sets up and run whole application""" # Set up endpoint for prometheus monitoring init_logging() LOGGER.info("Using BOOTSTRAP_SERVERS: %s", mqueue.BOOTSTRAP_SERVERS) LOGGER.info("Using GROUP_ID: %s", mqueue.GROUP_ID) LOGGER.info("Using TOPICS: %s", ", ".join(kafka_evaluator_topic)) LOGGER.info("Opening port [%s] for prometheus", prometheus_port) start_http_server(int(prometheus_port)) MAIN_LOOP.run_until_complete(run())
def test_init_logging_cw_err(self, caplog): """test_init_logging_cw_err""" os.environ["CW_AWS_ACCESS_KEY_ID"] = "cw-aws-access-key-id" os.environ["CW_AWS_SECRET_ACCESS_KEY"] = "cw-aws-secret-access-key" with caplog.at_level(logging.INFO): init_logging(num_servers=4) assert caplog.records[0].msg.startswith( "Unable to enable CloudWatch logging:") caplog.clear() del os.environ["CW_AWS_ACCESS_KEY_ID"] del os.environ["CW_AWS_SECRET_ACCESS_KEY"]
def run_task(*args, **kwargs): """Function to start syncing all CVEs.""" try: init_logging() init_db() controller = CvemapController() controller.store() except Exception as err: # pylint: disable=broad-except msg = "Internal server error <%s>" % err.__hash__() LOGGER.exception(msg) DatabaseHandler.rollback() return "ERROR" return "OK"
def main(): """Main entrypoint.""" init_logging() LOGGER.info("Starting (version %s).", VMAAS_VERSION) sync_interval = int(os.getenv('REPOSCAN_SYNC_INTERVAL_MINUTES', "720")) * 60000 if sync_interval > 0: PeriodicCallback(periodic_sync, sync_interval).start() else: LOGGER.info("Periodic syncing disabled.") app = ReposcanApplication() app.listen(8081) IOLoop.instance().start()
def run_task(*args, **kwargs): """Function to start deleting repos.""" try: repo = kwargs.get("repo", None) init_logging() init_db() repository_controller = RepositoryController() repository_controller.delete_content_set(repo) except Exception as err: # pylint: disable=broad-except msg = "Internal server error <%s>" % err.__hash__() LOGGER.exception(msg) DatabaseHandler.rollback() return "ERROR" return "OK"
def run_task(*args, **kwargs): """Function to start syncing all repositories available from database.""" try: init_logging() init_db() repository_controller = RepositoryController() repository_controller.add_db_repositories() repository_controller.store() except Exception as err: # pylint: disable=broad-except msg = "Internal server error <%s>" % err.__hash__() LOGGER.exception(msg) DatabaseHandler.rollback() return "ERROR" return "OK"
def main(): """Application entrypoint""" init_logging() status_app = create_status_app(LOGGER) _, status_site = create_status_runner(status_app, int(PROMETHEUS_PORT), LOGGER, MAIN_LOOP) MAIN_LOOP.run_until_complete(status_site.start()) MAIN_LOOP.run_until_complete(a_ensure_minimal_schema_version()) MAIN_LOOP.run_until_complete(setup_db_pool()) MAIN_LOOP.run_until_complete(db_init_caches()) LOGGER.info('Starting advisor listener.') MAIN_LOOP.run_until_complete(run())
def main(): """Creates schleduler, fills it up with tasks and runs it""" init_logging() scheduler = BlockingScheduler() for job_info in JOBS: job_name, interval = job_info.split(':') job = import_job(job_name) if job: LOGGER.info('Adding job: %s, cadence each %s minutes', job_name, interval) scheduler.add_job(job.run, IntervalTrigger(minutes=int(interval))) else: LOGGER.error('Couldn\'t find job data for job: %s', job_name) scheduler.start()
def create_app(): """Create reposcan app.""" init_logging() LOGGER.info("Starting (version %s).", VMAAS_VERSION) sync_interval = int(os.getenv('REPOSCAN_SYNC_INTERVAL_MINUTES', "360")) * 60000 if sync_interval > 0: PeriodicCallback(periodic_sync, sync_interval).start() else: LOGGER.info("Periodic syncing disabled.") app = ReposcanApplication() app.listen(8081) app.websocket_reconnect() app.reconnect_callback = PeriodicCallback( app.websocket_reconnect, WEBSOCKET_RECONNECT_INTERVAL * 1000) app.reconnect_callback.start()
def main(): """Sets up and run whole application""" # Set up endpoint for prometheus monitoring LOGGER.info("Opening port [%s] for prometheus", PROMETHEUS_PORT) init_logging() status_app = create_status_app(LOGGER) _, status_site = create_status_runner(status_app, int(PROMETHEUS_PORT), LOGGER, MAIN_LOOP) MAIN_LOOP.run_until_complete(status_site.start()) MAIN_LOOP.run_until_complete(a_ensure_minimal_schema_version()) LOGGER.info("Using BOOTSTRAP_SERVERS: %s", CFG.bootstrap_servers) LOGGER.info("Using GROUP_ID: %s", CFG.group_id) LOGGER.info("Using TOPICS: %s", ", ".join(CFG.evaluator_topics)) MAIN_LOOP.run_until_complete(run())
def main(): """Main kafka listener entrypoint.""" start_http_server(int(PROMETHEUS_PORT)) init_logging() init_db() LOGGER.info("Starting upload listener.") # get DB connection conn = DatabaseHandler.get_connection() session = requests.Session() loop = asyncio.get_event_loop() signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: loop.add_signal_handler( sig, lambda sig=sig: loop.create_task(terminate(sig, loop))) executor = ThreadPoolExecutor(WORKER_THREADS) def process_message(msg): """Message processing logic""" PROCESS_UPLOAD.inc() LOGGER.info('Received message from topic %s: %s', msg.topic, msg.value) upload_data = json.loads(msg.value.decode("utf8")) # Inventory ID is missing if 'id' not in upload_data or upload_data["id"] is None: MISSING_ID.inc() LOGGER.warning("Unable to store system, inventory ID is missing.") return future = executor.submit(process_upload, upload_data, session, conn, loop=loop) future.add_done_callback(on_thread_done) UPLOAD_QUEUE.listen(process_message) # wait until loop is stopped from terminate callback loop.run_forever() LOGGER.info("Shutting down.") executor.shutdown() session.close()
def main(): """Main platform mock entrypoint.""" init_logging() if not os.path.exists(STORAGE_PATH): os.makedirs(STORAGE_PATH) LOGGER.info("Starting platform mock.") app = ServerApplication() app.listen(8000) def terminate(*_): """Trigger shutdown.""" LOGGER.info("Signal received, stopping application.") IOLoop.instance().add_callback_from_signal(app.stop) signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: signal.signal(sig, terminate) app.start() LOGGER.info("Shutting down.")
def main(): """Main VMaaS listener entrypoint.""" start_http_server(int(PROMETHEUS_PORT)) init_logging() LOGGER.info("Starting VMaaS sync service.") with DatabasePool(1): app = ServerApplication() app.listen(8000) def terminate(*_): """Trigger shutdown.""" LOGGER.info("Signal received, stopping application.") IOLoop.instance().add_callback_from_signal(app.stop) signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: signal.signal(sig, terminate) app.start() LOGGER.info("Shutting down.")
def main(): """Creates schleduler, fills it up with tasks and runs it""" init_logging() LOGGER.info("Opening port [%s] for prometheus", PROMETHEUS_PORT) start_http_server(int(PROMETHEUS_PORT)) ensure_minimal_schema_version() scheduler = BlockingScheduler() for job_info in JOBS: job_name, interval = job_info.split(':') job = import_job(job_name) if job: LOGGER.info('Adding job: %s, cadence each %s minutes', job_name, interval) scheduler.add_job(job.run, IntervalTrigger(minutes=int(interval))) else: LOGGER.error('Couldn\'t find job data for job: %s', job_name) scheduler.start()
def main(): """Sets up and run whole application""" # Set up endpoint for prometheus monitoring init_logging() upgrader = DatabaseUpgrade() upgrader.upgrade()
def main(): # pylint: disable=too-many-statements """Main kafka listener entrypoint.""" start_http_server(int(PROMETHEUS_PORT)) init_logging() LOGGER.info("Starting upload listener.") loop = asyncio.get_event_loop() signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for sig in signals: loop.add_signal_handler( sig, lambda sig=sig: loop.create_task(terminate(sig, loop))) executor = BoundedExecutor(MAX_QUEUE_SIZE, max_workers=WORKER_THREADS) def process_message(msg): # pylint: disable=too-many-return-statements,too-many-branches """Message processing logic""" PROCESS_MESSAGES.inc() LOGGER.info('Received message from topic %s: %s', msg.topic, msg.value) try: msg_dict = json.loads(msg.value.decode("utf8")) except json.decoder.JSONDecodeError: MESSAGE_PARSE_ERROR.inc() LOGGER.exception("Unable to parse message: ") return FailedCache.process_failed_cache(FailedCache.upload_cache, executor, process_upload, loop) FailedCache.process_failed_cache(FailedCache.delete_cache, executor, process_delete, loop) if msg.topic == mqueue.UPLOAD_TOPIC: if not validate_msg(msg_dict, "upload", REQUIRED_UPLOAD_MESSAGE_FIELDS): return # send message to payload tracker send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'received', loop=loop) # proces only archives from smart_management accounts identity = get_identity( msg_dict["platform_metadata"]["b64_identity"]) if identity is None: INVALID_IDENTITY.inc() error_msg = "Skipped upload due to invalid identity header." LOGGER.warning(error_msg) send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'error', status_msg=error_msg, loop=loop) return if not is_entitled_smart_management(identity, allow_missing_section=True): MISSING_SMART_MANAGEMENT.inc() error_msg = "Skipped upload due to missing smart_management entitlement." LOGGER.debug(error_msg) send_msg_to_payload_tracker(PAYLOAD_TRACKER_PRODUCER, msg_dict, 'error', status_msg=error_msg, loop=loop) return process_func = process_upload elif msg.topic == mqueue.EVENTS_TOPIC: if not validate_msg(msg_dict, "event", REQUIRED_EVENT_MESSAGE_FIELDS): return if msg_dict['type'] == 'delete': process_func = process_delete else: UNKNOWN_EVENT_TYPE.inc() LOGGER.error("Received unknown event type: %s", msg_dict['type']) return else: UNKNOWN_TOPIC.inc() LOGGER.error("Received message on unsupported topic: %s", msg.topic) return future = executor.submit(process_func, msg_dict, loop=loop) future.add_done_callback(on_thread_done) with DatabasePool(WORKER_THREADS): # prepare repo name to id cache db_init_repo_cache() LISTENER_QUEUE.listen(process_message) # wait until loop is stopped from terminate callback loop.run_forever() LOGGER.info("Shutting down.") executor.shutdown()
def test_cw_disabled(self, caplog): """tests disabled cloudwatch logging""" with caplog.at_level(logging.INFO): init_logging(num_servers=4) assert caplog.records[0].msg == "CloudWatch logging disabled" caplog.clear()