def run(self): while True: try: with closing(Session()) as session: self.logger.debug("Expiring edges....") self.expire_edges(session) self.logger.debug( "Expiring nonauditor approvers in audited groups...") self.expire_nonauditors(session) self.logger.debug("Sending emails...") process_async_emails(self.settings, session, datetime.utcnow()) self.logger.debug("Pruning old traces....") prune_old_traces(session) session.commit() stats.set_gauge("successful-background-update", 1) stats.set_gauge("failed-background-update", 0) except OperationalError: Session.configure( bind=get_db_engine(get_database_url(self.settings))) self.logger.critical("Failed to connect to database.") stats.set_gauge("successful-background-update", 0) stats.set_gauge("failed-background-update", 1) self.capture_exception() except: stats.set_gauge("successful-background-update", 0) stats.set_gauge("failed-background-update", 1) self.capture_exception() raise sleep(60)
def run(self): while True: try: with closing(Session()) as session: self.logger.debug("Expiring edges....") self.expire_edges(session) self.logger.debug("Expiring nonauditor approvers in audited groups...") self.expire_nonauditors(session) self.logger.debug("Sending emails...") process_async_emails(self.settings, session, datetime.utcnow()) self.logger.debug("Pruning old traces....") prune_old_traces(session) session.commit() stats.set_gauge("successful-background-update", 1) stats.set_gauge("failed-background-update", 0) except OperationalError: Session.configure(bind=get_db_engine(get_database_url(self.settings))) self.logger.critical("Failed to connect to database.") stats.set_gauge("successful-background-update", 0) stats.set_gauge("failed-background-update", 1) self.capture_exception() except: stats.set_gauge("successful-background-update", 0) stats.set_gauge("failed-background-update", 1) self.capture_exception() raise sleep(60)
def get_current_user(self): username = self.request.headers.get(settings.user_auth_header) if not username: return # Users must be fully qualified if not re.match("^{}$".format(USERNAME_VALIDATION), username): raise InvalidUser() try: user, created = User.get_or_create(self.session, username=username) if created: logging.info("Created new user %s", username) self.session.commit() # Because the graph doesn't initialize until the updates table # is populated, we need to refresh the graph here in case this # is the first update. self.graph.update_from_db(self.session) except sqlalchemy.exc.OperationalError: # Failed to connect to database or create user, try to reconfigure the db. This invokes # the fetcher to try to see if our URL string has changed. Session.configure(bind=get_db_engine(get_database_url(settings))) raise DatabaseFailure() return user
def session(self): # type: () -> Session if not self._session: db_engine = get_db_engine(get_database_url(self.settings)) Session.configure(bind=db_engine) self._session = Session() return self._session
def send_async_email( session: Session, recipients: Iterable[str], subject: str, template: str, settings: Settings, context: Context, send_after: datetime, async_key: Optional[str] = None, ) -> None: """Construct a message object from a template and schedule it This is the main email sending method to send out a templated email. This is used to asynchronously queue up the email for sending. Args: recipients: Email addresses that will receive this mail subject: Subject of the email. template: Name of the template to use. context: Context for the template library. settings: Grouper settings send_after: Schedule the email to go out after this point in time. async_key: If you set this, it will be inserted into the db so that you can find this email in the future. """ msg = get_email_from_template(recipients, subject, template, settings, context) for rcpt in recipients: notif = AsyncNotification( key=async_key, email=rcpt, subject=subject, body=msg.as_string(), send_after=send_after ) notif.add(session) session.commit()
def grant_permission_to_service_account(session: Session, account: ServiceAccount, permission: Permission, argument: str = "") -> None: """Grant a permission to this service account. This will fail if the (permission, argument) has already been granted to this group. Args: session: Database session account: A ServiceAccount object being granted a permission permission: A Permission object being granted argument: Must match constants.ARGUMENT_VALIDATION Throws: AssertError if argument does not match ARGUMENT_VALIDATION regex """ assert re.match(ARGUMENT_VALIDATION + r"$", argument), "Invalid permission argument" mapping = ServiceAccountPermissionMap(permission_id=permission.id, service_account_id=account.id, argument=argument) mapping.add(session) Counter.incr(session, "updates") session.commit()
def grant_permission(session: Session, group_id: int, permission_id: int, argument: str = "") -> None: """Grant a permission to this group. This will fail if the (permission, argument) has already been granted to this group. Args: session: Database session group_id: ID of group to which to grant the permission permission_id: ID of permission to grant argument: Must match constants.ARGUMENT_VALIDATION Throws: AssertError if argument does not match ARGUMENT_VALIDATION regex """ assert re.match(ARGUMENT_VALIDATION + r"$", argument), "Invalid permission argument" mapping = PermissionMap(permission_id=permission_id, group_id=group_id, argument=argument) mapping.add(session) Counter.incr(session, "updates") session.commit()
def disable_permission_auditing(session: Session, permission_name: str, actor_user_id: int) -> None: """Set a permission as audited. Args: session: Database session permission_name: Name of permission in question actor_user_id: ID of user who is disabling auditing """ permission = get_permission(session, permission_name) if not permission: raise NoSuchPermission(name=permission_name) permission.audited = False AuditLog.log( session, actor_user_id, "disable_auditing", "Disabled auditing.", on_permission_id=permission.id, ) Counter.incr(session, "updates") session.commit()
def session(request, tmpdir): # type: (FixtureRequest, LocalPath) -> None settings = Settings() set_global_settings(settings) # Reinitialize plugins in case a previous test configured some. set_global_plugin_proxy(PluginProxy([])) db_engine = get_db_engine(db_url(tmpdir)) # Clean up from previous tests if using a persistent database. if "MEROU_TEST_DATABASE" in os.environ: Model.metadata.drop_all(db_engine) # Create the database schema and the corresponding session. Model.metadata.create_all(db_engine) Session.configure(bind=db_engine) session = Session() def fin(): # type: () -> None """Explicitly close the session to avoid any dangling transactions.""" session.close() request.addfinalizer(fin) return session
def start_server(args, sentry_client): # type: (Namespace, SentryProxy) -> None log_level = logging.getLevelName(logging.getLogger().level) logging.info("begin. log_level={}".format(log_level)) assert not (settings.debug and settings.num_processes > 1 ), "debug mode does not support multiple processes" try: initialize_plugins(settings.plugin_dirs, settings.plugin_module_paths, "grouper_fe") except PluginsDirectoryDoesNotExist as e: logging.fatal("Plugin directory does not exist: {}".format(e)) sys.exit(1) # setup database logging.debug("configure database session") database_url = args.database_url or get_database_url(settings) Session.configure(bind=get_db_engine(database_url)) usecase_factory = create_graph_usecase_factory(settings, Session()) application = create_fe_application(settings, usecase_factory, args.deployment_name) address = args.address or settings.address port = args.port or settings.port ssl_context = get_plugin_proxy().get_ssl_context() logging.info("Starting application server with %d processes on port %d", settings.num_processes, port) server = tornado.httpserver.HTTPServer(application, ssl_options=ssl_context) server.bind(port, address=address) # When using multiple processes, the forking happens here server.start(settings.num_processes) stats.set_defaults() # Create the Graph and start the config / graph update threads post fork to ensure each # process gets updated. settings.start_config_thread(args.config, "fe") with closing(Session()) as session: graph = Graph() graph.update_from_db(session) refresher = DbRefreshThread(settings, graph, settings.refresh_interval, sentry_client) refresher.daemon = True refresher.start() try: tornado.ioloop.IOLoop.instance().start() except KeyboardInterrupt: tornado.ioloop.IOLoop.instance().stop() finally: print("Bye")
def get(session: Session, pk: Optional[int] = None, name: Optional[str] = None) -> Optional[Group]: if pk is not None: return session.query(Group).filter_by(id=pk).scalar() if name is not None: return session.query(Group).filter_by(groupname=name).scalar() return None
def start_server(args, settings, plugins): # type: (Namespace, FrontendSettings, PluginProxy) -> None log_level = logging.getLevelName(logging.getLogger().level) logging.info("begin. log_level={}".format(log_level)) assert not (settings.debug and settings.num_processes > 1 ), "debug mode does not support multiple processes" # setup database logging.debug("configure database session") if args.database_url: settings.database = args.database_url Session.configure(bind=get_db_engine(settings.database)) application = create_fe_application(settings, args.deployment_name) ssl_context = plugins.get_ssl_context() if args.listen_stdin: logging.info("Starting application server with %d processes on stdin", settings.num_processes) server = HTTPServer(application, ssl_options=ssl_context) s = socket.socket(fileno=sys.stdin.fileno()) s.setblocking(False) s.listen() server.add_sockets([s]) else: address = args.address or settings.address port = args.port or settings.port logging.info( "Starting application server with %d processes on %s:%d", settings.num_processes, address, port, ) server = HTTPServer(application, ssl_options=ssl_context) server.bind(port, address=address) # When using multiple processes, the forking happens here server.start(settings.num_processes) # Create the Graph and start the graph update thread post fork to ensure each process gets # updated. with closing(Session()) as session: graph = Graph() graph.update_from_db(session) refresher = DbRefreshThread(settings, plugins, graph, settings.refresh_interval) refresher.daemon = True refresher.start() try: IOLoop.current().start() except KeyboardInterrupt: IOLoop.current().stop() finally: print("Bye")
def start_server(args, settings, plugins): # type: (Namespace, ApiSettings, PluginProxy) -> None log_level = logging.getLevelName(logging.getLogger().level) logging.info("begin. log_level={}".format(log_level)) assert not ( settings.debug and settings.num_processes > 1 ), "debug mode does not support multiple processes" # setup database logging.debug("configure database session") if args.database_url: settings.database = args.database_url Session.configure(bind=get_db_engine(settings.database)) with closing(Session()) as session: graph = Graph() graph.update_from_db(session) refresher = DbRefreshThread(settings, plugins, graph, settings.refresh_interval) refresher.daemon = True refresher.start() usecase_factory = create_graph_usecase_factory(settings, plugins, graph=graph) application = create_api_application(graph, settings, plugins, usecase_factory) if args.listen_stdin: logging.info("Starting application server on stdin") server = HTTPServer(application) if PY2: s = socket.fromfd(sys.stdin.fileno(), socket.AF_INET, socket.SOCK_STREAM) s.setblocking(False) s.listen(5) else: s = socket.socket(fileno=sys.stdin.fileno()) s.setblocking(False) s.listen() server.add_sockets([s]) else: address = args.address or settings.address port = args.port or settings.port logging.info("Starting application server on %s:%d", address, port) server = HTTPServer(application) server.bind(port, address=address) server.start(settings.num_processes) stats.set_defaults() try: IOLoop.current().start() except KeyboardInterrupt: IOLoop.current().stop() finally: print("Bye")
def create_session(self, tmpdir): # type: (LocalPath) -> Session db_engine = get_db_engine(db_url(tmpdir)) # If using a persistent database, clear the database first. if "MEROU_TEST_DATABASE" in os.environ: Model.metadata.drop_all(db_engine) # Create the database schema and the corresponding session. Model.metadata.create_all(db_engine) Session.configure(bind=db_engine) return Session()
def cancel_async_emails(session: Session, async_key: str) -> None: """Cancel pending async emails by key If you scheduled an asynchronous email with an async_key previously, this method can be used to cancel any unsent emails. Args: async_key: The async_key previously provided for your emails. """ session.query(AsyncNotification).filter( AsyncNotification.key == async_key, AsyncNotification.sent == False).update({"sent": True})
def mutate_group_command(session: Session, group: Group, args: Namespace) -> None: for username in args.username: user = User.get(session, name=username) if not user: logging.error("no such user '{}'".format(username)) return if args.subcommand == "add_member": if args.member: role = "member" elif args.owner: role = "owner" elif args.np_owner: role = "np-owner" elif args.manager: role = "manager" assert role logging.info("Adding {} as {} to group {}".format( username, role, args.groupname)) group.add_member(user, user, "grouper-ctl join", status="actioned", role=role) AuditLog.log( session, user.id, "join_group", "{} manually joined via grouper-ctl".format(username), on_group_id=group.id, ) session.commit() elif args.subcommand == "remove_member": logging.info("Removing {} from group {}".format( username, args.groupname)) try: group.revoke_member(user, user, "grouper-ctl remove") AuditLog.log( session, user.id, "leave_group", "{} manually left via grouper-ctl".format(username), on_group_id=group.id, ) session.commit() except PluginRejectedGroupMembershipUpdate as e: logging.error("%s", e)
def session(request, tmpdir): db_engine = get_db_engine(db_url(tmpdir)) Model.metadata.create_all(db_engine) Session.configure(bind=db_engine) session = Session() def fin(): session.close() # Useful if testing against MySQL # Model.metadata.drop_all(db_engine) request.addfinalizer(fin) return session
def get_changes_by_request_id( session: Session, request_id: int ) -> List[Tuple[PermissionRequestStatusChange, Comment]]: status_changes = (session.query(PermissionRequestStatusChange).filter( PermissionRequestStatusChange.request_id == request_id).all()) comments = (session.query(Comment).filter( Comment.obj_type == OBJ_TYPES_IDX.index( "PermissionRequestStatusChange"), Comment.obj_pk.in_([s.id for s in status_changes]), ).all()) comment_by_status_change_id = {c.obj_pk: c for c in comments} return [(sc, comment_by_status_change_id[sc.id]) for sc in status_changes]
def session(request, tmpdir): db_path = tmpdir.join("grouper.sqlite") db_engine = get_db_engine("sqlite:///%s" % db_path) Model.metadata.create_all(db_engine) Session.configure(bind=db_engine) session = Session() def fin(): session.close() # Useful if testing against MySQL # model_soup.Model.metadata.drop_all(db_engine) request.addfinalizer(fin) return session
def start_server(args, sentry_client): # type: (argparse.Namespace, SentryProxy) -> None log_level = logging.getLevelName(logging.getLogger().level) logging.info("begin. log_level={}".format(log_level)) assert not (settings.debug and settings.num_processes > 1), \ "debug mode does not support multiple processes" try: initialize_plugins(settings.plugin_dirs, settings.plugin_module_paths, "grouper_api") except PluginsDirectoryDoesNotExist as e: logging.fatal("Plugin directory does not exist: {}".format(e)) sys.exit(1) # setup database logging.debug("configure database session") database_url = args.database_url or get_database_url(settings) Session.configure(bind=get_db_engine(database_url)) settings.start_config_thread(args.config, "api") with closing(Session()) as session: graph = Graph() graph.update_from_db(session) refresher = DbRefreshThread(settings, graph, settings.refresh_interval, sentry_client) refresher.daemon = True refresher.start() application = get_application(graph, settings, sentry_client) address = args.address or settings.address port = args.port or settings.port logging.info("Starting application server on port %d", port) server = tornado.httpserver.HTTPServer(application) server.bind(port, address=address) server.start(settings.num_processes) stats.set_defaults() try: tornado.ioloop.IOLoop.instance().start() except KeyboardInterrupt: tornado.ioloop.IOLoop.instance().stop() finally: print "Bye"
def run(self): # type: () -> None initial_url = self.settings.database while True: try: if self.settings.database != initial_url: self.crash() with closing(Session()) as session: self.logger.info("Expiring edges....") self.expire_edges(session) self.logger.info("Promoting nonauditor approvers in audited groups...") self.promote_nonauditors(session) self.logger.info("Sending emails...") process_async_emails(self.settings, session, datetime.utcnow()) self.logger.info("Pruning old traces....") prune_old_traces(session) session.commit() stats.log_gauge("successful-background-update", 1) stats.log_gauge("failed-background-update", 0) except Exception: stats.log_gauge("successful-background-update", 0) stats.log_gauge("failed-background-update", 1) self.plugins.log_exception(None, None, *sys.exc_info()) self.logger.exception("Unexpected exception occurred in background thread") self.crash() self.logger.debug("Sleeping for {} seconds...".format(self.settings.sleep_interval)) sleep(self.settings.sleep_interval)
def get_pending_request_by_group(session: Session, group: Group) -> List[PermissionRequest]: """Load pending request for a particular group.""" return ( session.query(PermissionRequest) .filter(PermissionRequest.status == "pending", PermissionRequest.group_id == group.id) .all() )
def logdump_group_command(session: Session, group: Group, args: Namespace) -> None: log_entries = session.query(AuditLog).filter( AuditLog.on_group_id == group.id, AuditLog.log_time > args.start_date) if args.end_date: log_entries = log_entries.filter(AuditLog.log_time <= args.end_date) with open_file_or_stdout_for_write(args.outfile) as fh: csv_w = csv.writer(fh) for log_entry in log_entries: if log_entry.on_user: extra = "user: {}".format(log_entry.on_user.username) elif log_entry.on_group: extra = "group: {}".format(log_entry.on_group.groupname) else: extra = "" csv_w.writerow([ log_entry.log_time, log_entry.actor, log_entry.description, log_entry.action, extra, ])
def post(self, *args, **kwargs): # type: (*Any, **Any) -> None supplied_token = self.get_body_argument("token") match = TokenValidate.validator.match(supplied_token) if not match: return self.error(((1, "Token format not recognized"), )) token_name = match.group("token_name") token_secret = match.group("token_secret") username = match.group("name") with closing(Session()) as session: token = UserToken.get_by_value(session, username, token_name) if token is None: return self.error(((2, "Token specified does not exist"), )) if not token.enabled: return self.error(((3, "Token is disabled"), )) if not token.check_secret(token_secret): return self.error(((4, "Token secret mismatch"), )) return self.success({ "owner": username, "identity": str(token), "act_as_owner": True, "valid": True })
def post(self): supplied_token = self.get_body_argument("token") match = TokenValidate.validator.match(supplied_token) if not match: return self.error(((1, "Token format not recognized"), )) sess = Session() token_name = match.group("token_name") token_secret = match.group("token_secret") owner = User.get(sess, name=match.group("name")) token = UserToken.get(sess, owner, token_name) if token is None: return self.error(((2, "Token specified does not exist"), )) if not token.enabled: return self.error(((3, "Token is disabled"), )) if not token.check_secret(token_secret): return self.error(((4, "Token secret mismatch"), )) return self.success({ "owner": owner.username, "identity": str(token), "act_as_owner": True, "valid": True, })
def get(self): fh = StringIO() w_csv = csv.writer(fh, lineterminator="\n") # header w_csv.writerow([ 'username', 'created_at', 'type', 'size', 'fingerprint', 'comment', ]) user_key_list = Session().query( PublicKey, User).filter(User.id == PublicKey.user_id) for key, user in user_key_list: w_csv.writerow([ user.name, key.created_on.isoformat(), key.key_type, key.key_size, key.fingerprint, sshpubkey.PublicKey.from_str(key.public_key).comment, ]) self.set_header("Content-Type", "text/csv") self.write(fh.getvalue())
def get(self): fh = StringIO() w_csv = csv.writer(fh, lineterminator="\n") # header w_csv.writerow( [ "username", "created_at", "type", "size", "fingerprint", "fingerprint_sha256", "comment", ] ) with closing(Session()) as session: user_key_list = session.query(PublicKey, User).filter(User.id == PublicKey.user_id) for key, user in user_key_list: w_csv.writerow( [ user.name, key.created_on.isoformat(), key.key_type, key.key_size, key.fingerprint, key.fingerprint_sha256, key.comment, ] ) self.set_header("Content-Type", "text/csv") self.write(fh.getvalue())
def process_async_emails( settings: Settings, session: Session, now_ts: datetime, dry_run: bool = False ) -> int: """Send emails due before now This method finds and immediately sends any emails that have been scheduled to be sent before the now_ts. Meant to be called from the background processing thread. Args: settings: The current Settings object for this application. session: Object for db session. now_ts: The time to use as the cutoff (send emails before this point). dry_run: If True, do not actually send any email, just generate and return how many emails would have been sent. Returns: Number of emails that were sent. """ emails = ( session.query(AsyncNotification) .filter(AsyncNotification.sent == False, AsyncNotification.send_after < now_ts) .all() ) sent_ct = 0 for email in emails: # For atomicity, attempt to set the sent flag on this email to true if # and only if it's still false. update_ct = ( session.query(AsyncNotification) .filter(AsyncNotification.id == email.id, AsyncNotification.sent == False) .update({"sent": True}) ) # If it's 0, someone else won the race. Bail. if update_ct == 0: continue try: if not dry_run: send_email_raw(settings, email.email, email.body) email.sent = True sent_ct += 1 except smtplib.SMTPException: # Any sort of error with sending the email and we want to move on to # the next email. This email will be retried later. pass return sent_ct
def session(request, tmpdir): db_engine = get_db_engine(db_url(tmpdir)) # Create the database schema and the corresponding session. Model.metadata.create_all(db_engine) Session.configure(bind=db_engine) session = Session() def fin(): # type: () -> None """Explicitly close the session and clean up if using a persistent database.""" session.close() if "MEROU_TEST_DATABASE" in os.environ: Model.metadata.drop_all(db_engine) request.addfinalizer(fin) return session
def get(self, trace_uuid): try: flamegraph_svg = perf_profile.get_flamegraph_svg( Session(), trace_uuid) except perf_profile.InvalidUUID: pass else: self.set_header("Content-Type", "image/svg+xml") self.write(flamegraph_svg)
def start_processor(args, settings, sentry_client): # type: (Namespace, BackgroundSettings, SentryProxy) -> None log_level = logging.getLevelName(logging.getLogger().level) logging.info("begin. log_level={}".format(log_level)) try: plugins = PluginProxy.load_plugins(settings, "grouper-background") set_global_plugin_proxy(plugins) except PluginsDirectoryDoesNotExist as e: logging.fatal("Plugin directory does not exist: {}".format(e)) sys.exit(1) # setup database logging.debug("configure database session") Session.configure(bind=get_db_engine(settings.database)) background = BackgroundProcessor(settings, sentry_client) background.run()
def start_processor(args, settings): # type: (Namespace, BackgroundSettings) -> None log_level = logging.getLevelName(logging.getLogger().level) logging.info("begin. log_level={}".format(log_level)) try: plugins = PluginProxy.load_plugins(settings, "grouper-background") set_global_plugin_proxy(plugins) except PluginsDirectoryDoesNotExist as e: logging.fatal("Plugin directory does not exist: {}".format(e)) sys.exit(1) # setup database logging.debug("configure database session") Session.configure(bind=get_db_engine(settings.database)) background = BackgroundProcessor(settings, plugins) background.run()
def run(self): while True: sleep(self.refresh_interval) logging.debug("Updating Graph from Database.") try: session = Session() self.graph.update_from_db(session) session.close() stats.set_gauge("successful-db-update", 1) except OperationalError: Session.configure(bind=get_db_engine(get_database_url(self.settings))) logging.critical("Failed to connect to database.") stats.set_gauge("successful-db-update", 0) self.capture_exception() except: stats.set_gauge("successful-db-update", 0) self.capture_exception() raise
def start_processor(args, sentry_client): # type: (argparse.Namespace, SentryProxy) -> None log_level = logging.getLevelName(logging.getLogger().level) logging.info("begin. log_level={}".format(log_level)) try: initialize_plugins(settings.plugin_dirs, settings.plugin_module_paths, "grouper-background") except PluginsDirectoryDoesNotExist as e: logging.fatal("Plugin directory does not exist: {}".format(e)) sys.exit(1) # setup database logging.debug("configure database session") Session.configure(bind=get_db_engine(get_database_url(settings))) settings.start_config_thread(args.config, "background") background = BackgroundProcessor(settings, sentry_client) background.run()
def get_or_create_user(self, username): # type: (str) -> Optional[User] """Retrieve or create the User object for the authenticated user. This is done in a separate method called by prepare instead of in the magic Tornado get_current_user method because exceptions thrown by the latter are caught by Tornado and not propagated to the caller, and we want to use exceptions to handle invalid users and then return an error page in prepare. """ if not username: return None # Users must be fully qualified if not re.match("^{}$".format(USERNAME_VALIDATION), username): raise InvalidUser("{} does not match {}".format(username, USERNAME_VALIDATION)) # User must exist in the database and be active try: user, created = User.get_or_create(self.session, username=username) if created: logging.info("Created new user %s", username) self.session.commit() # Because the graph doesn't initialize until the updates table # is populated, we need to refresh the graph here in case this # is the first update. self.graph.update_from_db(self.session) except sqlalchemy.exc.OperationalError: # Failed to connect to database or create user, try to reconfigure the db. This invokes # the fetcher to try to see if our URL string has changed. Session.configure(bind=get_db_engine(settings().database)) raise DatabaseFailure() # service accounts are, by definition, not interactive users if user.is_service_account: raise InvalidUser("{} is a service account".format(username)) return user
def run(self): # type: () -> None while True: try: with closing(Session()) as session: self.logger.info("Expiring edges....") self.expire_edges(session) self.logger.info("Expiring nonauditor approvers in audited groups...") self.expire_nonauditors(session) self.logger.info("Sending emails...") process_async_emails(self.settings, session, datetime.utcnow()) self.logger.info("Pruning old traces....") prune_old_traces(session) session.commit() stats.log_gauge("successful-background-update", 1) stats.log_gauge("failed-background-update", 0) except OperationalError: Session.configure(bind=get_db_engine(get_database_url(self.settings))) self.logger.critical("Failed to connect to database.") stats.log_gauge("successful-background-update", 0) stats.log_gauge("failed-background-update", 1) self._capture_exception() except: stats.log_gauge("successful-background-update", 0) stats.log_gauge("failed-background-update", 1) self._capture_exception() self.logger.exception("Unexpected exception occurred in background thread.") raise self.logger.debug("Sleeping for {} seconds...".format(self.settings.sleep_interval)) sleep(self.settings.sleep_interval)
def make_session(): db_engine = get_db_engine(get_database_url(settings)) Session.configure(bind=db_engine) return Session()
def start_server(args, settings, sentry_client): # type: (Namespace, FrontendSettings, SentryProxy) -> None log_level = logging.getLevelName(logging.getLogger().level) logging.info("begin. log_level={}".format(log_level)) assert not ( settings.debug and settings.num_processes > 1 ), "debug mode does not support multiple processes" try: plugins = PluginProxy.load_plugins(settings, "grouper-fe") set_global_plugin_proxy(plugins) except PluginsDirectoryDoesNotExist as e: logging.fatal("Plugin directory does not exist: {}".format(e)) sys.exit(1) # setup database logging.debug("configure database session") if args.database_url: settings.database = args.database_url Session.configure(bind=get_db_engine(settings.database)) application = create_fe_application(settings, args.deployment_name) ssl_context = plugins.get_ssl_context() if args.listen_stdin: logging.info( "Starting application server with %d processes on stdin", settings.num_processes ) server = HTTPServer(application, ssl_options=ssl_context) if PY2: s = socket.fromfd(sys.stdin.fileno(), socket.AF_INET, socket.SOCK_STREAM) s.setblocking(False) s.listen(5) else: s = socket.socket(fileno=sys.stdin.fileno()) s.setblocking(False) s.listen() server.add_sockets([s]) else: address = args.address or settings.address port = args.port or settings.port logging.info( "Starting application server with %d processes on %s:%d", settings.num_processes, address, port, ) server = HTTPServer(application, ssl_options=ssl_context) server.bind(port, address=address) # When using multiple processes, the forking happens here server.start(settings.num_processes) stats.set_defaults() # Create the Graph and start the graph update thread post fork to ensure each process gets # updated. with closing(Session()) as session: graph = Graph() graph.update_from_db(session) refresher = DbRefreshThread(settings, graph, settings.refresh_interval, sentry_client) refresher.daemon = True refresher.start() try: IOLoop.current().start() except KeyboardInterrupt: IOLoop.current().stop() finally: print("Bye")
def create_session(self): # type: () -> Session db_engine = get_db_engine(self.settings.database) Session.configure(bind=db_engine) return Session()