예제 #1
0
파일: util.py 프로젝트: rra/grouper
    def get_current_user(self):
        username = self.request.headers.get(settings.user_auth_header)
        if not username:
            return

        # Users must be fully qualified
        if not re.match("^{}$".format(USERNAME_VALIDATION), username):
            raise InvalidUser()

        try:
            user, created = User.get_or_create(self.session, username=username)
            if created:
                logging.info("Created new user %s", username)
                self.session.commit()
                # Because the graph doesn't initialize until the updates table
                # is populated, we need to refresh the graph here in case this
                # is the first update.
                self.graph.update_from_db(self.session)
        except sqlalchemy.exc.OperationalError:
            # Failed to connect to database or create user, try to reconfigure the db. This invokes
            # the fetcher to try to see if our URL string has changed.
            Session.configure(bind=get_db_engine(get_database_url(settings)))
            raise DatabaseFailure()

        return user
예제 #2
0
    def run(self):
        while True:
            try:
                with closing(Session()) as session:
                    self.logger.debug("Expiring edges....")
                    self.expire_edges(session)
                    self.logger.debug(
                        "Expiring nonauditor approvers in audited groups...")
                    self.expire_nonauditors(session)
                    self.logger.debug("Sending emails...")
                    process_async_emails(self.settings, session,
                                         datetime.utcnow())
                    self.logger.debug("Pruning old traces....")
                    prune_old_traces(session)
                    session.commit()

                stats.set_gauge("successful-background-update", 1)
                stats.set_gauge("failed-background-update", 0)
            except OperationalError:
                Session.configure(
                    bind=get_db_engine(get_database_url(self.settings)))
                self.logger.critical("Failed to connect to database.")
                stats.set_gauge("successful-background-update", 0)
                stats.set_gauge("failed-background-update", 1)
                self.capture_exception()
            except:
                stats.set_gauge("successful-background-update", 0)
                stats.set_gauge("failed-background-update", 1)
                self.capture_exception()
                raise

            sleep(60)
예제 #3
0
def start_server(args, sentry_client):
    # type: (Namespace, SentryProxy) -> None

    log_level = logging.getLevelName(logging.getLogger().level)
    logging.info("begin. log_level={}".format(log_level))

    assert not (settings.debug and settings.num_processes > 1
                ), "debug mode does not support multiple processes"

    try:
        initialize_plugins(settings.plugin_dirs, settings.plugin_module_paths,
                           "grouper_fe")
    except PluginsDirectoryDoesNotExist as e:
        logging.fatal("Plugin directory does not exist: {}".format(e))
        sys.exit(1)

    # setup database
    logging.debug("configure database session")
    database_url = args.database_url or get_database_url(settings)
    Session.configure(bind=get_db_engine(database_url))

    usecase_factory = create_graph_usecase_factory(settings, Session())
    application = create_fe_application(settings, usecase_factory,
                                        args.deployment_name)

    address = args.address or settings.address
    port = args.port or settings.port

    ssl_context = get_plugin_proxy().get_ssl_context()

    logging.info("Starting application server with %d processes on port %d",
                 settings.num_processes, port)
    server = tornado.httpserver.HTTPServer(application,
                                           ssl_options=ssl_context)
    server.bind(port, address=address)
    # When using multiple processes, the forking happens here
    server.start(settings.num_processes)

    stats.set_defaults()

    # Create the Graph and start the config / graph update threads post fork to ensure each
    # process gets updated.

    settings.start_config_thread(args.config, "fe")

    with closing(Session()) as session:
        graph = Graph()
        graph.update_from_db(session)

    refresher = DbRefreshThread(settings, graph, settings.refresh_interval,
                                sentry_client)
    refresher.daemon = True
    refresher.start()

    try:
        tornado.ioloop.IOLoop.instance().start()
    except KeyboardInterrupt:
        tornado.ioloop.IOLoop.instance().stop()
    finally:
        print("Bye")
예제 #4
0
def dump_sql_command(args):
    # type: (argparse.Namespace) -> None
    db_engine = get_db_engine(get_database_url(settings))
    for table in Model.metadata.sorted_tables:
        print CreateTable(table).compile(db_engine)
        for index in table.indexes:
            print CreateIndex(index).compile(db_engine)
예제 #5
0
    def get_current_user(self):
        username = self.request.headers.get(settings.user_auth_header)
        if not username:
            return

        # Users must be fully qualified
        if not re.match("^{}$".format(USERNAME_VALIDATION), username):
            raise InvalidUser()

        try:
            user, created = User.get_or_create(self.session, username=username)
            if created:
                logging.info("Created new user %s", username)
                self.session.commit()
                # Because the graph doesn't initialize until the updates table
                # is populated, we need to refresh the graph here in case this
                # is the first update.
                self.graph.update_from_db(self.session)
        except sqlalchemy.exc.OperationalError:
            # Failed to connect to database or create user, try to reconfigure the db. This invokes
            # the fetcher to try to see if our URL string has changed.
            Session.configure(bind=get_db_engine(get_database_url(settings)))
            raise DatabaseFailure()

        return user
예제 #6
0
파일: background.py 프로젝트: rra/grouper
    def run(self):
        while True:
            try:
                with closing(Session()) as session:
                    self.logger.debug("Expiring edges....")
                    self.expire_edges(session)
                    self.logger.debug("Expiring nonauditor approvers in audited groups...")
                    self.expire_nonauditors(session)
                    self.logger.debug("Sending emails...")
                    process_async_emails(self.settings, session, datetime.utcnow())
                    self.logger.debug("Pruning old traces....")
                    prune_old_traces(session)
                    session.commit()

                stats.set_gauge("successful-background-update", 1)
                stats.set_gauge("failed-background-update", 0)
            except OperationalError:
                Session.configure(bind=get_db_engine(get_database_url(self.settings)))
                self.logger.critical("Failed to connect to database.")
                stats.set_gauge("successful-background-update", 0)
                stats.set_gauge("failed-background-update", 1)
                self.capture_exception()
            except:
                stats.set_gauge("successful-background-update", 0)
                stats.set_gauge("failed-background-update", 1)
                self.capture_exception()
                raise

            sleep(60)
예제 #7
0
def dump_sql_command(args):
    # type: (argparse.Namespace) -> None
    db_engine = get_db_engine(get_database_url(settings))
    for table in Model.metadata.sorted_tables:
        print CreateTable(table).compile(db_engine)
        for index in table.indexes:
            print CreateIndex(index).compile(db_engine)
예제 #8
0
 def session(self):
     # type: () -> Session
     if not self._session:
         db_engine = get_db_engine(get_database_url(self.settings))
         Session.configure(bind=db_engine)
         self._session = Session()
     return self._session
예제 #9
0
    def run(self):
        initial_url = get_database_url(self.settings)
        while True:
            self.logger.debug("Updating Graph from Database.")
            try:
                if get_database_url(self.settings) != initial_url:
                    self.crash()
                with closing(Session()) as session:
                    self.graph.update_from_db(session)

                stats.log_gauge("successful-db-update", 1)
                stats.log_gauge("failed-db-update", 0)
            except Exception:
                stats.log_gauge("successful-db-update", 0)
                stats.log_gauge("failed-db-update", 1)
                self.capture_exception()
                self.crash()

            sleep(self.refresh_interval)
예제 #10
0
파일: email.py 프로젝트: zorkian/grouper
 def run(self):
     while True:
         logging.debug("Sending emails...")
         try:
             session = Session()
             self.send_emails(session, datetime.utcnow())
             session.commit()
             session.close()
         except OperationalError:
             Session.configure(
                 bind=get_db_engine(get_database_url(self.settings)))
             logging.critical("Failed to connect to database.")
         sleep(60)
예제 #11
0
    def run(self):
        # type: () -> None
        initial_url = get_database_url(self.settings)
        while True:
            try:
                if get_database_url(self.settings) != initial_url:
                    self.crash()
                with closing(Session()) as session:
                    self.logger.info("Expiring edges....")
                    self.expire_edges(session)

                    self.logger.info(
                        "Promoting nonauditor approvers in audited groups...")
                    self.promote_nonauditors(session)

                    self.logger.info("Sending emails...")
                    process_async_emails(self.settings, session,
                                         datetime.utcnow())

                    self.logger.info("Pruning old traces....")
                    prune_old_traces(session)

                    session.commit()

                stats.log_gauge("successful-background-update", 1)
                stats.log_gauge("failed-background-update", 0)
            except Exception:
                stats.log_gauge("successful-background-update", 0)
                stats.log_gauge("failed-background-update", 1)
                self._capture_exception()
                self.logger.exception(
                    "Unexpected exception occurred in background thread.")
                self.crash()

            self.logger.debug("Sleeping for {} seconds...".format(
                self.settings.sleep_interval))
            sleep(self.settings.sleep_interval)
예제 #12
0
def start_server(args, sentry_client):
    # type: (argparse.Namespace, SentryProxy) -> None

    log_level = logging.getLevelName(logging.getLogger().level)
    logging.info("begin. log_level={}".format(log_level))

    assert not (settings.debug and settings.num_processes > 1), \
        "debug mode does not support multiple processes"

    try:
        initialize_plugins(settings.plugin_dirs, settings.plugin_module_paths, "grouper_api")
    except PluginsDirectoryDoesNotExist as e:
        logging.fatal("Plugin directory does not exist: {}".format(e))
        sys.exit(1)

    # setup database
    logging.debug("configure database session")
    database_url = args.database_url or get_database_url(settings)
    Session.configure(bind=get_db_engine(database_url))

    settings.start_config_thread(args.config, "api")

    with closing(Session()) as session:
        graph = Graph()
        graph.update_from_db(session)

    refresher = DbRefreshThread(settings, graph, settings.refresh_interval, sentry_client)
    refresher.daemon = True
    refresher.start()

    application = get_application(graph, settings, sentry_client)

    address = args.address or settings.address
    port = args.port or settings.port

    logging.info("Starting application server on port %d", port)
    server = tornado.httpserver.HTTPServer(application)
    server.bind(port, address=address)
    server.start(settings.num_processes)

    stats.set_defaults()

    try:
        tornado.ioloop.IOLoop.instance().start()
    except KeyboardInterrupt:
        tornado.ioloop.IOLoop.instance().stop()
    finally:
        print "Bye"
예제 #13
0
파일: sync_db.py 프로젝트: rra/grouper
def sync_db_command(args):
    # Models not implicitly or explictly imported above are explicitly imported
    # here:
    from grouper.models.perf_profile import PerfProfile  # noqa

    db_engine = get_db_engine(get_database_url(settings))
    Model.metadata.create_all(db_engine)

    # Add some basic database structures we know we will need if they don't exist.
    session = make_session()

    for name, description in SYSTEM_PERMISSIONS:
        test = Permission.get(session, name)
        if test:
            continue
        permission = Permission(name=name, description=description)
        try:
            permission.add(session)
            session.flush()
        except IntegrityError:
            session.rollback()
            raise Exception('Failed to create permission: %s' % (name, ))
        session.commit()

    # This group is needed to bootstrap a Grouper installation.
    admin_group = Group.get(session, name="grouper-administrators")
    if not admin_group:
        admin_group = Group(
                groupname="grouper-administrators",
                description="Administrators of the Grouper system.",
                canjoin="nobody",
        )

        try:
            admin_group.add(session)
            session.flush()
        except IntegrityError:
            session.rollback()
            raise Exception('Failed to create group: grouper-administrators')

        for permission_name in (GROUP_ADMIN, PERMISSION_ADMIN, USER_ADMIN):
            permission = Permission.get(session, permission_name)
            assert permission, "Permission should have been created earlier!"
            grant_permission(session, admin_group.id, permission.id)

        session.commit()
예제 #14
0
파일: sync_db.py 프로젝트: graham/grouper
def sync_db_command(args):
    # Models not implicitly or explictly imported above are explicitly imported
    # here:
    from grouper.models.perf_profile import PerfProfile  # noqa

    db_engine = get_db_engine(get_database_url(settings))
    Model.metadata.create_all(db_engine)

    # Add some basic database structures we know we will need if they don't exist.
    session = make_session()

    for name, description in SYSTEM_PERMISSIONS:
        test = Permission.get(session, name)
        if test:
            continue
        permission = Permission(name=name, description=description)
        try:
            permission.add(session)
            session.flush()
        except IntegrityError:
            session.rollback()
            raise Exception('Failed to create permission: %s' % (name, ))
        session.commit()

    # This group is needed to bootstrap a Grouper installation.
    admin_group = Group.get(session, name="grouper-administrators")
    if not admin_group:
        admin_group = Group(
            groupname="grouper-administrators",
            description="Administrators of the Grouper system.",
            canjoin="nobody",
        )

        try:
            admin_group.add(session)
            session.flush()
        except IntegrityError:
            session.rollback()
            raise Exception('Failed to create group: grouper-administrators')

        for permission_name in (GROUP_ADMIN, PERMISSION_ADMIN, USER_ADMIN):
            permission = Permission.get(session, permission_name)
            assert permission, "Permission should have been created earlier!"
            grant_permission(session, admin_group.id, permission.id)

        session.commit()
예제 #15
0
    def run(self):
        while True:
            sleep(self.refresh_interval)

            logging.debug("Updating Graph from Database.")
            try:
                session = Session()
                self.graph.update_from_db(session)
                session.close()
                stats.set_gauge("successful-db-update", 1)
            except OperationalError:
                Session.configure(bind=get_db_engine(get_database_url(self.settings)))
                logging.critical("Failed to connect to database.")
                stats.set_gauge("successful-db-update", 0)
                self.capture_exception()
            except:
                stats.set_gauge("successful-db-update", 0)
                self.capture_exception()
                raise
예제 #16
0
def start_processor(args, sentry_client):
    # type: (argparse.Namespace, SentryProxy) -> None

    log_level = logging.getLevelName(logging.getLogger().level)
    logging.info("begin. log_level={}".format(log_level))

    try:
        initialize_plugins(settings.plugin_dirs, settings.plugin_module_paths, "grouper-background")
    except PluginsDirectoryDoesNotExist as e:
        logging.fatal("Plugin directory does not exist: {}".format(e))
        sys.exit(1)

    # setup database
    logging.debug("configure database session")
    Session.configure(bind=get_db_engine(get_database_url(settings)))

    settings.start_config_thread(args.config, "background")

    background = BackgroundProcessor(settings, sentry_client)
    background.run()
예제 #17
0
    def run(self):
        while True:
            self.logger.debug("Updating Graph from Database.")
            try:
                with closing(Session()) as session:
                    self.graph.update_from_db(session)

                stats.set_gauge("successful-db-update", 1)
                stats.set_gauge("failed-db-update", 0)
            except OperationalError:
                Session.configure(bind=get_db_engine(get_database_url(self.settings)))
                self.logger.critical("Failed to connect to database.")
                stats.set_gauge("successful-db-update", 0)
                stats.set_gauge("failed-db-update", 1)
                self.capture_exception()
            except:
                stats.set_gauge("successful-db-update", 0)
                stats.set_gauge("failed-db-update", 1)
                self.capture_exception()
                raise

            sleep(self.refresh_interval)
예제 #18
0
 def run(self):
     while True:
         try:
             session = Session()
             logging.debug("Expiring edges....")
             self.expire_edges(session)
             logging.debug("Sending emails...")
             process_async_emails(self.settings, session, datetime.utcnow())
             logging.debug("Pruning old traces....")
             prune_old_traces(session)
             session.commit()
             session.close()
             stats.set_gauge("successful-background-run", 1)
         except OperationalError:
             Session.configure(bind=get_db_engine(get_database_url(self.settings)))
             logging.critical("Failed to connect to database.")
             stats.set_gauge("successful-background-run", 0)
             self.capture_exception()
         except:
             stats.set_gauge("successful-background-run", 0)
             self.capture_exception()
             raise
         sleep(60)
예제 #19
0
    def run(self):
        # type: () -> None
        while True:
            try:
                with closing(Session()) as session:
                    self.logger.info("Expiring edges....")
                    self.expire_edges(session)

                    self.logger.info("Expiring nonauditor approvers in audited groups...")
                    self.expire_nonauditors(session)

                    self.logger.info("Sending emails...")
                    process_async_emails(self.settings, session, datetime.utcnow())

                    self.logger.info("Pruning old traces....")
                    prune_old_traces(session)

                    session.commit()

                stats.log_gauge("successful-background-update", 1)
                stats.log_gauge("failed-background-update", 0)
            except OperationalError:
                Session.configure(bind=get_db_engine(get_database_url(self.settings)))
                self.logger.critical("Failed to connect to database.")
                stats.log_gauge("successful-background-update", 0)
                stats.log_gauge("failed-background-update", 1)
                self._capture_exception()
            except:
                stats.log_gauge("successful-background-update", 0)
                stats.log_gauge("failed-background-update", 1)
                self._capture_exception()
                self.logger.exception("Unexpected exception occurred in background thread.")
                raise

            self.logger.debug("Sleeping for {} seconds...".format(self.settings.sleep_interval))
            sleep(self.settings.sleep_interval)
예제 #20
0
def make_session():
    db_engine = get_db_engine(get_database_url(settings))
    Session.configure(bind=db_engine)
    return Session()
예제 #21
0
def sync_db_command(args):
    # Models not implicitly or explictly imported above are explicitly imported here
    from grouper.models.perf_profile import PerfProfile  # noqa: F401
    from grouper.models.user_token import UserToken  # noqa: F401

    db_engine = get_db_engine(get_database_url(settings))
    Model.metadata.create_all(db_engine)

    # Add some basic database structures we know we will need if they don't exist.
    session = make_session()

    for name, description in SYSTEM_PERMISSIONS:
        test = get_permission(session, name)
        if test:
            continue
        try:
            create_permission(session, name, description)
            session.flush()
        except IntegrityError:
            session.rollback()
            raise Exception("Failed to create permission: %s" % (name, ))
        session.commit()

    # This group is needed to bootstrap a Grouper installation.
    admin_group = Group.get(session, name="grouper-administrators")
    if not admin_group:
        admin_group = Group(
            groupname="grouper-administrators",
            description="Administrators of the Grouper system.",
            canjoin="nobody",
        )

        try:
            admin_group.add(session)
            session.flush()
        except IntegrityError:
            session.rollback()
            raise Exception("Failed to create group: grouper-administrators")

        for permission_name in (GROUP_ADMIN, PERMISSION_ADMIN, USER_ADMIN):
            permission = get_permission(session, permission_name)
            assert permission, "Permission should have been created earlier!"
            grant_permission(session, admin_group.id, permission.id)

        session.commit()

    auditors_group_name = get_auditors_group_name(settings)
    auditors_group = Group.get(session, name=auditors_group_name)
    if not auditors_group:
        auditors_group = Group(
            groupname=auditors_group_name,
            description=
            "Group for auditors, who can be owners of audited groups.",
            canjoin="canjoin",
        )

        try:
            auditors_group.add(session)
            session.flush()
        except IntegrityError:
            session.rollback()
            raise Exception(
                "Failed to create group: {}".format(auditors_group_name))

        permission = get_permission(session, PERMISSION_AUDITOR)
        assert permission, "Permission should have been created earlier!"
        grant_permission(session, auditors_group.id, permission.id)

        session.commit()
예제 #22
0
def make_session():
    db_engine = get_db_engine(get_database_url(settings))
    Session.configure(bind=db_engine)
    return Session()