def get_current_user(self):
username = self.request.headers.get(settings.user_auth_header)
if not username:
return
# Users must be fully qualified
if not re.match("^{}$".format(USERNAME_VALIDATION), username):
raise InvalidUser()
try:
user, created = User.get_or_create(self.session, username=username)
if created:
logging.info("Created new user %s", username)
self.session.commit()
# Because the graph doesn't initialize until the updates table
# is populated, we need to refresh the graph here in case this
# is the first update.
self.graph.update_from_db(self.session)
except sqlalchemy.exc.OperationalError:
# Failed to connect to database or create user, try to reconfigure the db. This invokes
# the fetcher to try to see if our URL string has changed.
Session.configure(bind=get_db_engine(get_database_url(settings)))
raise DatabaseFailure()
return user
def run(self):
while True:
try:
with closing(Session()) as session:
self.logger.debug("Expiring edges....")
self.expire_edges(session)
self.logger.debug(
"Expiring nonauditor approvers in audited groups...")
self.expire_nonauditors(session)
self.logger.debug("Sending emails...")
process_async_emails(self.settings, session,
datetime.utcnow())
self.logger.debug("Pruning old traces....")
prune_old_traces(session)
session.commit()
stats.set_gauge("successful-background-update", 1)
stats.set_gauge("failed-background-update", 0)
except OperationalError:
Session.configure(
bind=get_db_engine(get_database_url(self.settings)))
self.logger.critical("Failed to connect to database.")
stats.set_gauge("successful-background-update", 0)
stats.set_gauge("failed-background-update", 1)
self.capture_exception()
except:
stats.set_gauge("successful-background-update", 0)
stats.set_gauge("failed-background-update", 1)
self.capture_exception()
raise
sleep(60)
def start_server(args, sentry_client):
# type: (Namespace, SentryProxy) -> None
log_level = logging.getLevelName(logging.getLogger().level)
logging.info("begin. log_level={}".format(log_level))
assert not (settings.debug and settings.num_processes > 1
), "debug mode does not support multiple processes"
try:
initialize_plugins(settings.plugin_dirs, settings.plugin_module_paths,
"grouper_fe")
except PluginsDirectoryDoesNotExist as e:
logging.fatal("Plugin directory does not exist: {}".format(e))
sys.exit(1)
# setup database
logging.debug("configure database session")
database_url = args.database_url or get_database_url(settings)
Session.configure(bind=get_db_engine(database_url))
usecase_factory = create_graph_usecase_factory(settings, Session())
application = create_fe_application(settings, usecase_factory,
args.deployment_name)
address = args.address or settings.address
port = args.port or settings.port
ssl_context = get_plugin_proxy().get_ssl_context()
logging.info("Starting application server with %d processes on port %d",
settings.num_processes, port)
server = tornado.httpserver.HTTPServer(application,
ssl_options=ssl_context)
server.bind(port, address=address)
# When using multiple processes, the forking happens here
server.start(settings.num_processes)
stats.set_defaults()
# Create the Graph and start the config / graph update threads post fork to ensure each
# process gets updated.
settings.start_config_thread(args.config, "fe")
with closing(Session()) as session:
graph = Graph()
graph.update_from_db(session)
refresher = DbRefreshThread(settings, graph, settings.refresh_interval,
sentry_client)
refresher.daemon = True
refresher.start()
try:
tornado.ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
tornado.ioloop.IOLoop.instance().stop()
finally:
print("Bye")
def dump_sql_command(args):
# type: (argparse.Namespace) -> None
db_engine = get_db_engine(get_database_url(settings))
for table in Model.metadata.sorted_tables:
print CreateTable(table).compile(db_engine)
for index in table.indexes:
print CreateIndex(index).compile(db_engine)
def get_current_user(self):
username = self.request.headers.get(settings.user_auth_header)
if not username:
return
# Users must be fully qualified
if not re.match("^{}$".format(USERNAME_VALIDATION), username):
raise InvalidUser()
try:
user, created = User.get_or_create(self.session, username=username)
if created:
logging.info("Created new user %s", username)
self.session.commit()
# Because the graph doesn't initialize until the updates table
# is populated, we need to refresh the graph here in case this
# is the first update.
self.graph.update_from_db(self.session)
except sqlalchemy.exc.OperationalError:
# Failed to connect to database or create user, try to reconfigure the db. This invokes
# the fetcher to try to see if our URL string has changed.
Session.configure(bind=get_db_engine(get_database_url(settings)))
raise DatabaseFailure()
return user
def run(self):
while True:
try:
with closing(Session()) as session:
self.logger.debug("Expiring edges....")
self.expire_edges(session)
self.logger.debug("Expiring nonauditor approvers in audited groups...")
self.expire_nonauditors(session)
self.logger.debug("Sending emails...")
process_async_emails(self.settings, session, datetime.utcnow())
self.logger.debug("Pruning old traces....")
prune_old_traces(session)
session.commit()
stats.set_gauge("successful-background-update", 1)
stats.set_gauge("failed-background-update", 0)
except OperationalError:
Session.configure(bind=get_db_engine(get_database_url(self.settings)))
self.logger.critical("Failed to connect to database.")
stats.set_gauge("successful-background-update", 0)
stats.set_gauge("failed-background-update", 1)
self.capture_exception()
except:
stats.set_gauge("successful-background-update", 0)
stats.set_gauge("failed-background-update", 1)
self.capture_exception()
raise
sleep(60)
def dump_sql_command(args):
# type: (argparse.Namespace) -> None
db_engine = get_db_engine(get_database_url(settings))
for table in Model.metadata.sorted_tables:
print CreateTable(table).compile(db_engine)
for index in table.indexes:
print CreateIndex(index).compile(db_engine)
def session(self):
# type: () -> Session
if not self._session:
db_engine = get_db_engine(get_database_url(self.settings))
Session.configure(bind=db_engine)
self._session = Session()
return self._session
def run(self):
initial_url = get_database_url(self.settings)
while True:
self.logger.debug("Updating Graph from Database.")
try:
if get_database_url(self.settings) != initial_url:
self.crash()
with closing(Session()) as session:
self.graph.update_from_db(session)
stats.log_gauge("successful-db-update", 1)
stats.log_gauge("failed-db-update", 0)
except Exception:
stats.log_gauge("successful-db-update", 0)
stats.log_gauge("failed-db-update", 1)
self.capture_exception()
self.crash()
sleep(self.refresh_interval)
def run(self):
while True:
logging.debug("Sending emails...")
try:
session = Session()
self.send_emails(session, datetime.utcnow())
session.commit()
session.close()
except OperationalError:
Session.configure(
bind=get_db_engine(get_database_url(self.settings)))
logging.critical("Failed to connect to database.")
sleep(60)
def run(self):
# type: () -> None
initial_url = get_database_url(self.settings)
while True:
try:
if get_database_url(self.settings) != initial_url:
self.crash()
with closing(Session()) as session:
self.logger.info("Expiring edges....")
self.expire_edges(session)
self.logger.info(
"Promoting nonauditor approvers in audited groups...")
self.promote_nonauditors(session)
self.logger.info("Sending emails...")
process_async_emails(self.settings, session,
datetime.utcnow())
self.logger.info("Pruning old traces....")
prune_old_traces(session)
session.commit()
stats.log_gauge("successful-background-update", 1)
stats.log_gauge("failed-background-update", 0)
except Exception:
stats.log_gauge("successful-background-update", 0)
stats.log_gauge("failed-background-update", 1)
self._capture_exception()
self.logger.exception(
"Unexpected exception occurred in background thread.")
self.crash()
self.logger.debug("Sleeping for {} seconds...".format(
self.settings.sleep_interval))
sleep(self.settings.sleep_interval)
def start_server(args, sentry_client):
# type: (argparse.Namespace, SentryProxy) -> None
log_level = logging.getLevelName(logging.getLogger().level)
logging.info("begin. log_level={}".format(log_level))
assert not (settings.debug and settings.num_processes > 1), \
"debug mode does not support multiple processes"
try:
initialize_plugins(settings.plugin_dirs, settings.plugin_module_paths, "grouper_api")
except PluginsDirectoryDoesNotExist as e:
logging.fatal("Plugin directory does not exist: {}".format(e))
sys.exit(1)
# setup database
logging.debug("configure database session")
database_url = args.database_url or get_database_url(settings)
Session.configure(bind=get_db_engine(database_url))
settings.start_config_thread(args.config, "api")
with closing(Session()) as session:
graph = Graph()
graph.update_from_db(session)
refresher = DbRefreshThread(settings, graph, settings.refresh_interval, sentry_client)
refresher.daemon = True
refresher.start()
application = get_application(graph, settings, sentry_client)
address = args.address or settings.address
port = args.port or settings.port
logging.info("Starting application server on port %d", port)
server = tornado.httpserver.HTTPServer(application)
server.bind(port, address=address)
server.start(settings.num_processes)
stats.set_defaults()
try:
tornado.ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
tornado.ioloop.IOLoop.instance().stop()
finally:
print "Bye"
def sync_db_command(args):
# Models not implicitly or explictly imported above are explicitly imported
# here:
from grouper.models.perf_profile import PerfProfile # noqa
db_engine = get_db_engine(get_database_url(settings))
Model.metadata.create_all(db_engine)
# Add some basic database structures we know we will need if they don't exist.
session = make_session()
for name, description in SYSTEM_PERMISSIONS:
test = Permission.get(session, name)
if test:
continue
permission = Permission(name=name, description=description)
try:
permission.add(session)
session.flush()
except IntegrityError:
session.rollback()
raise Exception('Failed to create permission: %s' % (name, ))
session.commit()
# This group is needed to bootstrap a Grouper installation.
admin_group = Group.get(session, name="grouper-administrators")
if not admin_group:
admin_group = Group(
groupname="grouper-administrators",
description="Administrators of the Grouper system.",
canjoin="nobody",
)
try:
admin_group.add(session)
session.flush()
except IntegrityError:
session.rollback()
raise Exception('Failed to create group: grouper-administrators')
for permission_name in (GROUP_ADMIN, PERMISSION_ADMIN, USER_ADMIN):
permission = Permission.get(session, permission_name)
assert permission, "Permission should have been created earlier!"
grant_permission(session, admin_group.id, permission.id)
session.commit()
def sync_db_command(args):
# Models not implicitly or explictly imported above are explicitly imported
# here:
from grouper.models.perf_profile import PerfProfile # noqa
db_engine = get_db_engine(get_database_url(settings))
Model.metadata.create_all(db_engine)
# Add some basic database structures we know we will need if they don't exist.
session = make_session()
for name, description in SYSTEM_PERMISSIONS:
test = Permission.get(session, name)
if test:
continue
permission = Permission(name=name, description=description)
try:
permission.add(session)
session.flush()
except IntegrityError:
session.rollback()
raise Exception('Failed to create permission: %s' % (name, ))
session.commit()
# This group is needed to bootstrap a Grouper installation.
admin_group = Group.get(session, name="grouper-administrators")
if not admin_group:
admin_group = Group(
groupname="grouper-administrators",
description="Administrators of the Grouper system.",
canjoin="nobody",
)
try:
admin_group.add(session)
session.flush()
except IntegrityError:
session.rollback()
raise Exception('Failed to create group: grouper-administrators')
for permission_name in (GROUP_ADMIN, PERMISSION_ADMIN, USER_ADMIN):
permission = Permission.get(session, permission_name)
assert permission, "Permission should have been created earlier!"
grant_permission(session, admin_group.id, permission.id)
session.commit()
def run(self):
while True:
sleep(self.refresh_interval)
logging.debug("Updating Graph from Database.")
try:
session = Session()
self.graph.update_from_db(session)
session.close()
stats.set_gauge("successful-db-update", 1)
except OperationalError:
Session.configure(bind=get_db_engine(get_database_url(self.settings)))
logging.critical("Failed to connect to database.")
stats.set_gauge("successful-db-update", 0)
self.capture_exception()
except:
stats.set_gauge("successful-db-update", 0)
self.capture_exception()
raise
def start_processor(args, sentry_client):
# type: (argparse.Namespace, SentryProxy) -> None
log_level = logging.getLevelName(logging.getLogger().level)
logging.info("begin. log_level={}".format(log_level))
try:
initialize_plugins(settings.plugin_dirs, settings.plugin_module_paths, "grouper-background")
except PluginsDirectoryDoesNotExist as e:
logging.fatal("Plugin directory does not exist: {}".format(e))
sys.exit(1)
# setup database
logging.debug("configure database session")
Session.configure(bind=get_db_engine(get_database_url(settings)))
settings.start_config_thread(args.config, "background")
background = BackgroundProcessor(settings, sentry_client)
background.run()
def run(self):
while True:
self.logger.debug("Updating Graph from Database.")
try:
with closing(Session()) as session:
self.graph.update_from_db(session)
stats.set_gauge("successful-db-update", 1)
stats.set_gauge("failed-db-update", 0)
except OperationalError:
Session.configure(bind=get_db_engine(get_database_url(self.settings)))
self.logger.critical("Failed to connect to database.")
stats.set_gauge("successful-db-update", 0)
stats.set_gauge("failed-db-update", 1)
self.capture_exception()
except:
stats.set_gauge("successful-db-update", 0)
stats.set_gauge("failed-db-update", 1)
self.capture_exception()
raise
sleep(self.refresh_interval)
def run(self):
while True:
try:
session = Session()
logging.debug("Expiring edges....")
self.expire_edges(session)
logging.debug("Sending emails...")
process_async_emails(self.settings, session, datetime.utcnow())
logging.debug("Pruning old traces....")
prune_old_traces(session)
session.commit()
session.close()
stats.set_gauge("successful-background-run", 1)
except OperationalError:
Session.configure(bind=get_db_engine(get_database_url(self.settings)))
logging.critical("Failed to connect to database.")
stats.set_gauge("successful-background-run", 0)
self.capture_exception()
except:
stats.set_gauge("successful-background-run", 0)
self.capture_exception()
raise
sleep(60)
def run(self):
# type: () -> None
while True:
try:
with closing(Session()) as session:
self.logger.info("Expiring edges....")
self.expire_edges(session)
self.logger.info("Expiring nonauditor approvers in audited groups...")
self.expire_nonauditors(session)
self.logger.info("Sending emails...")
process_async_emails(self.settings, session, datetime.utcnow())
self.logger.info("Pruning old traces....")
prune_old_traces(session)
session.commit()
stats.log_gauge("successful-background-update", 1)
stats.log_gauge("failed-background-update", 0)
except OperationalError:
Session.configure(bind=get_db_engine(get_database_url(self.settings)))
self.logger.critical("Failed to connect to database.")
stats.log_gauge("successful-background-update", 0)
stats.log_gauge("failed-background-update", 1)
self._capture_exception()
except:
stats.log_gauge("successful-background-update", 0)
stats.log_gauge("failed-background-update", 1)
self._capture_exception()
self.logger.exception("Unexpected exception occurred in background thread.")
raise
self.logger.debug("Sleeping for {} seconds...".format(self.settings.sleep_interval))
sleep(self.settings.sleep_interval)
def make_session():
db_engine = get_db_engine(get_database_url(settings))
Session.configure(bind=db_engine)
return Session()
def sync_db_command(args):
# Models not implicitly or explictly imported above are explicitly imported here
from grouper.models.perf_profile import PerfProfile # noqa: F401
from grouper.models.user_token import UserToken # noqa: F401
db_engine = get_db_engine(get_database_url(settings))
Model.metadata.create_all(db_engine)
# Add some basic database structures we know we will need if they don't exist.
session = make_session()
for name, description in SYSTEM_PERMISSIONS:
test = get_permission(session, name)
if test:
continue
try:
create_permission(session, name, description)
session.flush()
except IntegrityError:
session.rollback()
raise Exception("Failed to create permission: %s" % (name, ))
session.commit()
# This group is needed to bootstrap a Grouper installation.
admin_group = Group.get(session, name="grouper-administrators")
if not admin_group:
admin_group = Group(
groupname="grouper-administrators",
description="Administrators of the Grouper system.",
canjoin="nobody",
)
try:
admin_group.add(session)
session.flush()
except IntegrityError:
session.rollback()
raise Exception("Failed to create group: grouper-administrators")
for permission_name in (GROUP_ADMIN, PERMISSION_ADMIN, USER_ADMIN):
permission = get_permission(session, permission_name)
assert permission, "Permission should have been created earlier!"
grant_permission(session, admin_group.id, permission.id)
session.commit()
auditors_group_name = get_auditors_group_name(settings)
auditors_group = Group.get(session, name=auditors_group_name)
if not auditors_group:
auditors_group = Group(
groupname=auditors_group_name,
description=
"Group for auditors, who can be owners of audited groups.",
canjoin="canjoin",
)
try:
auditors_group.add(session)
session.flush()
except IntegrityError:
session.rollback()
raise Exception(
"Failed to create group: {}".format(auditors_group_name))
permission = get_permission(session, PERMISSION_AUDITOR)
assert permission, "Permission should have been created earlier!"
grant_permission(session, auditors_group.id, permission.id)
session.commit()
def make_session():
db_engine = get_db_engine(get_database_url(settings))
Session.configure(bind=db_engine)
return Session()
