def test_exception_plugin():
# type: () -> None
test_logger = ExceptionLoggerTestPlugin()
proxy = PluginProxy([test_logger])
try:
raise RandomException("some string")
except RandomException:
proxy.log_exception(None, None, *sys.exc_info())
assert test_logger.request is None
assert test_logger.status is None
assert test_logger.exc_type == RandomException
assert str(test_logger.exc_value) == "some string"
assert test_logger.exc_tb is not None
# Reinitializing test_logger unconfuses mypy, which otherwise thinks that test_logger.request
# must still be None. See mypy/issues/4168.
test_logger = ExceptionLoggerTestPlugin()
proxy = PluginProxy([test_logger])
try:
raise RandomException("with request")
except RandomException:
request = HTTPRequest("GET", "/foo")
proxy.log_exception(request, 200, *sys.exc_info())
assert test_logger.request is not None
assert test_logger.request.uri == "/foo"
assert test_logger.status == 200
assert test_logger.exc_type == RandomException
assert str(test_logger.exc_value) == "with request"
assert test_logger.exc_tb is not None
def test_rejects_ecdsa_keys(get_plugin_proxy, session, users): # noqa: F811
get_plugin_proxy.return_value = PluginProxy([SshKeyPolicyPlugin()])
user = users["*****@*****.**"]
with pytest.raises(BadPublicKey):
add_public_key(session, user, SSH_KEY_ECDSA_P256)
def __init__(self, tmpdir):
# type: (LocalPath) -> None
self.settings = Settings()
self.settings.database = db_url(tmpdir)
self.plugins = PluginProxy([])
# Reinitialize the global plugin proxy with an empty set of plugins in case a previous test
# initialized plugins. This can go away once a plugin proxy is injected into everything
# that needs it instead of maintained as a global.
set_global_plugin_proxy(self.plugins)
self.initialize_database()
self.session = SessionFactory(self.settings).create_session()
self.graph = GroupGraph()
session_factory = SingletonSessionFactory(self.session)
self.repository_factory = GraphRepositoryFactory(
self.settings, self.plugins, session_factory, self.graph)
self.sql_repository_factory = SQLRepositoryFactory(
self.settings, self.plugins, session_factory)
self.service_factory = ServiceFactory(self.settings, self.plugins,
self.repository_factory)
self.usecase_factory = UseCaseFactory(self.settings, self.plugins,
self.service_factory)
self._transaction_service = self.service_factory.create_transaction_service(
)
def test_rejects_weak_rsa_keys(get_plugin_proxy, session, users):
get_plugin_proxy.return_value = PluginProxy([SshKeyPolicyPlugin()])
user = users["*****@*****.**"]
with pytest.raises(BadPublicKey):
add_public_key(session, user, SSH_KEY_RSA_1024)
def main(sys_argv=sys.argv):
# type: (List[str]) -> None
setup_signal_handlers()
# The curl HTTP client is required to support proxies.
AsyncHTTPClient.configure(CurlAsyncHTTPClient)
# get arguments
parser = build_arg_parser("Grouper Web Server.")
args = parser.parse_args(sys_argv[1:])
try:
settings = FrontendSettings.global_settings_from_config(args.config)
setup_logging(args, settings.log_format)
plugins = PluginProxy.load_plugins(settings, "grouper-fe")
set_global_plugin_proxy(plugins)
except PluginsDirectoryDoesNotExist as e:
logging.fatal("Plugin directory does not exist: {}".format(e))
sys.exit(1)
except Exception:
logging.exception("Uncaught exception in startup")
sys.exit(1)
try:
start_server(args, settings, plugins)
except Exception:
plugins.log_exception(None, None, *sys.exc_info())
logging.exception("Uncaught exception")
finally:
logging.info("end")
def initialize_plugins(plugin_dirs, plugin_module_paths, service_name):
# type: (List[str], List[str], str) -> None
plugins = load_plugins(BasePlugin, plugin_dirs, plugin_module_paths,
service_name)
global _plugin_proxy
_plugin_proxy = PluginProxy(plugins)
def test_group_add_remove_owner(
make_session,
get_plugin_proxy,
session,
users,
groups # noqa: F811
):
make_session.return_value = session
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
username = "******"
groupname = "team-sre"
# add
assert (u"User", username) not in groups[groupname].my_members()
call_main(session, "group", "add_member", "--owner", groupname, username)
all_members = Group.get(session, name=groupname).my_members()
assert (u"User", username) in all_members
_, _, _, role, _, _ = all_members[(u"User", username)]
assert GROUP_EDGE_ROLES[role] == "owner"
# remove (fails)
call_main(session, "group", "remove_member", groupname, username)
assert (u"User", username) in Group.get(session,
name=groupname).my_members()
def session(request, tmpdir):
# type: (FixtureRequest, LocalPath) -> None
settings = Settings()
set_global_settings(settings)
# Reinitialize plugins in case a previous test configured some.
set_global_plugin_proxy(PluginProxy([]))
db_engine = get_db_engine(db_url(tmpdir))
# Clean up from previous tests if using a persistent database.
if "MEROU_TEST_DATABASE" in os.environ:
Model.metadata.drop_all(db_engine)
# Create the database schema and the corresponding session.
Model.metadata.create_all(db_engine)
Session.configure(bind=db_engine)
session = Session()
def fin():
# type: () -> None
"""Explicitly close the session to avoid any dangling transactions."""
session.close()
request.addfinalizer(fin)
return session
def main(sys_argv=sys.argv):
# type: (List[str]) -> None
setup_signal_handlers()
# get arguments
parser = build_arg_parser("Grouper API Server")
args = parser.parse_args(sys_argv[1:])
try:
settings = ApiSettings.global_settings_from_config(args.config)
setup_logging(args, settings.log_format)
plugins = PluginProxy.load_plugins(settings, "grouper-api")
set_global_plugin_proxy(plugins)
except PluginsDirectoryDoesNotExist as e:
logging.fatal("Plugin directory does not exist: %s", e)
sys.exit(1)
except Exception:
logging.exception("Uncaught exception in startup")
sys.exit(1)
try:
start_server(args, settings, plugins)
except Exception:
plugins.log_exception(None, None, *sys.exc_info())
logging.exception("Uncaught exception")
finally:
logging.info("end")
def test_accepts_strong_keys(get_plugin_proxy, session, users): # noqa: F811
get_plugin_proxy.return_value = PluginProxy([SshKeyPolicyPlugin()])
user = users["*****@*****.**"]
add_public_key(session, user, SSH_KEY_1)
add_public_key(session, user, SSH_KEY_ED25519)
def test_can_add_owner_twice(get_plugin_proxy, session, groups, users):
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
group = groups["team-infra"]
owner = users["*****@*****.**"]
add_member(group, owner, role="owner")
add_member(group, owner, role="owner")
def test_can_disable_member(get_plugin_proxy, session, groups, users):
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
group = groups["team-infra"]
member = users["*****@*****.**"]
add_member(group, member)
disable_user(session, member)
def test_rejected_key(get_plugin_proxy, session, users):
get_plugin_proxy.return_value = PluginProxy([PublicKeyPlugin()])
user = users["*****@*****.**"]
with pytest.raises(BadPublicKey):
add_public_key(session, user, SSH_KEY_1)
assert get_public_keys_of_user(session, user.id) == []
def api_app(session, standard_graph):
# type: (Session, GroupGraph) -> GrouperApplication
settings = ApiSettings()
set_global_settings(settings)
session_factory = SingletonSessionFactory(session)
usecase_factory = create_graph_usecase_factory(
settings, PluginProxy([]), session_factory, standard_graph
)
return create_api_application(standard_graph, settings, usecase_factory)
def test_cant_disable_last_owner(get_plugin_proxy, session, groups, users):
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
group = groups["team-infra"]
owner = users["*****@*****.**"]
add_member(group, owner, role="owner")
with pytest.raises(PluginRejectedDisablingUser):
disable_user(session, owner)
def test_cant_demote_last_owner(get_plugin_proxy, groups, users):
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
group = groups["team-infra"]
owner = users["*****@*****.**"]
add_member(group, owner, role="owner")
with pytest.raises(PluginRejectedGroupMembershipUpdate):
group.edit_member(owner, owner, "Unit Testing", role="member")
def test_cant_expire_last_owner(get_plugin_proxy, groups, users):
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
group = groups["team-infra"]
owner = users["*****@*****.**"]
expiration = datetime.utcnow() + timedelta(1)
add_member(group, owner, role="owner")
with pytest.raises(PluginRejectedGroupMembershipUpdate):
group.edit_member(owner, owner, "Unit Testing", expiration=expiration)
def test_exception_plugin():
# type: () -> None
test_logger = ExceptionLoggerTestPlugin()
proxy = PluginProxy([test_logger])
try:
raise RandomException("some string")
except RandomException:
proxy.log_exception(None, None, *sys.exc_info())
assert test_logger.request is None
assert test_logger.status is None
assert test_logger.exc_type == RandomException
assert str(test_logger.exc_value) == "some string"
assert test_logger.exc_tb is not None
try:
raise RandomException("with request")
except RandomException:
request = HTTPRequest("GET", "/foo")
proxy.log_exception(request, 200, *sys.exc_info())
assert test_logger.request is not None
assert test_logger.request.uri == "/foo"
assert test_logger.status == 200
assert test_logger.exc_type == RandomException
assert str(test_logger.exc_value) == "with request"
assert test_logger.exc_tb is not None
def test_user_created_plugin(setup: SetupTest):
"""Test calls to the user_created plugin."""
plugin = UserCreatedPlugin()
# WARN: Relies on the user_created function being called from the global proxy.
# Will need to change once everything uses an injected plugin proxy.
set_global_plugin_proxy(PluginProxy([plugin]))
with setup.transaction():
setup.create_user("*****@*****.**")
assert plugin.calls == 1
plugin.expected_service_account = True
setup.create_service_account("*****@*****.**", "owner", "machine set", "desc")
assert plugin.calls == 2
def test_can_always_revoke_members(get_plugin_proxy, groups, users):
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
group = groups["team-infra"]
owner = users["*****@*****.**"]
member = users["*****@*****.**"]
expiration = datetime.utcnow() + timedelta(1)
add_member(group, owner, role="owner", expiration=expiration)
add_member(group, member)
revoke_member(group, member)
def __init__(self, tmpdir):
# type: (LocalPath) -> None
self.settings = Settings()
self.settings.database = db_url(tmpdir)
self.plugins = PluginProxy([])
self.graph = GroupGraph()
# Reinitialize the global plugin proxy with an empty set of plugins in case a previous test
# initialized plugins. This can go away once a plugin proxy is injected into everything
# that needs it instead of maintained as a global.
set_global_plugin_proxy(self.plugins)
self.initialize_database()
self.open_database()
def test_cant_revoke_last_permanent_owner(get_plugin_proxy, groups, users):
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
group = groups["team-infra"]
first_owner = users["*****@*****.**"]
second_owner = users["*****@*****.**"]
expiration = datetime.utcnow() + timedelta(1)
add_member(group, first_owner, role="owner", expiration=expiration)
add_member(group, second_owner, role="owner")
with pytest.raises(PluginRejectedGroupMembershipUpdate):
revoke_member(group, second_owner)
def test_group_disable_group_owner(user_make_session, group_make_session, get_plugin_proxy, session,
users, groups):
group_make_session.return_value = session
user_make_session.return_value = session
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
username = '******'
groupname = 'team-sre'
# add
call_main('group', 'add_member', '--owner', groupname, username)
assert (u'User', username) in Group.get(session, name=groupname).my_members()
# disable (fails)
call_main('user', 'disable', username)
assert (u'User', username) in Group.get(session, name=groupname).my_members()
def test_group_disable_group_owner(get_plugin_proxy, session, tmpdir, users,
groups): # noqa: F811
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
username = "******"
groupname = "team-sre"
# add
call_main(session, tmpdir, "group", "add_member", "--owner", groupname,
username)
assert (u"User", username) in Group.get(session,
name=groupname).my_members()
# disable (fails)
call_main(session, tmpdir, "user", "disable", username)
assert (u"User", username) in Group.get(session,
name=groupname).my_members()
def test_user_created_plugin(mocker, session, users, groups):
"""Test calls to the user_created plugin."""
plugin = UserCreatedPlugin()
mocker.patch('grouper.models.user.get_plugin_proxy',
return_value=PluginProxy([plugin]))
# Create a regular user. The service account flag should be false, and the plugin should be
# called.
user, created = User.get_or_create(session, username="******")
assert created == True
assert plugin.calls == 1
# Create a role user. This should cause another plugin call and the service account flag
# should now be true.
plugin.expected_service_account = True
create_role_user(session, user, "*****@*****.**", "description", "canask")
assert plugin.calls == 2
def test_cant_revoke_last_npowner(get_plugin_proxy, session, groups, users):
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
group = groups["team-infra"]
first_owner = users["*****@*****.**"]
second_owner = users["*****@*****.**"]
add_member(group, first_owner, role="np-owner")
add_member(group, second_owner, role="np-owner")
# Revoking the first owner does not raise an exception
revoke_member(group, first_owner)
session.commit()
with pytest.raises(PluginRejectedGroupMembershipUpdate):
revoke_member(group, second_owner)
def start_processor(args, settings, sentry_client):
# type: (Namespace, BackgroundSettings, SentryProxy) -> None
log_level = logging.getLevelName(logging.getLogger().level)
logging.info("begin. log_level={}".format(log_level))
try:
plugins = PluginProxy.load_plugins(settings, "grouper-background")
set_global_plugin_proxy(plugins)
except PluginsDirectoryDoesNotExist as e:
logging.fatal("Plugin directory does not exist: {}".format(e))
sys.exit(1)
# setup database
logging.debug("configure database session")
Session.configure(bind=get_db_engine(settings.database))
background = BackgroundProcessor(settings, sentry_client)
background.run()
def start_processor(args, settings):
# type: (Namespace, BackgroundSettings) -> None
log_level = logging.getLevelName(logging.getLogger().level)
logging.info("begin. log_level={}".format(log_level))
try:
plugins = PluginProxy.load_plugins(settings, "grouper-background")
set_global_plugin_proxy(plugins)
except PluginsDirectoryDoesNotExist as e:
logging.fatal("Plugin directory does not exist: {}".format(e))
sys.exit(1)
# setup database
logging.debug("configure database session")
Session.configure(bind=get_db_engine(settings.database))
background = BackgroundProcessor(settings, plugins)
background.run()
def test_group_add_remove_owner(make_session, get_plugin_proxy, session, users,
groups):
make_session.return_value = session
get_plugin_proxy.return_value = PluginProxy([GroupOwnershipPolicyPlugin()])
username = '******'
groupname = 'team-sre'
# add
assert (u'User', username) not in groups[groupname].my_members()
call_main('group', 'add_member', '--owner', groupname, username)
all_members = Group.get(session, name=groupname).my_members()
assert (u'User', username) in all_members
_, _, _, role, _, _ = all_members[(u'User', username)]
assert GROUP_EDGE_ROLES[role] == "owner"
# remove (fails)
call_main('group', 'remove_member', groupname, username)
assert (u'User', username) in Group.get(session,
name=groupname).my_members()
def start_server(args, settings, sentry_client):
# type: (Namespace, FrontendSettings, SentryProxy) -> None
log_level = logging.getLevelName(logging.getLogger().level)
logging.info("begin. log_level={}".format(log_level))
assert not (
settings.debug and settings.num_processes > 1
), "debug mode does not support multiple processes"
try:
plugins = PluginProxy.load_plugins(settings, "grouper-fe")
set_global_plugin_proxy(plugins)
except PluginsDirectoryDoesNotExist as e:
logging.fatal("Plugin directory does not exist: {}".format(e))
sys.exit(1)
# setup database
logging.debug("configure database session")
if args.database_url:
settings.database = args.database_url
Session.configure(bind=get_db_engine(settings.database))
application = create_fe_application(settings, args.deployment_name)
ssl_context = plugins.get_ssl_context()
if args.listen_stdin:
logging.info(
"Starting application server with %d processes on stdin", settings.num_processes
)
server = HTTPServer(application, ssl_options=ssl_context)
if PY2:
s = socket.fromfd(sys.stdin.fileno(), socket.AF_INET, socket.SOCK_STREAM)
s.setblocking(False)
s.listen(5)
else:
s = socket.socket(fileno=sys.stdin.fileno())
s.setblocking(False)
s.listen()
server.add_sockets([s])
else:
address = args.address or settings.address
port = args.port or settings.port
logging.info(
"Starting application server with %d processes on %s:%d",
settings.num_processes,
address,
port,
)
server = HTTPServer(application, ssl_options=ssl_context)
server.bind(port, address=address)
# When using multiple processes, the forking happens here
server.start(settings.num_processes)
stats.set_defaults()
# Create the Graph and start the graph update thread post fork to ensure each process gets
# updated.
with closing(Session()) as session:
graph = Graph()
graph.update_from_db(session)
refresher = DbRefreshThread(settings, graph, settings.refresh_interval, sentry_client)
refresher.daemon = True
refresher.start()
try:
IOLoop.current().start()
except KeyboardInterrupt:
IOLoop.current().stop()
finally:
print("Bye")
def main(sys_argv=sys.argv, session=None):
# type: (List[str], Optional[Session]) -> None
description_msg = "Grouper Control"
parser = argparse.ArgumentParser(description=description_msg)
parser.add_argument(
"-c", "--config", default=default_settings_path(), help="Path to config file."
)
parser.add_argument(
"-d", "--database-url", type=str, default=None, help="Override database URL in config."
)
parser.add_argument(
"-q", "--quiet", action="count", default=0, help="Decrease logging verbosity."
)
parser.add_argument(
"-v", "--verbose", action="count", default=0, help="Increase logging verbosity."
)
parser.add_argument(
"-V",
"--version",
action="version",
version="%%(prog)s %s" % __version__,
help="Display version information.",
)
subparsers = parser.add_subparsers(dest="command")
CtlCommandFactory.add_all_parsers(subparsers)
# Add parsers for legacy commands that have not been refactored.
for subcommand_module in [group, oneoff, service_account, shell]:
subcommand_module.add_parser(subparsers) # type: ignore
args = parser.parse_args(sys_argv[1:])
# Construct the CtlSettings object used for all commands, and set it as the global Settings
# object. All code in grouper.ctl.* takes the CtlSettings object as an argument if needed, but
# it may call other legacy code that requires the global Settings object be present.
settings = CtlSettings.global_settings_from_config(args.config)
if args.database_url:
settings.database = args.database_url
# Construct a session factory, which is passed into all the legacy commands that haven't been
# converted to usecases yet.
if session:
session_factory = SingletonSessionFactory(session) # type: SessionFactory
else:
session_factory = SessionFactory(settings)
log_level = get_loglevel(args, base=logging.INFO)
logging.basicConfig(level=log_level, format=settings.log_format)
if log_level < 0:
sa_log.setLevel(logging.INFO)
# Initialize plugins. The global plugin proxy is used by legacy code.
try:
plugins = PluginProxy.load_plugins(settings, "grouper-ctl")
except PluginsDirectoryDoesNotExist as e:
logging.fatal("Plugin directory does not exist: {}".format(e))
sys.exit(1)
set_global_plugin_proxy(plugins)
# Set up factories.
usecase_factory = create_sql_usecase_factory(settings, plugins, session_factory)
command_factory = CtlCommandFactory(settings, usecase_factory)
# Old-style subcommands store a func in callable when setting up their arguments. New-style
# subcommands are handled via a factory that constructs and calls the correct object.
if getattr(args, "func", None):
args.func(args, settings, session_factory)
else:
command = command_factory.construct_command(args.command)
command.run(args)
def start_server(args, settings, sentry_client):
# type: (Namespace, FrontendSettings, SentryProxy) -> None
log_level = logging.getLevelName(logging.getLogger().level)
logging.info("begin. log_level={}".format(log_level))
assert not (settings.debug and settings.num_processes > 1
), "debug mode does not support multiple processes"
try:
plugins = PluginProxy.load_plugins(settings, "grouper-fe")
set_global_plugin_proxy(plugins)
except PluginsDirectoryDoesNotExist as e:
logging.fatal("Plugin directory does not exist: {}".format(e))
sys.exit(1)
# setup database
logging.debug("configure database session")
if args.database_url:
settings.database = args.database_url
Session.configure(bind=get_db_engine(settings.database))
application = create_fe_application(settings, args.deployment_name)
ssl_context = plugins.get_ssl_context()
if args.listen_stdin:
logging.info("Starting application server with %d processes on stdin",
settings.num_processes)
server = HTTPServer(application, ssl_options=ssl_context)
if PY2:
s = socket.fromfd(sys.stdin.fileno(), socket.AF_INET,
socket.SOCK_STREAM)
s.setblocking(False)
s.listen(5)
else:
s = socket.socket(fileno=sys.stdin.fileno())
s.setblocking(False)
s.listen()
server.add_sockets([s])
else:
address = args.address or settings.address
port = args.port or settings.port
logging.info(
"Starting application server with %d processes on %s:%d",
settings.num_processes,
address,
port,
)
server = HTTPServer(application, ssl_options=ssl_context)
server.bind(port, address=address)
# When using multiple processes, the forking happens here
server.start(settings.num_processes)
stats.set_defaults()
# Create the Graph and start the graph update thread post fork to ensure each process gets
# updated.
with closing(Session()) as session:
graph = Graph()
graph.update_from_db(session)
refresher = DbRefreshThread(settings, graph, settings.refresh_interval,
sentry_client)
refresher.daemon = True
refresher.start()
try:
IOLoop.current().start()
except KeyboardInterrupt:
IOLoop.current().stop()
finally:
print("Bye")
def test_machine_set_plugin(
mocker,
session,
standard_graph,
graph,
http_client,
base_url # noqa: F811
):
mocker.patch("grouper.service_account.get_plugin_proxy",
return_value=PluginProxy([MachineSetPlugin()]))
admin = "*****@*****.**"
# Edit the metadata of an existing service account. This should fail (although return 200)
# including the appropriate error.
update = {
"description": "some service account",
"machine_set": "not valid"
}
fe_url = url(base_url, "/groups/team-sre/service/[email protected]/edit")
resp = yield http_client.fetch(fe_url,
method="POST",
headers={"X-Grouper-User": admin},
body=urlencode(update))
assert resp.code == 200
assert "[email protected] has invalid machine set" in resp.body
graph.update_from_db(session)
metadata = graph.user_metadata["*****@*****.**"]
assert metadata["service_account"]["machine_set"] == "some machines"
# Use a valid machine set, and then this should go through.
update["machine_set"] = "is okay"
resp = yield http_client.fetch(fe_url,
method="POST",
headers={"X-Grouper-User": admin},
body=urlencode(update))
assert resp.code == 200
graph.update_from_db(session)
metadata = graph.user_metadata["*****@*****.**"]
assert metadata["service_account"]["machine_set"] == "is okay"
# Try creating a new service account with an invalid machine set.
data = {
"name": "*****@*****.**",
"description": "some other service account",
"machine_set": "not valid",
}
fe_url = url(base_url, "/groups/team-sre/service/create")
resp = yield http_client.fetch(fe_url,
method="POST",
headers={"X-Grouper-User": admin},
body=urlencode(data))
assert resp.code == 200
assert "[email protected] has invalid machine set" in resp.body
graph.update_from_db(session)
assert "*****@*****.**" not in graph.users
# But this should go through with a valid machine set.
data["machine_set"] = "is okay"
resp = yield http_client.fetch(fe_url,
method="POST",
headers={"X-Grouper-User": admin},
body=urlencode(data))
assert resp.code == 200
graph.update_from_db(session)
metadata = graph.user_metadata["*****@*****.**"]
assert metadata["service_account"][
"description"] == "some other service account"
assert metadata["service_account"]["machine_set"] == "is okay"
