def test_create_bad(self, mock_authzero, mock_secrets, mock_request_get, mock_request_post):
"""
test that we'll correctly fixup profiles on creation
"""
mock_authzero.return_value = "dinopark"
mock_secrets.return_value = "is_pretty_cool"
class FakeResponse:
def __init__(self, fake={}):
self.fake = fake
self.text = str(fake)
def json(self):
return self.fake
def ok(self):
return True
mock_request_post.return_value = FakeResponse()
mock_request_get.return_value = FakeResponse(fake=self.mu)
u = cis_profile.User()
# Bad bad
u.user_id.value = "anotherauser"
u.fun_title.metadata.display = "private"
u.fun_title.value = None
profiles = [u]
publisher = cis_publisher.Publish(profiles, login_method="ad", publisher_name="ldap")
publisher.filter_known_cis_users(profiles)
assert publisher.profiles[0].fun_title.metadata.display != "private"
def test_post_specific_user(self, mock_validate, mock_authzero,
mock_secrets, mock_request_get,
mock_request_post):
mock_authzero.return_value = "dinopark"
mock_secrets.return_value = "is_pretty_cool"
mock_validate.return_value = True
class FakeResponse:
def __init__(self, fake={}):
self.fake = fake
self.text = str(fake)
def json(self):
return self.fake
def ok(self):
return True
mock_request_post.return_value = FakeResponse()
mock_request_get.return_value = FakeResponse()
profiles = [cis_profile.User(user_id="test")]
publisher = cis_publisher.Publish(profiles,
login_method="ad",
publisher_name="ldap")
publisher.post_all(user_ids=["test"])
assert publisher.profiles[0].user_id.value == "test"
def test_known_users(self, mock_request_get, mock_authzero, mock_secrets):
mock_secrets.return_value = "hi"
mock_authzero.return_value = "hi"
class FakeResponse:
def __init__(self, fake={}):
self.fake = fake
self.text = str(fake)
def json(self):
return self.fake
def ok(self):
return True
mu = [{
"user_id": "auser",
"uuid": "093249324",
"primary_email": "*****@*****.**"
}]
mock_request_get.return_value = FakeResponse(fake=mu)
profiles = [cis_profile.User()]
publisher = cis_publisher.Publish(profiles,
login_method="ad",
publisher_name="ldap")
u = publisher.get_known_cis_users()
assert u == mu
def publish(self, user_ids=None, chunk_size=30):
"""
Glue to create or fetch cis_profile.User profiles for this publisher
Then pass everything over to the Publisher class
None, ALL profiles are sent.
@user_ids: list of str - user ids to publish. If None, all users are published.
@chunk_size: int when no user_id is selected, this is the size of the chunk/slice we'll create to divide the
work between function calls (to self)
"""
if user_ids is None:
le = "All"
else:
le = len(user_ids)
logger.info("Starting Auth0 Publisher [{} users]".format(le))
# XXX login_method is overridden when posting the user or listing users, i.e. the one here does not matter
publisher = cis_publisher.Publish([],
login_method="github",
publisher_name="auth0")
# These are the users auth0 knows about
self.az_users = self.fetch_az_users(user_ids)
# Should we fan-out processing to multiple function calls?
if user_ids is None:
# These are the users CIS knows about
all_cis_users = []
for c in self.az_whitelisted_connections:
publisher.login_method = c
publisher.get_known_cis_users(include_inactive=False)
all_cis_users += publisher.known_cis_users_by_user_id.keys()
logger.info(
"Got {} known CIS users for all whitelisted login methods".
format(len(all_cis_users)))
# Because we do not care about most attributes update, we only process new users, or users that will be
# deactivated in order to save time. Note that there is (currently) no auth0 hook to notify of new user
# event, so this (the auth0 publisher that is) function needs to be reasonably fast to avoid delays when
# provisioning users
# So first, remove all known users from the requested list
user_ids_to_process = list(
set(self.get_az_user_ids()) - set(all_cis_users))
# Add blocked users so that they get deactivated
for u in self.az_users:
if u["user_id"] in self.get_az_user_ids():
if "blocked" in u.keys() and u["blocked"] is True:
user_ids_to_process.append(u["user_id"])
logger.info(
"After filtering out known CIS users/in auth0 blocked users, we will process {} users"
.format(len(user_ids_to_process)))
self.fan_out(publisher, chunk_size, user_ids_to_process)
else:
# Don't cache auth0 list if we're just getting a single user, so that we get the most up to date data
# and because it's pretty fast for a single user
if len(user_ids) == 1:
os.environ["CIS_AUTHZERO_CACHE_TIME_SECONDS"] = 0
logger.info(
"CIS_AUTHZERO_CACHE_TIME_SECONDS was set to 0 (caching disabled) for this run"
)
self.process(publisher, user_ids)
def test_user_deactivate(self, mock_cis_user):
hris_data = {}
with open("tests/fixture/workday.json") as fd:
hris_data = json.load(fd)
def side_effect(*args, **kwargs):
fake_user = cis_profile.User(user_id=args[0])
if args[0] == "ad|Mozilla-LDAP|community":
fake_user.staff_information.staff.value = False
else:
fake_user.staff_information.staff.value = True
return fake_user
mock_cis_user.side_effect = side_effect
hris = cis_publisher.hris.HRISPublisher()
publisher = cis_publisher.Publish([],
login_method="ad",
publisher_name="hris")
# we pass 2 fake users, ndonna is in the fixture, nolongerexist is not in the fixture but "CIS" "has it"
publisher.known_cis_users_by_user_id = {
"ad|Mozilla-LDAP|NDonna": "*****@*****.**",
"ad|Mozilla-LDAP|notexist": "*****@*****.**",
"ad|Mozilla-LDAP|community": "*****@*****.**",
}
publisher.known_cis_users_by_email = {
"*****@*****.**": "ad|Mozilla-LDAP|NDonna",
"*****@*****.**": "ad|Mozilla-LDAP|notexist",
"*****@*****.**": "ad|Mozilla-LDAP|community",
}
for uid in publisher.known_cis_users_by_user_id:
p = cis_profile.User(
user_id=uid,
primary_email=publisher.known_cis_users_by_user_id[uid])
if uid == "ad|Mozilla-LDAP|community":
p.staff_information.staff.value = False
else:
p.staff_information.staff.value = True
publisher.all_known_profiles[uid] = p
profiles = hris.convert_hris_to_cis_profiles(
hris_data,
publisher.known_cis_users_by_user_id,
publisher.known_cis_users_by_email,
user_ids=[
"ad|Mozilla-LDAP|NDonna", "ad|Mozilla-LDAP|notexist",
"ad|Mozilla-LDAP|community"
],
)
profiles = hris.deactivate_users(publisher, profiles, hris_data)
# nolongerexist is returned by fake cis reply, but is not in hris workday fixture, so it should be active.value
# = false
# Community doesnt exist in HRIS but should not be touched so we should have 2 profiles back (ie community is
# excluded)
assert len(profiles) == 2
assert profiles[1].active.value is False
assert profiles[1].primary_email.value == "*****@*****.**"
assert profiles[0].active.value is True
assert profiles[0].primary_email.value == "*****@*****.**"
def test_post(self, mock_authzero, mock_secrets, mock_request_post):
mock_authzero.return_value = "dinopark"
mock_secrets.return_value = "is_pretty_cool"
class FakeResponse:
def ok(self):
return True
mock_request_post.return_value = FakeResponse()
profiles = [cis_profile.User()]
cis_publisher.Publish(profiles, login_method="ad", publisher_name="ldap")
def publish(self, user_ids=None, chunk_size=50):
"""
Glue to create or fetch cis_profile.User profiles for this publisher
Then pass everything over to the Publisher class
None, ALL profiles are sent.
@user_ids: list of str - user ids to publish. If None, all users are published.
@chunk_size: int when no user_id is selected, this is the size of the chunk/slice we'll create to divide the
work between function calls (to self)
"""
logger.info("Starting HRIS Publisher")
report_profiles = self.fetch_report()
# Should we fan-out processing to multiple function calls?
if user_ids is None:
all_user_ids = []
if os.environ.get("CIS_ENVIRONMENT") == "development":
user_id_tpl = "ad|Mozilla-LDAP-Dev|"
else:
user_id_tpl = "ad|Mozilla-LDAP|"
for u in report_profiles.get("Report_Entry"):
all_user_ids.append("{}{}".format(user_id_tpl, u.get("PrimaryWorkEmail").split("@")[0]))
sliced = [all_user_ids[i : i + chunk_size] for i in range(0, len(all_user_ids), chunk_size)]
logger.info(
"No user_id selected. Creating slices of work, chunck size: {}, slices: {}, total users: {} and "
"faning-out work to self".format(chunk_size, len(sliced), len(all_user_ids))
)
lambda_client = boto3.client("lambda")
for s in sliced:
lambda_client.invoke(
FunctionName=self.context.function_name, InvocationType="Event", Payload=json.dumps(s)
)
logger.info("Exiting slicing function successfully")
return 0
profiles = self.convert_hris_to_cis_profiles(report_profiles, user_ids)
del report_profiles
logger.info("Processing {} profiles".format(len(profiles)))
publisher = cis_publisher.Publish(profiles, publisher_name="hris", login_method="ad")
failures = []
try:
failures = publisher.post_all(user_ids=user_ids)
except Exception as e:
logger.error("Failed to post_all() HRIS profiles. Trace: {}".format(format_exc()))
raise e
if len(failures) > 0:
logger.error("Failed to post {} profiles: {}".format(len(failures), failures))
def test_filter_cis_users(self, mock_authzero, mock_secrets,
mock_request_get, mock_request_post):
mock_authzero.return_value = "dinopark"
mock_secrets.return_value = "is_pretty_cool"
class FakeResponse:
def __init__(self, fake={}):
self.fake = fake
self.text = str(fake)
def json(self):
return self.fake
def ok(self):
return True
mock_request_post.return_value = FakeResponse()
mu = [{
"user_id": "auser",
"uuid": "0932493241",
"primary_email": "*****@*****.**"
}]
mock_request_get.return_value = FakeResponse(fake=mu)
profiles = [cis_profile.User()]
profiles[0].user_id.value = "auser"
profiles[0].first_name.value = "firstname"
profiles[0].first_name.signature.publisher.name = "wrong"
profiles[0].access_information.hris.values = {"test": "test"}
profiles[0].access_information.hris.signature.publisher.name = "wrong"
profiles[0].access_information.ldap.values = {"test": "test"}
profiles[0].access_information.ldap.signature.publisher.name = "ldap"
publisher = cis_publisher.Publish(profiles,
login_method="ad",
publisher_name="ldap")
publisher.filter_known_cis_users()
# Should be filtered out because publisher = "wrong"
assert publisher.profiles[0].first_name.value != "firstname"
# Should not because publisher = "ldap" and thats "us"
assert publisher.profiles[0].as_dict(
)["access_information"]["ldap"]["values"] == {
"test": "test"
}
# Should be filtered out because publisher = "wrong"
assert publisher.profiles[0].as_dict(
)["access_information"]["hris"]["values"] is None
def publish(self, user_ids=None, chunk_size=30):
"""
Glue to create or fetch cis_profile.User profiles for this publisher
Then pass everything over to the Publisher class
None, ALL profiles are sent.
@user_ids: list of str - user ids to publish. If None, all users are published.
@chunk_size: int when no user_id is selected, this is the size of the chunk/slice we'll create to divide the
work between function calls (to self)
"""
if user_ids is None:
le = "All"
else:
le = len(user_ids)
logger.info("Starting HRIS Publisher [{} users]".format(le))
cache = self.get_s3_cache()
# Get access to the known_users function first
# We override profiles from `publisher` object later on
publisher = cis_publisher.Publish([],
login_method="ad",
publisher_name="hris")
attributes = {"staff_information.staff": True, "active": True}
self.hkey = hash(json.dumps(attributes))
if cache is None:
publisher.get_known_cis_users(include_inactive=True)
publisher.get_known_cis_user_by_attribute_paginated(attributes)
self.fetch_report()
self.save_s3_cache({
"known_profiles":
publisher.known_profiles[self.hkey],
"known_cis_users":
publisher.known_cis_users,
"report":
self.report,
})
else:
publisher.known_profiles[self.hkey] = cache["known_profiles"]
publisher.known_cis_users = cache["known_cis_users"]
for u in publisher.known_cis_users:
publisher.known_cis_users_by_user_id[
u["user_id"]] = u["primary_email"]
publisher.known_cis_users_by_email[
u["primary_email"]] = u["user_id"]
self.report = cache["report"]
# Should we fan-out processing to multiple function calls?
if user_ids is None:
self.fan_out(publisher, chunk_size)
else:
self.process(publisher, user_ids)
def publish(self, user_ids=None):
"""
Glue to create or fetch cis_profile.User profiles for this publisher
Then pass everything over to the Publisher class
None, ALL profiles are sent.
@user_ids: list of str - user ids to publish. If None, all users are published.
"""
logger.info("Starting LDAP Publisher")
profiles_xz = self.fetch_from_s3()
# If there are memory issues here, use lzma.LZMADecompressor() instead
raw = lzma.decompress(profiles_xz)
profiles_json = json.loads(raw)
# Free some memory
del profiles_xz
del raw
profiles = []
logger.info("Processing {} profiles".format(len(profiles_json)))
for p in profiles_json:
str_p = json.dumps(profiles_json[p])
if (user_ids is None) or (profiles_json[p]["user_id"]["value"]
in user_ids):
profiles.append(cis_profile.User(user_structure_json=str_p))
logger.info("Will publish {} profiles".format(len(profiles)))
publisher = cis_publisher.Publish(profiles,
publisher_name="ldap",
login_method="ad")
failures = []
try:
publisher.filter_known_cis_users()
failures = publisher.post_all(user_ids=user_ids)
except Exception as e:
logger.error(
"Failed to post_all() LDAP profiles. Trace: {}".format(
format_exc()))
raise e
if len(failures) > 0:
logger.error("Failed to post {} profiles: {}".format(
len(failures), failures))
def test_known_users(self, mock_request_get, mock_authzero, mock_secrets):
mock_secrets.return_value = "hi"
mock_authzero.return_value = "hi"
class FakeResponse:
def __init__(self, fake={}):
self.fake = fake
self.text = str(fake)
def json(self):
return self.fake
def ok(self):
return True
mock_request_get.return_value = FakeResponse(fake=self.mu)
profiles = [cis_profile.User()]
publisher = cis_publisher.Publish(profiles, login_method="ad", publisher_name="ldap")
u = publisher.get_known_cis_users()
assert u == self.mu["users"]
def test_known_users_by_attribute(self, mock_request_get, mock_authzero, mock_secrets):
mock_secrets.return_value = "hi"
mock_authzero.return_value = "hi"
class FakeResponse:
def __init__(self, fake={}):
self.fake = fake
self.text = str(fake)
def json(self):
return self.fake
def ok(self):
return True
mock_request_get.return_value = FakeResponse(fake=self.mu2)
profiles = [cis_profile.User()]
publisher = cis_publisher.Publish(profiles, login_method="ad", publisher_name="ldap")
attributes = {"staff_information.staff": True, "active": True}
u = publisher.get_known_cis_user_by_attribute_paginated(attributes)
print(u)
assert u["auser"] == self.mu2["users"][0]
def test_profile_validate(self):
profiles = [cis_profile.User()]
publisher = cis_publisher.Publish(profiles,
login_method="ad",
publisher_name="ldap")
publisher.validate()
def test_obj(self):
profiles = [cis_profile.User()]
publisher = cis_publisher.Publish(profiles,
login_method="ad",
publisher_name="ldap")
assert isinstance(publisher, object)
def __init__(self):
self.secret_manager = cis_publisher.secret.Manager()
self.publisher = cis_publisher.Publish([], login_method=None, publisher_name="mozilliansorg")
