def _create_without_transaction(self, user_profile):
if user_profile["sequence_number"] is None:
user_profile["sequence_number"] = str(uuid.uuid4().int)
cis_profile_user_object = User(
user_structure_json=json.loads(user_profile["profile"]))
return self.table.put_item(
Item={
"id":
user_profile["id"],
"user_uuid":
user_profile["user_uuid"],
"profile":
user_profile["profile"],
"primary_email":
user_profile["primary_email"],
"primary_username":
user_profile["primary_username"],
"sequence_number":
user_profile["sequence_number"],
"active":
bool(json.loads(user_profile["profile"])["active"]["value"]),
"flat_profile": {
k: self.deserializer.deserialize(v)
for k, v in
cis_profile_user_object.as_dynamo_flat_dict().items()
},
})
def _create_items_with_transaction(self, list_of_profiles):
transact_items = []
for user_profile in list_of_profiles:
if user_profile["sequence_number"] is None:
user_profile["sequence_number"] = str(uuid.uuid4().int)
# XXX TBD cover this with tests. Currently dynalite does not support tests for transactions.
cis_profile_user_object = User(user_structure_json=json.loads(user_profile["profile"]))
transact_item = {
"Put": {
"Item": {
"id": {"S": user_profile["id"]},
"user_uuid": {"S": user_profile["user_uuid"]},
"profile": {"S": user_profile["profile"]},
"primary_email": {"S": user_profile["primary_email"]},
"primary_username": {"S": user_profile["primary_username"]},
"sequence_number": {"S": user_profile["sequence_number"]},
"active": {"BOOL": json.loads(user_profile["profile"])["active"]["value"]},
"flat_profile": {"M": cis_profile_user_object.as_dynamo_flat_dict()},
},
"ConditionExpression": "attribute_not_exists(id)",
"TableName": self.table.name,
"ReturnValuesOnConditionCheckFailure": "NONE",
}
}
transact_items.append(transact_item)
logger.debug("Attempting to create batch of transactions for: {}".format(transact_items))
return self._run_transaction(transact_items)
def _update_batch_with_transaction(self, list_of_profiles):
transact_items = []
for user_profile in list_of_profiles:
cis_profile_user_object = User(user_structure_json=json.loads(user_profile["profile"]))
logger.info(cis_profile_user_object.as_dynamo_flat_dict())
transact_item = {
"Update": {
"Key": {"id": {"S": user_profile["id"]}},
"ExpressionAttributeValues": {
":p": {"S": user_profile["profile"]},
":u": {"S": user_profile["user_uuid"]},
":pe": {"S": user_profile["primary_email"]},
":pn": {"S": user_profile["primary_username"]},
":sn": {"S": user_profile["sequence_number"]},
":a": {"BOOL": json.loads(user_profile["profile"])["active"]["value"]},
":fp": {"M": cis_profile_user_object.as_dynamo_flat_dict()},
},
"ConditionExpression": "attribute_exists(id)",
"UpdateExpression": "SET profile = :p, primary_email = :pe, sequence_number = :sn, user_uuid = :u, primary_username = :pn,"
"active = :a, flat_profile = :fp",
"TableName": self.table.name,
"ReturnValuesOnConditionCheckFailure": "NONE",
}
}
transact_items.append(transact_item)
logger.debug("Attempting to update batch of transactions for: {}".format(transact_items))
return self._run_transaction(transact_items)
def test_profile_object_returns_none_for_a_non_existant_user(self):
from cis_processor.profile import ProfileDelegate
new_user_id = "ad|Mozilla-LDAP-Dev|newzilla"
new_user_stub = {
"user_id": {
"value": new_user_id
},
"primary_email": {
"value": "*****@*****.**"
},
"primary_username": {
"value": "newzillian123"
},
"uuid": {
"value": str(uuid.uuid4())
},
}
kinesis_event = kinesis_event_generate(user_profile=new_user_stub)
profile_delegate = ProfileDelegate(kinesis_event, self.dynamodb_client,
self.table)
result = profile_delegate.load_old_user_profile()
# Check returned profile is empty
assert result.as_dict() == User().as_dict()
def _update_with_transaction(self, user_profile):
cis_profile_user_object = User(
user_structure_json=json.loads(user_profile["profile"]))
transact_items = {
"Update": {
"Key": {
"id": {
"S": user_profile["id"]
}
},
"ExpressionAttributeValues": {
":p": {
"S": user_profile["profile"]
},
":u": {
"S": user_profile["user_uuid"]
},
":pe": {
"S": user_profile["primary_email"]
},
":pn": {
"S": user_profile["primary_username"]
},
":sn": {
"S": user_profile["sequence_number"]
},
":a": {
"BOOL":
json.loads(user_profile["profile"])["active"]["value"]
},
":fp": {
"M": cis_profile_user_object.as_dynamo_flat_dict()
},
},
"ConditionExpression":
"attribute_exists(id)",
"UpdateExpression":
"SET profile = :p, primary_email = :pe, sequence_number = :sn, user_uuid = :u,"
"primary_username = :pn, active = :a, flat_profile = :fp",
"TableName":
self.table.name,
"ReturnValuesOnConditionCheckFailure":
"NONE",
}
}
return self._run_transaction([transact_items])
def _create_with_transaction(self, user_profile):
if user_profile["sequence_number"] is None:
user_profile["sequence_number"] = str(uuid.uuid4().int)
cis_profile_user_object = User(
user_structure_json=json.loads(user_profile["profile"]))
transact_items = {
"Put": {
"Item": {
"id": {
"S": user_profile["id"]
},
"user_uuid": {
"S": user_profile["user_uuid"]
},
"profile": {
"S": user_profile["profile"]
},
"primary_email": {
"S": user_profile["primary_email"]
},
"primary_username": {
"S": user_profile["primary_username"]
},
"sequence_number": {
"S": user_profile["sequence_number"]
},
"active": {
"BOOL":
json.loads(user_profile["profile"])["active"]["value"]
},
"flat_profile": {
"M": cis_profile_user_object.as_dynamo_flat_dict()
},
},
"ConditionExpression": "attribute_not_exists(id)",
"TableName": self.table.name,
"ReturnValuesOnConditionCheckFailure": "NONE",
}
}
return self._run_transaction([transact_items])
def test_publish(self, mock_authzero, mock_secrets, mock_request_get,
mock_request_post):
mock_authzero.return_value = "dinopark"
mock_secrets.return_value = "is_pretty_cool"
mock_request_post.return_value = FakePostResponse()
mock_request_get.return_value = FakeGetResponse(
User(user_id="email|123").as_json())
mozilliansorg_group_publisher = cis_publisher.MozilliansorgGroupsPublisher(
)
mozilliansorg_group_publisher.publish(EVENT)
def test_post_profiles_and_update_it_and_retrieving_status_it_should_succeed(
self, fake_jwks):
os.environ["CIS_ENVIRONMENT"] = "local"
os.environ["CIS_CONFIG_INI"] = "tests/mozilla-cis.ini"
os.environ["AWS_XRAY_SDK_ENABLED"] = "false"
os.environ["CIS_DYNALITE_PORT"] = self.dynalite_port
from cis_change_service import api
# Post a new user
f = FakeBearer()
fake_jwks.return_value = json_form_of_pk
token = f.generate_bearer_without_scope()
api.app.testing = True
self.app = api.app.test_client()
my_fake_user = User(user_id="userA")
my_fake_user.active.value = True
my_fake_user.primary_email.value = "*****@*****.**"
my_fake_user.uuid.value = None
my_fake_user.primary_username.value = None
result = self.app.post(
"/v2/user?user_id={}".format(my_fake_user.user_id.value),
headers={"Authorization": "Bearer " + token},
json=my_fake_user.as_dict(),
content_type="application/json",
follow_redirects=True,
)
results = json.loads(result.get_data())
# Post it again
result = self.app.post(
"/v2/user?user_id={}".format(my_fake_user.user_id.value),
headers={"Authorization": "Bearer " + token},
json=my_fake_user.as_dict(),
content_type="application/json",
follow_redirects=True,
)
results = json.loads(result.get_data())
assert results is not None
assert results.get("status_code") == 202 or results.get(
"status_code") == 200
def _put_items_without_transaction(self, list_of_profiles):
sequence_numbers = []
with self.table.batch_writer() as batch:
for profile in list_of_profiles:
cis_profile_user_object = User(user_structure_json=json.loads(profile["profile"]))
batch.put_item(
Item={
"id": profile["id"],
"user_uuid": profile["user_uuid"],
"profile": profile["profile"],
"primary_email": profile["primary_email"],
"primary_username": profile["primary_username"],
"sequence_number": profile["sequence_number"],
"active": bool(json.loads(profile["profile"])["active"]["value"]),
"flat_profile": {
k: self.deserializer.deserialize(v)
for k, v in cis_profile_user_object.as_dynamo_flat_dict().items()
},
}
)
sequence_numbers.append(profile["sequence_number"])
return {"status": "200", "ResponseMetadata": {"HTTPStatusCode": 200}, "sequence_numbers": sequence_numbers}
def get_cis_user(self, user_id):
"""
Call CIS Person API and return the matching user profile
@user_id str a user_id
"""
self.__deferred_init()
logger.info("Requesting CIS Person API for a user profile {}".format(user_id))
access_token = self._get_authzero_token()
qs = "/v2/user/user_id/{}".format(quote_plus(user_id))
response = self._request_get(
self.api_url_person, qs, headers={"authorization": "Bearer {}".format(access_token)}
)
if not response.ok:
logger.error(
"Failed to query CIS Person API: {}{} response: {}".format(self.api_url_person, qs, response.text)
)
raise PublisherError("Failed to query CIS Person API", response.text)
return User(response.json())
def test_prepare_update(self, mock_authzero, mock_secrets,
mock_request_get, mock_request_post):
mock_authzero.return_value = "dinopark"
mock_secrets.return_value = "is_pretty_cool"
mock_request_post.return_value = FakePostResponse()
mock_request_get.return_value = FakeGetResponse(
User(user_id="email|123").as_json())
update = cis_publisher.MozilliansorgGroupUpdate.from_record(
EVENT["Records"][0])
mozilliansorg_group_publisher = cis_publisher.MozilliansorgGroupsPublisher(
)
update_profile = mozilliansorg_group_publisher._prepare_update(update)
assert update_profile.user_id.value == update.user_id
assert update_profile.access_information.mozilliansorg.metadata.display == "ndaed"
assert len(
update_profile.access_information.mozilliansorg["values"]) == 3
def filter_full_profiles(scopes, filter_display, vault_profiles):
v2_profiles = []
for profile in vault_profiles:
if isinstance(profile.get("profile"), str):
vault_profile = json.loads(profile.get("profile"))
else:
vault_profile = profile.get("profile")
v2_profile = User(user_structure_json=vault_profile)
if "read:fullprofile" in scopes:
# Assume someone has asked for all the data.
logger.debug("The provided token has access to all of the data.",
extra={"scopes": scopes})
pass
else:
# Assume the we are filtering falls back to public with no scopes
logger.debug("This is a limited scoped query.",
extra={"scopes": scopes})
v2_profile.filter_scopes(
scope_to_mozilla_data_classification(scopes))
if "display:all" in scopes:
logger.debug("display:all in token not filtering profile.",
extra={"scopes": scopes})
else:
logger.debug("display filtering engaged for query.",
extra={"scopes": scopes})
v2_profile.filter_display(scope_to_display_level(scopes))
if filter_display is not None:
v2_profile.filter_display(DisplayLevelParms.map(filter_display))
v2_profiles.append(
dict(id=v2_profile.user_id, profile=v2_profile.as_dict()))
return v2_profiles
def filter_known_cis_users(self, profiles=None, save=True):
"""
Filters out fields that are not allowed to be updated by this publisher from the profile before posting
This is for "new" users
"""
self.__deferred_init()
self.get_known_cis_users()
if profiles is None:
profiles = self.profiles
# Never NULL/None these fields during filtering as they're used for knowing where to post
whitelist = ["user_id", "active"]
null_user = User()
allowed_updates = self.publisher_rules["update"]
allowed_creates = self.publisher_rules["create"]
for n in range(0, len(profiles)):
p = profiles[n]
if p.user_id.value is None:
user_id = self.known_cis_users_by_email[p.primary_email.value]
else:
user_id = p.user_id.value
if user_id in self.known_cis_users_by_user_id:
logger.debug(
"Filtering out non-updatable values from user {} because it already exist in CIS".format(user_id)
)
for pfield in p.__dict__:
# Skip? (see below for sub item)
if pfield in whitelist:
continue
if pfield not in allowed_updates:
continue
# sub-item?
elif pfield in ["identities", "staff_information", "access_information"]:
for subpfield in p.__dict__[pfield]:
# Skip?
if subpfield in whitelist:
continue
# XXX access_information.{hris,ldap, ...} - this needs refactor
exit_loop = False
if isinstance(allowed_updates[pfield], dict):
for sub_au in allowed_updates[pfield]:
if (
p.__dict__[pfield][subpfield]["signature"]["publisher"]["name"]
== self.publisher_name
):
exit_loop = True
break
if exit_loop:
continue
if allowed_updates[pfield] != self.publisher_name:
p.__dict__[pfield][subpfield]["signature"]["publisher"]["value"] = ""
if "value" in p.__dict__[pfield][subpfield].keys():
p.__dict__[pfield][subpfield]["value"] = None
elif "values" in p.__dict__[pfield][subpfield].keys():
p.__dict__[pfield][subpfield]["values"] = None
else:
if allowed_updates[pfield] != self.publisher_name:
p.__dict__[pfield]["signature"]["publisher"]["value"] = ""
if "value" in p.__dict__[pfield].keys():
p.__dict__[pfield]["value"] = None
elif "values" in p.__dict__[pfield].keys():
p.__dict__[pfield]["values"] = None
else:
# User is not yet in CIS, its a new user
logger.debug(f"Filtering out None/null fields from creation since these aren't needed for {user_id}")
for pfield in p.__dict__:
if pfield in whitelist:
continue
if pfield not in allowed_creates:
continue
# XXX filter more sub-fields on create? ["staff_information", "access_information"]
# Refactor me to be recursive (just like above code)
if pfield == "identities":
for subpfield in p.__dict__[pfield]:
f = p.__dict__[pfield][subpfield]
if "value" in f.keys() and f["value"] is None:
p.__dict__[pfield][subpfield] = null_user.__dict__[pfield][subpfield] # reset
elif "values" in f.keys() and (f["values"] is None or len(f["values"]) == 0):
p.__dict__[pfield][subpfield] = null_user.__dict__[pfield][subpfield] # reset
else:
f = p.__dict__[pfield]
if "value" in f.keys() and f["value"] is None:
p.__dict__[pfield] = null_user.__dict__[pfield] # reset
elif "values" in f.keys() and (f["values"] is None or len(f["values"]) == 0):
p.__dict__[pfield] = null_user.__dict__[pfield] # reset
logger.debug("Filtered fields for user {}".format(user_id))
profiles[n] = p
if save:
self.profiles = profiles
return profiles
def __init__(self, ret=User().as_json(), ok=True):
self.ret = ret
self.ok = ok
self.text = ""
