def parse(self):
self._resource = Resource()
self._resource.datasource_id = self._datasource_id
resource_id = self._payload['id']
self._resource.resource_id = resource_id
self._resource.resource_name = self._payload['name']
self._resource.resource_type = gutils.get_file_type_from_mimetype(
self._payload['mimeType'])
self._resource.resource_owner_id = self._payload['owners'][0].get(
'emailAddress')
self._resource.resource_size = self._payload.get('size')
self._resource.creation_time = self._payload['createdTime'][:-1]
self._resource.last_modified_time = self._payload['modifiedTime'][:-1]
self._resource.web_content_link = self._payload.get("webContentLink")
self._resource.web_view_link = self._payload.get("webViewLink")
self._resource.icon_link = self._payload.get("iconLink")
self._resource.thumthumbnail_link = self._payload.get("thumbnailLink")
self._resource.description = self._payload.get("description")
self._resource.parent_id = self._payload.get(
'parents')[0] if self._payload.get('parents') else None
self._resource.last_modifying_user_email = ""
if self._payload.get("lastModifyingUser"):
self._resource.last_modifying_user_email = self._payload[
"lastModifyingUser"].get("emailAddress")
#1. Set resource exposure type based on highest exposure from all permissions
#2. Collect all external users
resource_exposure_type = constants.EntityExposureType.PRIVATE.value
external_user_map = {}
permissions_payload = self._payload.get('permissions')
self._resource.permissions = []
if permissions_payload:
for payload in permissions_payload:
permission = GsuitePermission(self._datasource_id,
self._resource.resource_id,
self._resource.resource_owner_id,
payload)
permission_model = permission.get_model()
if not permission_model:
continue
self._resource.permissions.append(permission_model)
resource_exposure_type = utils.get_highest_exposure_type(
permission_model.exposure_type, resource_exposure_type)
if (not permission_model.email in external_user_map
) and permission.get_external_user():
external_user_map[permission_model.
email] = permission.get_external_user()
self._resource.exposure_type = resource_exposure_type
self._external_users = external_user_map.values()
def parse(self):
self._permission = ResourcePermission()
self._permission.datasource_id = self._datasource_id
self._permission.resource_id = self._resource_id
user_info = self._user_info_map[
self.
_shared_id] if self._shared_id in self._user_info_map else None
self._permission.email = user_info.email if user_info else self._shared_id
self._permission.permission_id = self._shared_id
self._permission.permission_type = constants.Role.WRITER.value if self._is_editable else constants.Role.READER.value
user_member_type = user_info.member_type if user_info else None
exposure_type = utils.get_highest_exposure_type(
self._permission_exposure_type, user_member_type)
self._permission.exposure_type = exposure_type
self.set_permission_exposure_is_external(exposure_type)
def delete_resource_permission(initiated_by_email, datasource_id, updated_permissions):
db_session = db_connection().get_session()
external_users = {}
for resource_id in updated_permissions:
deleted_permissions = updated_permissions[resource_id]
for perm in deleted_permissions:
if perm["exposure_type"] == constants.EntityExposureType.EXTERNAL.value and not perm['email'] in external_users:
external_users[perm['email']] = 1
db_session.query(ResourcePermission).filter(and_(ResourcePermission.datasource_id == datasource_id,
ResourcePermission.email == perm['email'],
ResourcePermission.resource_id == resource_id)).delete()
db_connection().commit()
updated_resource = db_session.query(Resource).filter(and_(Resource.datasource_id == datasource_id,
Resource.resource_id == resource_id)).first()
highest_exposure = constants.EntityExposureType.PRIVATE.value
if updated_resource:
for resource_perm in updated_resource.permissions:
highest_exposure = utils.get_highest_exposure_type(resource_perm.exposure_type, highest_exposure)
# Update the resource with highest exposure
if not updated_resource.exposure_type == highest_exposure:
updated_resource.exposure_type = highest_exposure
updated_resource.last_modifying_user_email = initiated_by_email
updated_resource.last_modified_time = datetime.datetime.utcnow()
db_connection().commit()
anything_changed = False
for external_user in external_users:
permissions_count = db_session.query(ResourcePermission).filter(and_(ResourcePermission.datasource_id ==
datasource_id,
ResourcePermission.email == external_user)).count()
if permissions_count < 1:
db_session.query(DomainUser).filter(
and_(DomainUser.email == external_user, DomainUser.datasource_id == datasource_id,
DomainUser.member_type == constants.EntityExposureType.EXTERNAL.value)).delete()
anything_changed = True
if anything_changed:
db_connection().commit()
def add_new_permission_to_db(updated_permission, resource_id, datasource_id, initiated_by_email, role, domain_id):
# If the user does not exist in DomainUser table add now
db_session = db_connection().get_session()
existing_user = db_session.query(DomainUser).filter(
and_(DomainUser.datasource_id == datasource_id,
DomainUser.email == updated_permission['email'])).first()
if not existing_user:
# Update the exposure type of the permission
new_user_email = updated_permission['email']
exposure_type = utils.check_if_external_user(db_session, domain_id, new_user_email)
updated_permission['exposure_type'] = exposure_type
domainUser = DomainUser()
domainUser.datasource_id = datasource_id
domainUser.email = new_user_email
domainUser.member_type = exposure_type
display_name = updated_permission['displayName']
name = display_name.split(' ')
if len(name) > 0 and name[0]:
domainUser.first_name = name[0]
if len(name) > 1:
domainUser.last_name = name[1]
else:
domainUser.first_name = domainUser.email
domainUser.last_name = ""
db_session.add(domainUser)
db_connection().commit()
else:
# case: add permission to external user if that user already exist , than exposure type of permission should also be external
updated_permission['exposure_type'] = existing_user.member_type
if role == 'owner':
updated_permission['exposure_type'] = constants.EntityExposureType.PRIVATE.value
permission = ResourcePermission()
permission.datasource_id = datasource_id
permission.resource_id = resource_id
permission.email = updated_permission['email']
permission.permission_type = updated_permission['permission_type']
permission.permission_id = updated_permission['permission_id']
permission.exposure_type = updated_permission['exposure_type']
db_session.add(permission)
# Update the exposure type of the resource based on the updated permission
existing_resource = db_session.query(Resource).filter(and_(Resource.resource_id == resource_id,
Resource.datasource_id == datasource_id)).first()
existing_resource.exposure_type = utils.get_highest_exposure_type(permission.exposure_type, existing_resource.exposure_type)
# if permission.exposure_type == constants.EntityExposureType.EXTERNAL.value:
# if not (existing_resource.exposure_type == constants.EntityExposureType.EXTERNAL.value and
# existing_resource.exposure_type == constants.EntityExposureType.PUBLIC.value and
# existing_resource.exposure_type == constants.EntityExposureType.ANYONEWITHLINK.value):
# existing_resource.exposure_type = constants.EntityExposureType.EXTERNAL.value
#
# elif permission.exposure_type == constants.EntityExposureType.TRUSTED.value:
# if not(existing_resource.exposure_type == constants.EntityExposureType.EXTERNAL.value and
# existing_resource.exposure_type == constants.EntityExposureType.PUBLIC.value and
# existing_resource.exposure_type == constants.EntityExposureType.ANYONEWITHLINK.value and
# existing_resource.exposure_type == constants.EntityExposureType.TRUSTED.value):
# existing_resource.exposure_type = constants.EntityExposureType.TRUSTED.value
#
# else:
# if existing_resource.exposure_type == constants.EntityExposureType.PRIVATE.value:
# existing_resource.exposure_type = constants.EntityExposureType.INTERNAL.value
existing_resource.last_modifying_user_email = initiated_by_email
existing_resource.last_modified_time = datetime.datetime.utcnow()
if role == constants.Role.OWNER.value:
existing_resource.resource_owner_id = updated_permission['email']
update_old_owner_permission(db_session, datasource_id, resource_id, updated_permission['email'])
db_connection().commit()
return permission
def parse(self):
self._file = Resource()
self._file.datasource_id = self._datasource_id
self._file.resource_id = self._payload['id']
self._file.resource_name = self._payload['name']
self._file.resource_type = self._payload['filetype']
self._file.resource_size = self._payload['size']
self._file.resource_owner_id = self._payload['resource_owner_email']
self._file.creation_time = datetime.datetime.fromtimestamp(
self._payload['timestamp'])
self._file.last_modified_time = datetime.datetime.fromtimestamp(
self._payload['timestamp'])
self._file.web_content_link = self._payload[
'url_private_download'] if 'url_private_download' in self._payload else None
self._file.web_view_link = self._payload[
'url_private'] if 'url_private' in self._payload else None
file_exposure_type = constants.EntityExposureType.ANYONEWITHLINK.value if self._payload['public_url_shared'] \
else (constants.EntityExposureType.DOMAIN.value if self._payload['is_public'] else
constants.EntityExposureType.PRIVATE.value)
is_editable = self._payload["editable"]
db_session = db_connection().get_session()
# construct user_info map for checking for removed users from channels
existing_users = db_session.query(DomainUser).filter(
DomainUser.datasource_id == self._datasource_id).all()
user_info_map = {}
for user in existing_users:
user_info_map[user.user_id] = user
self._file.permissions = []
permission_exposure = constants.EntityExposureType.PRIVATE.value
shared_in_channels = self._payload['channels']
if shared_in_channels:
permission_exposure_type = constants.EntityExposureType.DOMAIN.value
for channel_id in shared_in_channels:
permission = SlackPermission(self._datasource_id,
user_info_map, is_editable,
channel_id,
permission_exposure_type,
self._payload)
permission_model = permission.get_model()
if not permission_model:
continue
self._file.permissions.append(permission_model)
permission_exposure = constants.EntityExposureType.EXTERNAL.value if \
permission.get_permission_exposure_is_external() else permission_exposure_type
shared_in_private_group = self._payload['groups']
if shared_in_private_group:
permission_exposure_type = constants.EntityExposureType.INTERNAL.value
for group_id in shared_in_private_group:
permission = SlackPermission(self._datasource_id,
user_info_map, is_editable,
group_id,
permission_exposure_type,
self._payload)
permission_model = permission.get_model()
if not permission_model:
continue
self._file.permissions.append(permission_model)
permission_exposure = constants.EntityExposureType.EXTERNAL.value if \
permission.get_permission_exposure_is_external() else permission_exposure_type
if file_exposure_type == constants.EntityExposureType.ANYONEWITHLINK.value:
permission = SlackPermission(self._datasource_id, user_info_map,
is_editable, file_exposure_type,
file_exposure_type, self._payload)
permission_model = permission.get_model()
if permission_model:
self._file.permissions.append(permission_model)
resource_exposure_type = utils.get_highest_exposure_type(
permission_exposure, file_exposure_type)
self._file.exposure_type = resource_exposure_type
def process(db_session, auth_token, query_params, scanner_data):
#start_time = datetime.datetime.utcnow()
#Logger().info("File processing started at - {}".format(start_time))
domain_id = query_params["domainId"]
datasource_id = query_params["dataSourceId"]
user_email = query_params["userEmail"]
trusted_domains = (utils.get_trusted_entity_for_domain(
db_session, domain_id))['trusted_domains']
resource_count = 0
try:
Logger().info(
"Initiating processing of drive resources for files using email: {}"
.format(user_email))
resources = scanner_data["entities"]
resourceList = []
db_session = db_connection().get_session()
data_for_permission_table = []
external_user_map = {}
for resourcedata in resources:
resource_count = resource_count + 1
resource = {}
resource["datasource_id"] = datasource_id
resource_id = resourcedata['id']
resource["resource_id"] = resource_id
resource["resource_name"] = resourcedata['name']
mime_type = gutils.get_file_type_from_mimetype(
resourcedata['mimeType'])
resource["resource_type"] = mime_type
resource["resource_owner_id"] = resourcedata['owners'][0].get(
'emailAddress')
resource["resource_size"] = resourcedata.get('size')
resource["creation_time"] = resourcedata['createdTime'][:-1]
resource["last_modified_time"] = resourcedata['modifiedTime'][:-1]
resource["web_content_link"] = resourcedata.get("webContentLink")
resource["web_view_link"] = resourcedata.get("webViewLink")
resource["icon_link"] = resourcedata.get("iconLink")
resource["thumthumbnail_link"] = resourcedata.get("thumbnailLink")
resource["description"] = resourcedata.get("description")
resource["last_modifying_user_email"] = ""
if resourcedata.get("lastModifyingUser"):
resource["last_modifying_user_email"] = resourcedata[
"lastModifyingUser"].get("emailAddress")
resource_exposure_type = constants.EntityExposureType.PRIVATE.value
resource_permissions = resourcedata.get('permissions')
if resource_permissions:
for permission in resource_permissions:
permission_id = permission.get('id')
email_address = permission.get('emailAddress')
display_name = permission.get('displayName')
expiration_time = permission.get('expirationTime')
is_deleted = permission.get('deleted')
if is_deleted:
continue
if email_address:
if email_address == resource["resource_owner_id"]:
permission_exposure = constants.EntityExposureType.PRIVATE.value
elif email_address in external_user_map:
permission_exposure = external_user_map[
email_address]["member_type"]
else:
permission_exposure = utils.check_if_external_user(
db_session, domain_id, email_address,
trusted_domains)
if permission_exposure == constants.EntityExposureType.EXTERNAL.value or permission_exposure == constants.EntityExposureType.TRUSTED.value:
## insert non domain user as External user in db, Domain users will be
## inserted during processing Users
if not email_address in external_user_map:
externaluser = {}
externaluser["datasource_id"] = datasource_id
externaluser["email"] = email_address
externaluser["first_name"] = ""
externaluser["last_name"] = ""
if display_name and display_name != "":
name_list = display_name.split(' ')
externaluser["first_name"] = name_list[0]
if len(name_list) > 1:
externaluser["last_name"] = name_list[
1]
externaluser[
"member_type"] = permission_exposure
external_user_map[email_address] = externaluser
#Shared with everyone in domain
elif display_name:
email_address = "__ANYONE__@" + display_name
permission_exposure = constants.EntityExposureType.DOMAIN.value
# Shared with anyone with link
elif permission_id == 'anyoneWithLink':
email_address = constants.EntityExposureType.ANYONEWITHLINK.value
permission_exposure = constants.EntityExposureType.ANYONEWITHLINK.value
#Shared with everyone in public
else:
email_address = constants.EntityExposureType.PUBLIC.value
permission_exposure = constants.EntityExposureType.PUBLIC.value
resource_permission = {}
resource_permission["datasource_id"] = datasource_id
resource_permission["resource_id"] = resource_id
resource_permission["email"] = email_address
resource_permission["permission_id"] = permission_id
resource_permission["permission_type"] = permission['role']
resource_permission["exposure_type"] = permission_exposure
if expiration_time:
resource_permission[
"expiration_time"] = expiration_time[:-1]
resource_permission["is_deleted"] = is_deleted
data_for_permission_table.append(resource_permission)
resource_exposure_type = utils.get_highest_exposure_type(
permission_exposure, resource_exposure_type)
resource["exposure_type"] = resource_exposure_type
resource["parent_id"] = resourcedata.get(
'parents')[0] if resourcedata.get('parents') else None
resourceList.append(resource)
db_session.bulk_insert_mappings(Resource, resourceList)
db_session.bulk_insert_mappings(ResourcePermission,
data_for_permission_table)
#Logger().info("File processing - collected the data in - {}".format(datetime.datetime.utcnow() - start_time))
#db_session.execute(Resource.__table__.insert().prefix_with("IGNORE").values(resourceList))
#Logger().info("File processing - inserted in resource table in - {}".format(datetime.datetime.utcnow() - start_time))
#db_session.execute(ResourcePermission.__table__.insert().prefix_with("IGNORE").values(data_for_permission_table))
#Logger().info("File processing - inserted in permissions table in - {}".format(datetime.datetime.utcnow() - start_time))
if len(external_user_map) > 0:
db_session.execute(
DomainUser.__table__.insert().prefix_with("IGNORE").values(
external_user_map.values()))
db_connection().commit()
#Logger().info("File processing - committed everything in - {}".format(datetime.datetime.utcnow() - start_time))
return resource_count
except Exception as ex:
Logger().exception(
"Exception occurred while processing data for drive resources using email: {} - {}"
.format(user_email, ex))
db_session.rollback()
return 0
def process(db_session, auth_token, query_params, scanner_data):
collaborators_list = scanner_data["entities"]
datasource_id = query_params["dataSourceId"]
domain_id = query_params["domainId"]
repo_id = query_params["repo_id"]
processed_collaborators_count = 0
max_repository_exposure = constants.EntityExposureType.PRIVATE.value
new_collaborator_list = []
new_permission_list = []
for collaborator in collaborators_list:
collaborator_info = {}
collaborator_info["datasource_id"] = datasource_id
collaborator_info["email"] = collaborator["email"] if collaborator[
"email"] else github_utils.get_default_github_email(
collaborator["id"], collaborator["login"])
collaborator_info["full_name"] = collaborator["name"] if collaborator[
"name"] else collaborator["login"]
name_split = collaborator_info["full_name"].split(" ")
if len(name_split) > 1:
collaborator_info["first_name"] = name_split[0]
collaborator_info["last_name"] = name_split[1]
else:
collaborator_info["first_name"] = name_split[0]
collaborator_info["last_name"] = ''
collaborator_info["creation_time"] = datetime.strptime(
collaborator["created_at"], "%Y-%m-%dT%H:%M:%SZ")
collaborator_info["last_updated"] = datetime.strptime(
collaborator["updated_at"], "%Y-%m-%dT%H:%M:%SZ")
collaborator_info["photo_url"] = collaborator["avatar_url"]
collaborator_info["user_id"] = collaborator["id"]
collaborator_info[
"member_type"] = constants.EntityExposureType.INTERNAL.value
if github_utils.is_external_user(domain_id,
collaborator_info["email"]):
collaborator_info[
"member_type"] = constants.EntityExposureType.EXTERNAL.value
#Update the Resource permissions table for the current repository
repo_permission_dict = {}
repo_permission_dict["datasource_id"] = datasource_id
repo_permission_dict["resource_id"] = repo_id
repo_permission_dict["email"] = collaborator_info["email"]
repo_permission_dict["permission_id"] = collaborator["id"]
repo_permission_dict["exposure_type"] = collaborator_info[
"member_type"]
repo_permission_dict["permission_type"] = collaborator[
"permission_type"]
max_repository_exposure = utils.get_highest_exposure_type(
collaborator_info["member_type"], max_repository_exposure)
new_collaborator_list.append(collaborator_info)
new_permission_list.append(repo_permission_dict)
processed_collaborators_count += 1
try:
db_session.execute(DomainUser.__table__.insert().prefix_with(
"IGNORE").values(new_collaborator_list))
db_session.execute(ResourcePermission.__table__.insert().prefix_with(
"IGNORE").values(new_permission_list))
db_session.query(Resource).filter(Resource.datasource_id == datasource_id, Resource.resource_id == repo_id). \
update({ Resource.exposure_type: max_repository_exposure })
db_connection().commit()
except Exception as ex:
Logger().exception(
"Exception occurred while adding respository collaborators with exception - {}"
.format(ex))
db_session.rollback()
return processed_collaborators_count
def validate_permission_change_policy(db_session, auth_token, datasource_obj,
policy, resource, new_permissions):
datasource_id = datasource_obj.datasource_id
Logger().info(
"validating_policy : resource : {} , new permission : {} ".format(
resource, new_permissions))
is_policy_violated = False
violated_permissions = []
new_permissions_left = []
highest_exposure_type = constants.EntityExposureType.PRIVATE.value
for permission in new_permissions:
is_permission_violated = 1
for policy_condition in policy.conditions:
if policy_condition.match_type == constants.PolicyMatchType.DOCUMENT_NAME.value:
is_permission_violated = is_permission_violated & check_value_violation(
policy_condition, resource["resource_name"])
elif policy_condition.match_type == constants.PolicyMatchType.DOCUMENT_OWNER.value:
is_permission_violated = is_permission_violated & check_value_violation(
policy_condition, resource["resource_owner_id"])
elif policy_condition.match_type == constants.PolicyMatchType.DOCUMENT_EXPOSURE.value:
is_permission_violated = is_permission_violated & check_value_violation(
policy_condition, permission["exposure_type"])
elif policy_condition.match_type == constants.PolicyMatchType.PERMISSION_EMAIL.value:
is_permission_violated = is_permission_violated & check_value_violation(
policy_condition, permission["email"])
if is_permission_violated:
is_policy_violated = True
if not permission["permission_type"] == constants.Role.OWNER.value:
violated_permissions.append(permission)
highest_exposure_type = get_highest_exposure_type(
permission["exposure_type"], highest_exposure_type)
else:
new_permissions_left.append(permission)
else:
new_permissions_left.append(permission)
send_email_action = []
check_if_revert_action = False
if is_policy_violated:
if highest_exposure_type in constants.permission_exposure_to_event_constants:
tags = {
"resource_id": resource["resource_id"],
"resource_name": resource["resource_name"],
"new_permissions": violated_permissions
}
activity_db().add_event(
domain_id=datasource_obj.domain_id,
connector_type=datasource_obj.datasource_type,
event_type=constants.
permission_exposure_to_event_constants[highest_exposure_type],
actor=resource["resource_owner_id"],
tags=tags)
Logger().info(
"Policy \"{}\" is violated, so triggering corresponding actions".
format(policy.name))
for action in policy.actions:
if action.action_type == constants.PolicyActionType.SEND_EMAIL.value:
send_email_action.append(action)
elif action.action_type == constants.PolicyActionType.REVERT.value and len(
violated_permissions) > 0:
Logger().info(
"violated permissions : {}".format(violated_permissions))
check_if_revert_action = True
datasource_type = datasource_obj.datasource_type
body = json.dumps(violated_permissions, cls=alchemy_encoder())
payload = {
"permissions":
body,
"datasource_id":
datasource_id,
"domain_id":
datasource_obj.domain_id,
"user_email":
resource["resource_owner_id"],
"action_type":
action_constants.ActionNames.
REMOVE_EXTERNAL_ACCESS_TO_RESOURCE.value
}
response = messaging.trigger_post_event(
datasource_execute_action_map[datasource_type], auth_token,
None, payload, connector_servicename_map[datasource_type],
constants.TriggerType.SYNC.value)
if response and not response.response_code == constants.SUCCESS_STATUS_CODE:
violated_permissions = []
if len(send_email_action) > 0:
to_address = json.loads(send_email_action[0].config)["to"]
Logger().info("validate_policy : send email")
if not check_if_revert_action:
violated_permissions = None
adya_emails.send_permission_change_policy_violate_email(
to_address, policy, resource, new_permissions,
violated_permissions, new_permissions_left)
payload = {}
payload["datasource_id"] = datasource_id
payload["name"] = policy.name
payload["policy_id"] = policy.policy_id
payload["severity"] = policy.severity
payload[
"description_template"] = "Permission changes on {{resource_owner_id}}'s document \"{{resource_name}}\" has violated policy \"{{policy_name}}\""
payload["payload"] = resource
messaging.trigger_post_event(urls.ALERTS_PATH, auth_token, None,
payload)
