def process_notifications(notification_type, datasource_id, channel_id, body):
if notification_type == "sync":
return
db_session = db_connection().get_session()
subscription = db_session.query(PushNotificationsSubscription).filter(
PushNotificationsSubscription.channel_id == channel_id).first()
if not subscription:
Logger().warn("Subscription does not exist for datasource_id: {} and channel_id: {}, hence ignoring the notification.".format(
datasource_id, channel_id))
return
activities = [body]
try:
# If notification type is adya, then that means its triggered either manually or scheduled sync
# So we need to fetch the events from last sync time
if notification_type == "adya":
reports_service = gutils.get_gdrive_reports_service(None, subscription.user_email, db_session)
results = reports_service.activities().list(userKey="all", applicationName=subscription.notification_type, startTime=subscription.page_token).execute()
if results and "items" in results:
activities = results["items"]
for activity in activities:
process_incoming_activity(datasource_id, activity)
db_session.refresh(subscription)
subscription.last_accessed = datetime.datetime.utcnow()
subscription.page_token = datetime.datetime.utcnow().isoformat("T") + "Z"
db_connection().commit()
except Exception as e:
Logger().exception("Exception occurred while processing activity notification for datasource_id: {} channel_id: {} - {}".format(datasource_id, channel_id, e))
def delete_cloudwatch_event(cloudwatch_event_name, function_name):
try:
session = boto3.Session()
cloudwatch_client = session.client('events')
Logger().info("delete_cloudwatch_event : ")
# remove all the targets from the rule
response = cloudwatch_client.remove_targets(Rule=cloudwatch_event_name,
Ids=[
function_name,
])
Logger().info("removed target : ")
if response and response['ResponseMetadata'][
'HTTPStatusCode'] == constants.SUCCESS_STATUS_CODE:
# after removing all the targets , now delete the rule
response = cloudwatch_client.delete_rule(
Name=cloudwatch_event_name)
Logger().info("removed rule : ")
except Exception as ex:
Logger().exception(
"Exception occurred while deleting the cloudwatch event - ")
return False
def trigger_post_event_with_headers(
endpoint,
auth_token,
query_params,
headers,
body,
service_name="core",
trigger_type=constants.TriggerType.ASYNC.value):
result = None
if constants.DEPLOYMENT_ENV == 'local':
session = FuturesSession()
endpoint = _add_query_params_to_url(endpoint, query_params)
Logger().info("Making a POST request on the following url - " +
str(endpoint))
response = utils.post_call_with_authorization_header(
session, endpoint, auth_token, body, headers)
if trigger_type == constants.TriggerType.SYNC.value:
api_response = response.result()
result = ResponseMessage(api_response.status_code, None,
json.loads(api_response.content))
else:
body = _add_query_params_to_body(body, query_params)
body = _add_headers_to_body(body, headers)
endpoint = service_name + "-" + \
constants.DEPLOYMENT_ENV + "-post-" + slugify(endpoint)
Logger().info(
"Making a POST lambda invoke on the following function - " +
str(endpoint))
result = aws_utils.invoke_lambda(endpoint, auth_token, body,
trigger_type)
return result
def query(auth_token, query_params, scanner):
domain_id = query_params["domainId"]
next_page_token = query_params["nextPageNumber"]
directory_service = gutils.get_directory_service(auth_token)
users = []
try:
results = directory_service.users().list(customer='my_customer',
maxResults=20,
pageToken=next_page_token,
orderBy='email').execute()
except RefreshError as ex:
Logger().info("User query : Not able to refresh credentials")
results = gutils.create_user_payload_for_nonadmin_nonserviceaccount(
auth_token)
except HttpError as ex:
Logger().info("User query : Domain not found error")
results = gutils.create_user_payload_for_nonadmin_nonserviceaccount(
auth_token)
if results and "users" in results:
for user in results["users"]:
user_email = user.get("primaryEmail")
if user_email.endswith(domain_id):
users.append(user)
next_page_token = results.get('nextPageToken')
return {"payload": users, "nextPageNumber": next_page_token}
def delete_public_and_external_sharing_for_file(self):
for permission in self.permissions:
if permission[
'exposure_type'] == constants.EntityExposureType.ANYONEWITHLINK.value:
file_id = permission['resource_id']
try:
removed_file = self.slack_client.api_call(
"files.revokePublicURL", file=file_id)
self.response = removed_file
except Exception as ex:
Logger().exception(
"Exception ocuured while removing the permission - {}".
format(ex))
self.exception_messages.append(ex.message)
if not file_id in self.updated_permissions:
self.updated_permissions[file_id] = [permission]
else:
self.updated_permissions[file_id].append(permission)
try:
delete_resource_permission(self.initiated_by_email,
self.datasource_id,
self.updated_permissions)
except Exception as ex:
Logger().exception(
"Exception occurred while removing permission from db")
self.exception_messages.append(
"Exception occurred while removing permission from db")
return self.response
def process_login_activity(datasource_id, incoming_activity):
Logger().info('Processing login activity {}'.format(incoming_activity))
db_session = db_connection().get_session()
actor_email = incoming_activity['actor']['email']
events = incoming_activity["events"]
login_time = datetime.datetime.strptime(incoming_activity["id"]["time"], '%Y-%m-%dT%H:%M:%S.%fZ')
ninety_days_ago = datetime.datetime.utcnow() - datetime.timedelta(days=90)
is_inactive = login_time < ninety_days_ago
last_is_inactive = None
for event in events:
if event["name"] == 'login_success':
last_login_time = db_session.query(DomainUser).filter(DomainUser.datasource_id == datasource_id, DomainUser.email == actor_email).first().last_login_time
if last_login_time:
last_is_inactive = last_login_time < ninety_days_ago
if not last_login_time or (login_time > last_login_time):
db_session.query(DomainUser).filter(DomainUser.datasource_id == datasource_id, DomainUser.email == actor_email).update({"last_login_time":login_time}, synchronize_session = 'fetch')
datasource = db_session.query(DataSource).filter(DataSource.datasource_id == datasource_id).first()
app_name = constants.datasource_to_installed_app_map[datasource.datasource_type]
if (last_is_inactive is None) or not last_is_inactive :
if is_inactive:
db_session.query(Application).filter(and_(Application.domain_id == datasource.domain_id, Application.display_text == app_name)).update({"inactive_users":Application.inactive_users + 1}, synchronize_session = 'fetch')
Logger().info("new inactive user : {}".format(actor_email))
elif last_is_inactive and not is_inactive:
Logger().info("inactive user is now active : {}".format(actor_email))
db_session.query(Application).filter(and_(Application.domain_id == datasource.domain_id, Application.display_text == app_name, Application.inactive_users > 0)).update({"inactive_users":Application.inactive_users - 1}, synchronize_session = 'fetch')
db_connection().commit()
def process_user_related_activities(datasource_id, actor_email, event):
event_name = event['name']
activity_events_parameters = event['parameters']
user_email = None
user_obj = None
db_session = db_connection().get_session()
for param in activity_events_parameters:
name = param['name']
if name == 'USER_EMAIL':
user_email = param['value']
datasource_obj = get_datasource(datasource_id)
tags = {"user_email": user_email}
if event_name == 'CREATE_USER':
if user_email:
directory_service = gutils.get_directory_service(None, actor_email)
results = None
try:
results = directory_service.users().get(userKey=user_email).execute()
except RefreshError as ex:
Logger().info("User query : Not able to refresh credentials")
except HttpError as ex:
Logger().info("User query : Domain not found error")
if results:
gsuite_user = user.GsuiteUser(datasource_id, results)
user_obj = gsuite_user.get_model()
db_session.execute(DomainUser.__table__.insert().prefix_with("IGNORE").
values(db_utils.get_model_values(DomainUser, user_obj)))
call_validate_policies_for_admin_user(user_obj, datasource_id)
additional_payload = {"user_email": user_email, "is_admin": user_obj.is_admin}
tags["is_admin"] = user_obj.is_admin
elif event_name == 'GRANT_ADMIN_PRIVILEGE':
user_obj = db_session.query(DomainUser).filter(and_(DomainUser.datasource_id == datasource_id,
DomainUser.email == user_email)).first()
if user_obj:
user_obj.is_admin = True
call_validate_policies_for_admin_user(user_obj, datasource_id)
tags["is_admin"] = user_obj.is_admin
elif event_name == "SUSPEND_USER":
db_session.query(DomainUser).filter(and_(DomainUser.datasource_id == datasource_id,
DomainUser.email == user_email)).update({DomainUser.is_suspended: True})
elif event_name == "DELETE_USER":
delete_user_info(db_session, user_email, datasource_id)
activity_db().add_event(domain_id=datasource_obj.domain_id,
connector_type=constants.ConnectorTypes.GSUITE.value,
event_type=event_name, actor=actor_email, tags=tags)
db_connection().commit()
def initiate_action(auth_token, action_payload):
try:
action_key = action_payload['key']
initiated_by = action_payload['initiated_by']
action_parameters = action_payload['parameters']
datasource_id = action_payload[
'datasource_id'] if 'datasource_id' in action_payload else 'MANUAL'
db_session = db_connection().get_session()
login_user_info = db_session.query(LoginUser).filter(
LoginUser.auth_token == auth_token).first()
domain_id = login_user_info.domain_id
action_config = get_action(action_key)
if not action_config or not validate_action_parameters(
action_config, action_parameters):
return ResponseMessage(
400, "Failed to execute action - Validation failed")
log_id = action_payload[
'log_id'] if 'log_id' in action_payload else None
if log_id:
log_entry = db_session.query(AuditLog).filter(
and_(AuditLog.log_id == log_id)).first()
else:
log_entry = audit_action(domain_id, datasource_id, initiated_by,
action_config, action_parameters)
execution_status = execute_action(auth_token, domain_id, datasource_id,
action_config, action_payload,
log_entry)
db_connection().commit()
Logger().info("initiate_action : response body - {}".format(
execution_status.get_response_body()))
response_body = json.loads(
json.dumps(execution_status.get_response_body()))
response_body['id'] = log_entry.log_id
if execution_status.response_code == constants.ACCEPTED_STATUS_CODE:
action_payload['log_id'] = log_entry.log_id
messaging.trigger_post_event(urls.INITIATE_ACTION_PATH, auth_token,
None, action_payload)
return ResponseMessage(execution_status.response_code, None,
response_body)
except Exception as e:
Logger().exception(
"Exception occurred while initiating action using payload " +
str(action_payload))
return ResponseMessage(500, "Failed to execute action - {}".format(e))
def update_scan(auth_token, datasource_id, domain_id):
time.sleep(2)
db_session = db_connection().get_session()
scan_complete = db_session.query(DatasourceScanners).filter(and_(DatasourceScanners.datasource_id == datasource_id, DatasourceScanners.in_progress > 0)).count()
#print "Total scanners in progress - {}".format(scan_complete)
if scan_complete == 0:
scan_complete = db_session.query(DataSource).filter(and_(DataSource.datasource_id == datasource_id, DataSource.is_push_notifications_enabled == 0)). \
update({DataSource.is_push_notifications_enabled: 1})
Logger().info("update_scan: scan_complete - {}".format(scan_complete))
#print "Scan complete status - {}".format(scan_complete)
if scan_complete > 0:
Logger().info("update_scan : call Scan completed processing method")
scan_complete_processing(db_session, auth_token, datasource_id)
def query(auth_token, query_params, scanner):
datasource_id = query_params["dataSourceId"]
repo_name = query_params["repo_name"]
github_client = github_utils.get_github_client(datasource_id)
collaborators_list = []
#collaborator_permissions_dict = {}
repo = github_client.get_repo(repo_name)
if repo:
for collaborator in repo.get_collaborators():
collaborator.raw_data[
"permission_type"] = constants.Role.READER.value
if collaborator.permissions.raw_data:
if collaborator.permissions.raw_data["admin"]:
collaborator.raw_data[
"permission_type"] = constants.Role.OWNER.value
elif collaborator.permissions.raw_data["push"]:
collaborator.raw_data[
"permission_type"] = constants.Role.WRITER.value
collaborators_list.append(collaborator.raw_data)
#Creating a webhook for the repository
if not constants.DEPLOYMENT_ENV == "local":
try:
config = {
"url": urls.GITHUB_NOTIFICATIONS_URL,
"content_type": "json"
}
events = [
"repository", "repository_vulnerability_alert", "fork",
"member", "public", "push", "create"
]
repo.create_hook(name="web",
config=config,
events=events,
active=True)
except github.GithubException as ex:
if ex.status == 422:
Logger().info(
"Webhook already exist for repository - {} with exception - {}"
.format(repo_name, ex))
else:
Logger().exception(
"Github Exception occurred while subscribing for push notification for repository = {} with exception - {}"
.format(repo_name, ex))
except Exception as ex:
Logger().exception(
"Exception occurred while subscribing for push notification for repository = {} with exception - {}"
.format(repo_name, ex))
return {"payload": collaborators_list}
def async_delete_datasource(auth_token, datasource_id, complete_delete):
complete_delete = 0 if complete_delete == 0 else 1
db_session = db_connection().get_session()
existing_datasource = db_session.query(DataSource).filter(
DataSource.datasource_id == datasource_id).first()
try:
app_name = constants.datasource_to_installed_app_map[existing_datasource.datasource_type]
db_session.execute(DirectoryStructure.__table__.delete().where(DirectoryStructure.datasource_id == datasource_id))
db_session.execute(ResourcePermission.__table__.delete().where(ResourcePermission.datasource_id == datasource_id))
db_session.execute(Resource.__table__.delete().where(Resource.datasource_id == datasource_id))
app_query = db_session.query(ApplicationUserAssociation).filter(
ApplicationUserAssociation.datasource_id == datasource_id)
app_ids = [r.application_id for r in app_query.all()]
app_query.delete(synchronize_session=False)
try:
db_session.execute(Application.__table__.delete().where(Application.id.in_(app_ids)))
# delete app wrt to datasource
db_session.query(Application).filter(Application.display_text == app_name).delete(synchronize_session=False)
except:
Logger().info('App is present corresponding to other datasource')
db_session.execute(PushNotificationsSubscription.__table__.delete().where(PushNotificationsSubscription.datasource_id == datasource_id))
db_session.execute(DomainUser.__table__.delete().where(DomainUser.datasource_id == datasource_id))
db_session.execute(DatasourceScanners.__table__.delete().where(DatasourceScanners.datasource_id == datasource_id))
#If its not complete delete, then just clear out the scan entities, and reset the scan state
if complete_delete:
db_session.execute(AuditLog.__table__.delete().where(AuditLog.datasource_id == datasource_id))
db_session.execute(Alert.__table__.delete().where(Alert.datasource_id == datasource_id))
db_session.execute(PolicyAction.__table__.delete().where(PolicyAction.datasource_id == datasource_id))
db_session.execute(PolicyCondition.__table__.delete().where(PolicyCondition.datasource_id == datasource_id))
db_session.execute(Policy.__table__.delete().where(Policy.datasource_id == datasource_id))
db_session.execute(DatasourceCredentials.__table__.delete().where(DatasourceCredentials.datasource_id == datasource_id))
db_session.delete(existing_datasource)
else:
existing_datasource.processed_file_count = 0
existing_datasource.processed_group_count = 0
existing_datasource.processed_user_count = 0
existing_datasource.total_file_count = 0
existing_datasource.total_group_count = 0
existing_datasource.total_user_count = 0
existing_datasource.file_scan_status = 0
existing_datasource.group_scan_status = 0
existing_datasource.user_scan_status = 0
existing_datasource.is_push_notifications_enabled = 0
db_connection().commit()
Logger().info("Datasource deleted successfully")
except:
Logger().exception("Exception occurred during datasource data delete")
def process_incoming_activity(datasource_id, incoming_activity):
if not "id" in incoming_activity:
Logger().info("Incoming activity is not valid type - {}".format(incoming_activity))
return
Logger().info("Incoming activity - {}".format(incoming_activity))
app_name = incoming_activity["id"]["applicationName"]
if app_name == "token":
process_token_activity(datasource_id, incoming_activity)
elif app_name == "admin":
process_admin_activities(datasource_id, incoming_activity)
elif app_name == 'login':
process_login_activity(datasource_id, incoming_activity)
elif app_name == "drive":
process_drive_activity(datasource_id, incoming_activity)
def update_permissions(auth_token, permissions, owner_email, initiated_by_email, datasource_id):
drive_service = gutils.get_gdrive_service(auth_token, owner_email)
updated_permissions = {}
deleted_permissions = {}
is_success = True
exception_messages = ""
for permission in permissions:
resource_id = permission['resource_id']
permission_id = permission['permission_id']
role = permission['permission_type']
update_permission_object = {
"role": role,
}
try:
drive_service.permissions().update(fileId=resource_id, body=update_permission_object, quotaUser= owner_email[0:41],
permissionId=permission_id,
transferOwnership=True if role == 'owner' else False).execute()
if not permission['resource_id'] in updated_permissions:
updated_permissions[permission['resource_id']] = [permission]
else:
updated_permissions[permission['resource_id']].append(permission)
except HttpError as e:
if e.resp.status == 404:
Logger().info("Permission not found in gsuite, hence delete from db")
if not permission['resource_id'] in deleted_permissions:
deleted_permissions[permission['resource_id']] = [permission]
else:
deleted_permissions[permission['resource_id']].append(permission)
else:
Logger().exception("HttpError Exception occurred while updating permissions in gsuite ; {}".format(e))
is_success = False
if e.content:
content = json.loads(e.content)
exception_messages += content['error']['message'] if content['error']['message'] else "Exception occurred while updating permissions in gsuite"
except Exception as ex:
Logger().exception("Exception occurred while updating permissions in gsuite : {}".format(ex))
is_success = False
exception_messages += "Exception occurred while updating permissions in gsuite"
try:
update_resource_permissions(initiated_by_email, datasource_id, updated_permissions)
if len(deleted_permissions) > 0:
delete_resource_permission(initiated_by_email, datasource_id, deleted_permissions)
except Exception as ex:
Logger().exception("Exception occurred while updating permission from db : {}".format(ex))
is_success = False
exception_messages += "Exception occurred while updating permission from db"
return ResponseMessage(200 if is_success else 500, exception_messages)
def process_user(db_session, datasource, payload):
datasource_credentials = get_datasource_credentials(
db_session, datasource.datasource_id)
if datasource_credentials:
domain_id = datasource_credentials['domain_id']
userObj = entities.SlackUser(domain_id, datasource.datasource_id,
payload)
user_model_obj = userObj.get_model()
db_session.execute(
DomainUser.__table__.insert().prefix_with("IGNORE").values(
db_utils.get_model_values(DomainUser, user_model_obj)))
db_session.commit()
#check if new external member is added to a team
if user_model_obj.member_type == constants.EntityExposureType.EXTERNAL.value:
payload = {}
payload["user"] = json.dumps(user_model_obj, cls=alchemy_encoder())
policy_params = {
'dataSourceId': datasource.datasource_id,
'policy_trigger': constants.PolicyTriggerType.NEW_USER.value
}
Logger().info("new_user : payload : {}".format(payload))
messaging.trigger_post_event(urls.SLACK_POLICIES_VALIDATE_PATH,
constants.INTERNAL_SECRET,
policy_params, payload, "slack")
activity_db().add_event(
domain_id=datasource.domain_id,
connector_type=constants.ConnectorTypes.SLACK.value,
event_type='USER_ADDED',
actor=user_model_obj.email,
tags={
"exposure_type": user_model_obj.member_type,
"user_email": user_model_obj.email
})
def update_report(auth_token, payload):
if not auth_token:
return None
db_session = db_connection().get_session()
if payload:
report = {}
report["name"] = payload["name"]
if 'description' in payload:
report["description"] = payload["description"]
report["frequency"] = payload["frequency"]
report["receivers"] = payload["receivers"]
config_input = {
"report_type": payload["report_type"],
"selected_entity_type": payload["selected_entity_type"],
"selected_entity": payload["selected_entity"],
"datasource_id": payload["datasource_id"],
"selected_entity_name": payload["selected_entity_name"]
}
report["config"] = json.dumps(config_input)
report["is_active"] = payload["is_active"]
report_id = payload["report_id"]
db_session.query(Report).filter(
Report.report_id == report_id).update(report)
try:
db_connection().commit()
except Exception as ex:
Logger().exception()
return payload
else:
return None
def invoke_lambda(function_name,
auth_token,
body,
trigger_type=constants.TriggerType.ASYNC.value):
if not body:
body = {}
body['Authorization'] = auth_token
client = boto3.client('lambda')
response = client.invoke(
FunctionName=function_name,
InvocationType='Event' if trigger_type
== constants.TriggerType.ASYNC.value else 'RequestResponse',
LogType='None',
Payload=bytes(json.dumps(body)))
response_payload = {"statusCode": 200, "body": ""}
if trigger_type == constants.TriggerType.SYNC.value:
response_payload = json.loads(
response['Payload'].read().decode("utf-8"))
Logger().info("Response from sync lambda invocation is - {}".format(
response_payload))
if "errorMessage" in response_payload:
return ResponseMessage(500, response_payload['errorMessage'])
return ResponseMessage(response_payload['statusCode'], None,
response_payload['body'])
def send_email(user_list, email_subject, rendered_html):
try:
session = boto3.Session()
ses_client = session.client('ses')
ses_client.send_email(Source='*****@*****.**',
Destination={
'ToAddresses': user_list,
'BccAddresses': [
'*****@*****.**',
]
},
Message={
'Subject': {
'Data': email_subject
},
'Body': {
'Text': {
'Data': email_subject
},
'Html': {
'Data': rendered_html
}
}
})
except Exception as e:
Logger().exception("Exception occurred sending " + str(email_subject) +
" email to: " + str(user_list))
def send_clean_files_email(datasource_id, user_email, full_name, initiated_by):
try:
if not datasource_id:
return "Invalid datasource! Aborting..."
db_session = db_connection().get_session()
datasource = db_session.query(DataSource).filter(
DataSource.datasource_id == datasource_id).first()
login_user = db_session.query(LoginUser).filter(
LoginUser.domain_id == datasource.domain_id,
LoginUser.email == initiated_by).first()
admin_user = login_user.first_name + " " + login_user.last_name
template_name = "clean_files"
template_parameters = {
"user_name": full_name,
"admin_user": admin_user,
"user_first_name": full_name.split(" ")[0]
}
rendered_html = get_rendered_html(template_name, template_parameters)
email_subject = "Please log in to Adya to manage your G Suite data"
aws_utils.send_email([user_email], email_subject, rendered_html)
return True
except Exception as e:
Logger().exception("Exception occurred sending clean files email")
return False
def add_permissions(auth_token, permissions, owner_email, initiated_by_email, datasource_id, domain_id):
is_success = True
exception_messages = ""
drive_service = gutils.get_gdrive_service(auth_token, owner_email)
for permission in permissions:
resource_id = permission['resource_id']
role = permission['permission_type']
email = permission['email']
email_type = "user"
db_session = db_connection().get_session()
existing_group = db_session.query(DomainUser).filter(
and_(DomainUser.datasource_id == permission['datasource_id'],
DomainUser.email == email)).first()
if existing_group:
email_type = "group"
add_permission_object = {
"role": role,
"type": email_type,
"emailAddress": email
}
request = drive_service.permissions().create(fileId=resource_id, body=add_permission_object, quotaUser= owner_email[0:41],
transferOwnership=True if role == 'owner' else False,
fields='id, emailAddress, type, kind, displayName')
try:
# check for existing permission only if action role is of 'owner'
response = request.execute()
permission['displayName'] = response['displayName'] if 'displayName' in response else ""
Logger().info("Add permission response from google is - {}".format(response))
permission['permission_id'] = response['id']
add_new_permission_to_db(permission, resource_id, datasource_id, initiated_by_email, role, domain_id)
except Exception as ex:
Logger().exception("Exception occurred while adding a new permission")
is_success = False
if ex.content:
content = json.loads(ex.content)
exception_messages += content['error']['message'] if content['error']['message'] else "Exception occurred while adding new permission to db"
else:
exception_messages += "Exception occurred while adding new permission to db"
return ResponseMessage(200 if is_success else 500, exception_messages)
def check_if_user_isadmin(auth_token, user_email=None, db_session=None):
try:
directory_service = get_directory_service(auth_token, user_email,
db_session)
directory_service.users().get(userKey=user_email).execute()
return ""
except HttpError as ex:
ex_msg = json.loads(ex.content)["error"]["message"]
Logger().info(
"Could not check if user is admin user, so mark user as non admin - {}"
.format(ex_msg))
return ex_msg
except Exception as ex:
Logger().info(
"Could not check if user is admin user, so mark user as non admin - {}"
.format(ex.message))
return ex.message
def close_connection(self):
try:
if self._session:
self._session.remove()
if self._engine:
self._engine.dispose()
except:
Logger().exception("Exception occurred while closing the db connection")
def process_group_related_activities(datasource_id, actor_email, event):
event_name = event['name']
if event_name == 'ADD_GROUP_MEMBER':
activity_events_parameters = event['parameters']
group_email = None
user_email = None
for param in activity_events_parameters:
name = param['name']
if name == 'GROUP_EMAIL':
group_email = param['value']
elif name == 'USER_EMAIL':
user_email = param['value']
user_directory_struct = DirectoryStructure()
user_directory_struct.datasource_id = datasource_id
user_directory_struct.member_email = user_email
user_directory_struct.parent_email = group_email
user_directory_struct.member_role = 'MEMBER'
user_directory_struct.member_type = 'USER' # TODO : check whether type is group or user
db_session = db_connection().get_session()
db_session.execute(DirectoryStructure.__table__.insert().prefix_with("IGNORE").
values(db_utils.get_model_values(DirectoryStructure, user_directory_struct)))
if user_email:
datasource_obj = get_datasource(datasource_id)
domain_id = datasource_obj.domain_id
exposure_type = utils.check_if_external_user(db_session, domain_id, user_email)
if exposure_type == constants.EntityExposureType.EXTERNAL.value:
# check if external user present in domain user table
existing_user = db_session.query(DomainUser).filter(and_(DomainUser.datasource_id == datasource_id,
DomainUser.email == user_email)).first()
external_user = None
if not existing_user:
external_user = DomainUser()
external_user.datasource_id = datasource_id
external_user.email = user_email
external_user.member_type = constants.EntityExposureType.EXTERNAL.value
external_user.type = 'USER'
# TODO: find the first name and last name of external user
external_user.first_name = ""
external_user.last_name = ""
db_session.add(external_user)
user_obj = existing_user if existing_user else external_user
payload = {}
payload["user"] = json.dumps(user_obj, cls=alchemy_encoder())
policy_params = {'dataSourceId': datasource_id,
'policy_trigger': constants.PolicyTriggerType.NEW_USER.value}
Logger().info("new_user : payload : {}".format(payload))
messaging.trigger_post_event(urls.GSUITE_POLICIES_VALIDATE_PATH, constants.INTERNAL_SECRET, policy_params, payload,
"gsuite")
datasource_obj = get_datasource(datasource_id)
tags = {"group_email": group_email, "user_email":user_email}
activity_db().add_event(domain_id=datasource_obj.domain_id, connector_type=constants.ConnectorTypes.GSUITE.value,
event_type='ADD_GROUP_MEMBER', actor=actor_email, tags=tags)
db_connection().commit()
def delete_user_from_group(auth_token, group_email, user_email):
directory_service = gutils.get_directory_service(auth_token)
Logger().info("Initiating removal of user {} from group {} ...".format(user_email, group_email))
try:
response = directory_service.members().delete(groupKey=group_email, memberKey=user_email).execute()
return ResponseMessage(200, None, response)
except Exception as ex:
content = json.loads(ex.content) if ex.content else {}
return ResponseMessage(500, None, content)
def delete_alert_for_a_policy(policy_id):
delete_response = None
if policy_id:
db_session = db_connection().get_session()
Logger().info("delete alert for policy id: {}".format(policy_id))
delete_response = db_session.query(Alert).filter(
Alert.policy_id == policy_id).delete()
db_connection().commit()
return delete_response
def call_validate_policies_for_admin_user(user_obj, datasource_id):
if user_obj and user_obj.is_admin:
payload = {}
payload["user"] = json.dumps(user_obj, cls=alchemy_encoder())
policy_params = {'dataSourceId': datasource_id,
'policy_trigger': constants.PolicyTriggerType.NEW_USER.value}
Logger().info("new_user : payload : {}".format(payload))
messaging.trigger_post_event(urls.GSUITE_POLICIES_VALIDATE_PATH, constants.INTERNAL_SECRET, policy_params,
payload, "gsuite")
def request_scanner_data(auth_token, query_params):
datasource_id = query_params["dataSourceId"]
scanner_id = query_params["scannerId"]
db_session = db_connection().get_session()
scanner = db_session.query(DatasourceScanners).filter(DatasourceScanners.datasource_id == datasource_id, DatasourceScanners.id == scanner_id).first()
if not scanner:
return
response = None
try:
response = get_scanner_processor(scanner.scanner_type).query(auth_token, query_params, scanner)
except Exception as ex:
Logger().exception("Exception occurred while querying scan data for - {} - {} ".format(query_params, ex))
db_session.query(DatasourceScanners).filter(and_(DatasourceScanners.datasource_id == datasource_id, DatasourceScanners.id == scanner_id)). \
update({DatasourceScanners.in_progress: 0})
db_connection().commit()
return
next_page_token = response["nextPageNumber"] if "nextPageNumber" in response else None
if next_page_token:
scanner.next_page_token = str(next_page_token)
query_params["nextPageNumber"] = scanner.next_page_token
messaging.trigger_get_event(urls.GITHUB_SCAN_ENTITIES, auth_token, query_params, "github")
else:
scanner.next_page_token = ""
entities_list = response["payload"]
fetched_entities_count = len(entities_list)
in_progress = 0 if fetched_entities_count < 1 else 1
db_session.query(DatasourceScanners).filter(and_(DatasourceScanners.datasource_id == datasource_id, DatasourceScanners.id == scanner_id)). \
update({DatasourceScanners.total_count: DatasourceScanners.total_count + fetched_entities_count,
DatasourceScanners.query_status: DatasourceScanners.query_status + 1})
if in_progress == 0:
db_session.query(DatasourceScanners).filter(and_(DatasourceScanners.datasource_id == datasource_id, DatasourceScanners.id == scanner_id)). \
update({DatasourceScanners.in_progress: in_progress})
db_connection().commit()
messaging.trigger_post_event(urls.GITHUB_SCAN_UPDATE, auth_token, query_params, {}, "github")
return
datasource_metric_column = get_datasource_column(scanner.scanner_type)
if datasource_metric_column:
db_session.query(DataSource).filter(DataSource.datasource_id == datasource_id). \
update({datasource_metric_column: datasource_metric_column + fetched_entities_count})
db_connection().commit()
sent_member_count = 0
batch_size = response["batchSize"] if "batchSize" in response else fetched_entities_count
while sent_member_count < fetched_entities_count:
scanner_data = {}
scanner_data["entities"] = entities_list[sent_member_count:sent_member_count + batch_size]
#If this is the last set of users, in the process call, send the next page number as empty
if fetched_entities_count - sent_member_count <= batch_size and not scanner.next_page_token:
query_params["nextPageNumber"] = ""
messaging.trigger_post_event(urls.GITHUB_SCAN_ENTITIES, auth_token, query_params, scanner_data, "github")
sent_member_count += batch_size
def delete_permissions(auth_token, permissions, owner_email, initiated_by_email, datasource_id):
drive_service = gutils.get_gdrive_service(auth_token, owner_email)
updated_permissions = {}
is_success = True
exception_messages = ""
for permission in permissions:
resource_id = permission['resource_id']
permission_id = permission['permission_id']
retry = 0
while True:
retry += 1
try:
drive_service.permissions().delete(fileId=resource_id, quotaUser= owner_email[0:41], permissionId=permission_id).execute()
if not permission['resource_id'] in updated_permissions:
updated_permissions[permission['resource_id']] = [permission]
else:
updated_permissions[permission['resource_id']].append(permission)
break
except HttpError as ex:
if ex.resp.status == 404:
Logger().info("Permission not found : permission - {} : ex - {}".format(permission, ex))
if not permission['resource_id'] in updated_permissions:
updated_permissions[permission['resource_id']] = [permission]
else:
updated_permissions[permission['resource_id']].append(permission)
break
elif ex.resp.status == 403 and retry < 6:
#API limit reached, so retry after few seconds for 5 times
sleep_secs = min(64, (2 ** retry)) + (random.randint(0, 1000) / 1000.0)
Logger().warn("API limit reached while deleting the permission in gsuite, will retry after {} secs: {}".format(sleep_secs, permission))
time.sleep(sleep_secs)
elif ex.resp.status == 500 and retry < 6:
sleep_secs = min(64, (2 ** retry)) + (random.randint(0, 1000) / 1000.0)
Logger().warn("Backend Error while deleting the permission in gsuite, will retry after {} secs: {}".format(
sleep_secs, permission))
time.sleep(sleep_secs)
else:
Logger().exception("Exception occurred while deleting permissions in gsuite : permission - {}".format(permission))
is_success = False
if ex.content:
content = json.loads(ex.content)
exception_messages += content['error']['message'] if content['error']['message'] else "Exception occurred while deleting permissions in gsuite"
break
except Exception as ex:
Logger().exception("Exception occurred while deleting permissions in gsuite : ex - {}".format(ex))
is_success = False
exception_messages += "Exception occurred while deleting permissions in gsuite"
break
try:
Logger().info("updated_permissions to be deleted from db - {}".format(updated_permissions))
delete_resource_permission(initiated_by_email, datasource_id, updated_permissions)
except Exception:
Logger().exception("Exception occurred while removing permission from db")
is_success = False
exception_messages += "Exception occurred while removing permission from db"
return ResponseMessage(200 if is_success else 500, exception_messages)
def unsubscribed_all_the_previous_subscription(datasource_id):
db_session = db_connection().get_session()
subscriptions = db_session.query(PushNotificationsSubscription).filter(
PushNotificationsSubscription.datasource_id == datasource_id).all()
for subscription in subscriptions:
unsubscribe_subscription(subscription)
Logger().info("unsubscribed all the channel for datasource - {} ".format(
datasource_id))
def validate_apps_installed_policy(db_session, auth_token, datasource_id,
policy, application):
Logger().info("validating_policy : application : {}".format(application))
is_violated = 1
for policy_condition in policy.conditions:
if policy_condition.match_type == constants.PolicyMatchType.APP_NAME.value:
is_violated = is_violated & check_value_violation(
policy_condition, application["display_text"])
elif policy_condition.match_type == constants.PolicyMatchType.APP_RISKINESS.value:
is_violated = is_violated & check_value_violation(
policy_condition, application["score"])
elif policy_condition.match_type == constants.PolicyMatchType.IS_APP_WHITELISTED.vlaue:
is_violated = is_violated & check_value_violation(
policy_condition, application["is_whitelisted"])
send_email_action = []
is_reverted = False
if is_violated:
Logger().info(
"Policy \"{}\" is violated, so triggering corresponding actions".
format(policy.name))
for action in policy.actions:
if action.action_type == constants.PolicyActionType.SEND_EMAIL.value:
send_email_action.append(policy)
elif action.action_type == constants.PolicyActionType.REVERT.value:
remove_app_for_domain(auth_token, application["id"])
is_reverted = True
if len(send_email_action) > 0:
to_address = json.loads(send_email_action[0].config)["to"]
adya_emails.send_app_install_policy_violate_email(
to_address, policy, application, is_reverted)
payload = {}
payload["datasource_id"] = datasource_id
payload["name"] = policy.name
payload["policy_id"] = policy.policy_id
payload["severity"] = policy.severity
payload[
"description_template"] = "New app install \"{{display_text}}\" for \"{{user_email}}\" has violated policy \"{{policy_name}}\""
payload["payload"] = application
messaging.trigger_post_event(urls.ALERTS_PATH, auth_token, None,
payload)
def transfer_ownership(auth_token, old_owner_email, new_owner_email):
datatransfer_service = gutils.get_gdrive_datatransfer_service(auth_token)
Logger().info("Initiating data transfer...")
applicationDataTransfers = get_applicationDataTransfers_for_gdrive(datatransfer_service)
directory_service = gutils.get_directory_service(auth_token)
old_user_id = directory_service.users().get(userKey=old_owner_email).execute()
old_user_id = old_user_id.get('id')
new_user_id = directory_service.users().get(userKey=new_owner_email).execute()
new_user_id = new_user_id.get('id')
transfersResource = {"oldOwnerUserId": old_user_id, "newOwnerUserId": new_user_id,
"applicationDataTransfers": applicationDataTransfers}
try:
response = datatransfer_service.transfers().insert(body=transfersResource).execute()
Logger().info(str(response))
return ResponseMessage(200, None, response)
except Exception as ex:
content = json.loads(ex.content) if ex.content else {}
return ResponseMessage(500, None, content)