def process_notifications(notification_type, datasource_id, channel_id, body): if notification_type == "sync": return db_session = db_connection().get_session() subscription = db_session.query(PushNotificationsSubscription).filter( PushNotificationsSubscription.channel_id == channel_id).first() if not subscription: Logger().warn("Subscription does not exist for datasource_id: {} and channel_id: {}, hence ignoring the notification.".format( datasource_id, channel_id)) return activities = [body] try: # If notification type is adya, then that means its triggered either manually or scheduled sync # So we need to fetch the events from last sync time if notification_type == "adya": reports_service = gutils.get_gdrive_reports_service(None, subscription.user_email, db_session) results = reports_service.activities().list(userKey="all", applicationName=subscription.notification_type, startTime=subscription.page_token).execute() if results and "items" in results: activities = results["items"] for activity in activities: process_incoming_activity(datasource_id, activity) db_session.refresh(subscription) subscription.last_accessed = datetime.datetime.utcnow() subscription.page_token = datetime.datetime.utcnow().isoformat("T") + "Z" db_connection().commit() except Exception as e: Logger().exception("Exception occurred while processing activity notification for datasource_id: {} channel_id: {} - {}".format(datasource_id, channel_id, e))
def delete_cloudwatch_event(cloudwatch_event_name, function_name): try: session = boto3.Session() cloudwatch_client = session.client('events') Logger().info("delete_cloudwatch_event : ") # remove all the targets from the rule response = cloudwatch_client.remove_targets(Rule=cloudwatch_event_name, Ids=[ function_name, ]) Logger().info("removed target : ") if response and response['ResponseMetadata'][ 'HTTPStatusCode'] == constants.SUCCESS_STATUS_CODE: # after removing all the targets , now delete the rule response = cloudwatch_client.delete_rule( Name=cloudwatch_event_name) Logger().info("removed rule : ") except Exception as ex: Logger().exception( "Exception occurred while deleting the cloudwatch event - ") return False
def trigger_post_event_with_headers( endpoint, auth_token, query_params, headers, body, service_name="core", trigger_type=constants.TriggerType.ASYNC.value): result = None if constants.DEPLOYMENT_ENV == 'local': session = FuturesSession() endpoint = _add_query_params_to_url(endpoint, query_params) Logger().info("Making a POST request on the following url - " + str(endpoint)) response = utils.post_call_with_authorization_header( session, endpoint, auth_token, body, headers) if trigger_type == constants.TriggerType.SYNC.value: api_response = response.result() result = ResponseMessage(api_response.status_code, None, json.loads(api_response.content)) else: body = _add_query_params_to_body(body, query_params) body = _add_headers_to_body(body, headers) endpoint = service_name + "-" + \ constants.DEPLOYMENT_ENV + "-post-" + slugify(endpoint) Logger().info( "Making a POST lambda invoke on the following function - " + str(endpoint)) result = aws_utils.invoke_lambda(endpoint, auth_token, body, trigger_type) return result
def query(auth_token, query_params, scanner): domain_id = query_params["domainId"] next_page_token = query_params["nextPageNumber"] directory_service = gutils.get_directory_service(auth_token) users = [] try: results = directory_service.users().list(customer='my_customer', maxResults=20, pageToken=next_page_token, orderBy='email').execute() except RefreshError as ex: Logger().info("User query : Not able to refresh credentials") results = gutils.create_user_payload_for_nonadmin_nonserviceaccount( auth_token) except HttpError as ex: Logger().info("User query : Domain not found error") results = gutils.create_user_payload_for_nonadmin_nonserviceaccount( auth_token) if results and "users" in results: for user in results["users"]: user_email = user.get("primaryEmail") if user_email.endswith(domain_id): users.append(user) next_page_token = results.get('nextPageToken') return {"payload": users, "nextPageNumber": next_page_token}
def delete_public_and_external_sharing_for_file(self): for permission in self.permissions: if permission[ 'exposure_type'] == constants.EntityExposureType.ANYONEWITHLINK.value: file_id = permission['resource_id'] try: removed_file = self.slack_client.api_call( "files.revokePublicURL", file=file_id) self.response = removed_file except Exception as ex: Logger().exception( "Exception ocuured while removing the permission - {}". format(ex)) self.exception_messages.append(ex.message) if not file_id in self.updated_permissions: self.updated_permissions[file_id] = [permission] else: self.updated_permissions[file_id].append(permission) try: delete_resource_permission(self.initiated_by_email, self.datasource_id, self.updated_permissions) except Exception as ex: Logger().exception( "Exception occurred while removing permission from db") self.exception_messages.append( "Exception occurred while removing permission from db") return self.response
def process_login_activity(datasource_id, incoming_activity): Logger().info('Processing login activity {}'.format(incoming_activity)) db_session = db_connection().get_session() actor_email = incoming_activity['actor']['email'] events = incoming_activity["events"] login_time = datetime.datetime.strptime(incoming_activity["id"]["time"], '%Y-%m-%dT%H:%M:%S.%fZ') ninety_days_ago = datetime.datetime.utcnow() - datetime.timedelta(days=90) is_inactive = login_time < ninety_days_ago last_is_inactive = None for event in events: if event["name"] == 'login_success': last_login_time = db_session.query(DomainUser).filter(DomainUser.datasource_id == datasource_id, DomainUser.email == actor_email).first().last_login_time if last_login_time: last_is_inactive = last_login_time < ninety_days_ago if not last_login_time or (login_time > last_login_time): db_session.query(DomainUser).filter(DomainUser.datasource_id == datasource_id, DomainUser.email == actor_email).update({"last_login_time":login_time}, synchronize_session = 'fetch') datasource = db_session.query(DataSource).filter(DataSource.datasource_id == datasource_id).first() app_name = constants.datasource_to_installed_app_map[datasource.datasource_type] if (last_is_inactive is None) or not last_is_inactive : if is_inactive: db_session.query(Application).filter(and_(Application.domain_id == datasource.domain_id, Application.display_text == app_name)).update({"inactive_users":Application.inactive_users + 1}, synchronize_session = 'fetch') Logger().info("new inactive user : {}".format(actor_email)) elif last_is_inactive and not is_inactive: Logger().info("inactive user is now active : {}".format(actor_email)) db_session.query(Application).filter(and_(Application.domain_id == datasource.domain_id, Application.display_text == app_name, Application.inactive_users > 0)).update({"inactive_users":Application.inactive_users - 1}, synchronize_session = 'fetch') db_connection().commit()
def process_user_related_activities(datasource_id, actor_email, event): event_name = event['name'] activity_events_parameters = event['parameters'] user_email = None user_obj = None db_session = db_connection().get_session() for param in activity_events_parameters: name = param['name'] if name == 'USER_EMAIL': user_email = param['value'] datasource_obj = get_datasource(datasource_id) tags = {"user_email": user_email} if event_name == 'CREATE_USER': if user_email: directory_service = gutils.get_directory_service(None, actor_email) results = None try: results = directory_service.users().get(userKey=user_email).execute() except RefreshError as ex: Logger().info("User query : Not able to refresh credentials") except HttpError as ex: Logger().info("User query : Domain not found error") if results: gsuite_user = user.GsuiteUser(datasource_id, results) user_obj = gsuite_user.get_model() db_session.execute(DomainUser.__table__.insert().prefix_with("IGNORE"). values(db_utils.get_model_values(DomainUser, user_obj))) call_validate_policies_for_admin_user(user_obj, datasource_id) additional_payload = {"user_email": user_email, "is_admin": user_obj.is_admin} tags["is_admin"] = user_obj.is_admin elif event_name == 'GRANT_ADMIN_PRIVILEGE': user_obj = db_session.query(DomainUser).filter(and_(DomainUser.datasource_id == datasource_id, DomainUser.email == user_email)).first() if user_obj: user_obj.is_admin = True call_validate_policies_for_admin_user(user_obj, datasource_id) tags["is_admin"] = user_obj.is_admin elif event_name == "SUSPEND_USER": db_session.query(DomainUser).filter(and_(DomainUser.datasource_id == datasource_id, DomainUser.email == user_email)).update({DomainUser.is_suspended: True}) elif event_name == "DELETE_USER": delete_user_info(db_session, user_email, datasource_id) activity_db().add_event(domain_id=datasource_obj.domain_id, connector_type=constants.ConnectorTypes.GSUITE.value, event_type=event_name, actor=actor_email, tags=tags) db_connection().commit()
def initiate_action(auth_token, action_payload): try: action_key = action_payload['key'] initiated_by = action_payload['initiated_by'] action_parameters = action_payload['parameters'] datasource_id = action_payload[ 'datasource_id'] if 'datasource_id' in action_payload else 'MANUAL' db_session = db_connection().get_session() login_user_info = db_session.query(LoginUser).filter( LoginUser.auth_token == auth_token).first() domain_id = login_user_info.domain_id action_config = get_action(action_key) if not action_config or not validate_action_parameters( action_config, action_parameters): return ResponseMessage( 400, "Failed to execute action - Validation failed") log_id = action_payload[ 'log_id'] if 'log_id' in action_payload else None if log_id: log_entry = db_session.query(AuditLog).filter( and_(AuditLog.log_id == log_id)).first() else: log_entry = audit_action(domain_id, datasource_id, initiated_by, action_config, action_parameters) execution_status = execute_action(auth_token, domain_id, datasource_id, action_config, action_payload, log_entry) db_connection().commit() Logger().info("initiate_action : response body - {}".format( execution_status.get_response_body())) response_body = json.loads( json.dumps(execution_status.get_response_body())) response_body['id'] = log_entry.log_id if execution_status.response_code == constants.ACCEPTED_STATUS_CODE: action_payload['log_id'] = log_entry.log_id messaging.trigger_post_event(urls.INITIATE_ACTION_PATH, auth_token, None, action_payload) return ResponseMessage(execution_status.response_code, None, response_body) except Exception as e: Logger().exception( "Exception occurred while initiating action using payload " + str(action_payload)) return ResponseMessage(500, "Failed to execute action - {}".format(e))
def update_scan(auth_token, datasource_id, domain_id): time.sleep(2) db_session = db_connection().get_session() scan_complete = db_session.query(DatasourceScanners).filter(and_(DatasourceScanners.datasource_id == datasource_id, DatasourceScanners.in_progress > 0)).count() #print "Total scanners in progress - {}".format(scan_complete) if scan_complete == 0: scan_complete = db_session.query(DataSource).filter(and_(DataSource.datasource_id == datasource_id, DataSource.is_push_notifications_enabled == 0)). \ update({DataSource.is_push_notifications_enabled: 1}) Logger().info("update_scan: scan_complete - {}".format(scan_complete)) #print "Scan complete status - {}".format(scan_complete) if scan_complete > 0: Logger().info("update_scan : call Scan completed processing method") scan_complete_processing(db_session, auth_token, datasource_id)
def query(auth_token, query_params, scanner): datasource_id = query_params["dataSourceId"] repo_name = query_params["repo_name"] github_client = github_utils.get_github_client(datasource_id) collaborators_list = [] #collaborator_permissions_dict = {} repo = github_client.get_repo(repo_name) if repo: for collaborator in repo.get_collaborators(): collaborator.raw_data[ "permission_type"] = constants.Role.READER.value if collaborator.permissions.raw_data: if collaborator.permissions.raw_data["admin"]: collaborator.raw_data[ "permission_type"] = constants.Role.OWNER.value elif collaborator.permissions.raw_data["push"]: collaborator.raw_data[ "permission_type"] = constants.Role.WRITER.value collaborators_list.append(collaborator.raw_data) #Creating a webhook for the repository if not constants.DEPLOYMENT_ENV == "local": try: config = { "url": urls.GITHUB_NOTIFICATIONS_URL, "content_type": "json" } events = [ "repository", "repository_vulnerability_alert", "fork", "member", "public", "push", "create" ] repo.create_hook(name="web", config=config, events=events, active=True) except github.GithubException as ex: if ex.status == 422: Logger().info( "Webhook already exist for repository - {} with exception - {}" .format(repo_name, ex)) else: Logger().exception( "Github Exception occurred while subscribing for push notification for repository = {} with exception - {}" .format(repo_name, ex)) except Exception as ex: Logger().exception( "Exception occurred while subscribing for push notification for repository = {} with exception - {}" .format(repo_name, ex)) return {"payload": collaborators_list}
def async_delete_datasource(auth_token, datasource_id, complete_delete): complete_delete = 0 if complete_delete == 0 else 1 db_session = db_connection().get_session() existing_datasource = db_session.query(DataSource).filter( DataSource.datasource_id == datasource_id).first() try: app_name = constants.datasource_to_installed_app_map[existing_datasource.datasource_type] db_session.execute(DirectoryStructure.__table__.delete().where(DirectoryStructure.datasource_id == datasource_id)) db_session.execute(ResourcePermission.__table__.delete().where(ResourcePermission.datasource_id == datasource_id)) db_session.execute(Resource.__table__.delete().where(Resource.datasource_id == datasource_id)) app_query = db_session.query(ApplicationUserAssociation).filter( ApplicationUserAssociation.datasource_id == datasource_id) app_ids = [r.application_id for r in app_query.all()] app_query.delete(synchronize_session=False) try: db_session.execute(Application.__table__.delete().where(Application.id.in_(app_ids))) # delete app wrt to datasource db_session.query(Application).filter(Application.display_text == app_name).delete(synchronize_session=False) except: Logger().info('App is present corresponding to other datasource') db_session.execute(PushNotificationsSubscription.__table__.delete().where(PushNotificationsSubscription.datasource_id == datasource_id)) db_session.execute(DomainUser.__table__.delete().where(DomainUser.datasource_id == datasource_id)) db_session.execute(DatasourceScanners.__table__.delete().where(DatasourceScanners.datasource_id == datasource_id)) #If its not complete delete, then just clear out the scan entities, and reset the scan state if complete_delete: db_session.execute(AuditLog.__table__.delete().where(AuditLog.datasource_id == datasource_id)) db_session.execute(Alert.__table__.delete().where(Alert.datasource_id == datasource_id)) db_session.execute(PolicyAction.__table__.delete().where(PolicyAction.datasource_id == datasource_id)) db_session.execute(PolicyCondition.__table__.delete().where(PolicyCondition.datasource_id == datasource_id)) db_session.execute(Policy.__table__.delete().where(Policy.datasource_id == datasource_id)) db_session.execute(DatasourceCredentials.__table__.delete().where(DatasourceCredentials.datasource_id == datasource_id)) db_session.delete(existing_datasource) else: existing_datasource.processed_file_count = 0 existing_datasource.processed_group_count = 0 existing_datasource.processed_user_count = 0 existing_datasource.total_file_count = 0 existing_datasource.total_group_count = 0 existing_datasource.total_user_count = 0 existing_datasource.file_scan_status = 0 existing_datasource.group_scan_status = 0 existing_datasource.user_scan_status = 0 existing_datasource.is_push_notifications_enabled = 0 db_connection().commit() Logger().info("Datasource deleted successfully") except: Logger().exception("Exception occurred during datasource data delete")
def process_incoming_activity(datasource_id, incoming_activity): if not "id" in incoming_activity: Logger().info("Incoming activity is not valid type - {}".format(incoming_activity)) return Logger().info("Incoming activity - {}".format(incoming_activity)) app_name = incoming_activity["id"]["applicationName"] if app_name == "token": process_token_activity(datasource_id, incoming_activity) elif app_name == "admin": process_admin_activities(datasource_id, incoming_activity) elif app_name == 'login': process_login_activity(datasource_id, incoming_activity) elif app_name == "drive": process_drive_activity(datasource_id, incoming_activity)
def update_permissions(auth_token, permissions, owner_email, initiated_by_email, datasource_id): drive_service = gutils.get_gdrive_service(auth_token, owner_email) updated_permissions = {} deleted_permissions = {} is_success = True exception_messages = "" for permission in permissions: resource_id = permission['resource_id'] permission_id = permission['permission_id'] role = permission['permission_type'] update_permission_object = { "role": role, } try: drive_service.permissions().update(fileId=resource_id, body=update_permission_object, quotaUser= owner_email[0:41], permissionId=permission_id, transferOwnership=True if role == 'owner' else False).execute() if not permission['resource_id'] in updated_permissions: updated_permissions[permission['resource_id']] = [permission] else: updated_permissions[permission['resource_id']].append(permission) except HttpError as e: if e.resp.status == 404: Logger().info("Permission not found in gsuite, hence delete from db") if not permission['resource_id'] in deleted_permissions: deleted_permissions[permission['resource_id']] = [permission] else: deleted_permissions[permission['resource_id']].append(permission) else: Logger().exception("HttpError Exception occurred while updating permissions in gsuite ; {}".format(e)) is_success = False if e.content: content = json.loads(e.content) exception_messages += content['error']['message'] if content['error']['message'] else "Exception occurred while updating permissions in gsuite" except Exception as ex: Logger().exception("Exception occurred while updating permissions in gsuite : {}".format(ex)) is_success = False exception_messages += "Exception occurred while updating permissions in gsuite" try: update_resource_permissions(initiated_by_email, datasource_id, updated_permissions) if len(deleted_permissions) > 0: delete_resource_permission(initiated_by_email, datasource_id, deleted_permissions) except Exception as ex: Logger().exception("Exception occurred while updating permission from db : {}".format(ex)) is_success = False exception_messages += "Exception occurred while updating permission from db" return ResponseMessage(200 if is_success else 500, exception_messages)
def process_user(db_session, datasource, payload): datasource_credentials = get_datasource_credentials( db_session, datasource.datasource_id) if datasource_credentials: domain_id = datasource_credentials['domain_id'] userObj = entities.SlackUser(domain_id, datasource.datasource_id, payload) user_model_obj = userObj.get_model() db_session.execute( DomainUser.__table__.insert().prefix_with("IGNORE").values( db_utils.get_model_values(DomainUser, user_model_obj))) db_session.commit() #check if new external member is added to a team if user_model_obj.member_type == constants.EntityExposureType.EXTERNAL.value: payload = {} payload["user"] = json.dumps(user_model_obj, cls=alchemy_encoder()) policy_params = { 'dataSourceId': datasource.datasource_id, 'policy_trigger': constants.PolicyTriggerType.NEW_USER.value } Logger().info("new_user : payload : {}".format(payload)) messaging.trigger_post_event(urls.SLACK_POLICIES_VALIDATE_PATH, constants.INTERNAL_SECRET, policy_params, payload, "slack") activity_db().add_event( domain_id=datasource.domain_id, connector_type=constants.ConnectorTypes.SLACK.value, event_type='USER_ADDED', actor=user_model_obj.email, tags={ "exposure_type": user_model_obj.member_type, "user_email": user_model_obj.email })
def update_report(auth_token, payload): if not auth_token: return None db_session = db_connection().get_session() if payload: report = {} report["name"] = payload["name"] if 'description' in payload: report["description"] = payload["description"] report["frequency"] = payload["frequency"] report["receivers"] = payload["receivers"] config_input = { "report_type": payload["report_type"], "selected_entity_type": payload["selected_entity_type"], "selected_entity": payload["selected_entity"], "datasource_id": payload["datasource_id"], "selected_entity_name": payload["selected_entity_name"] } report["config"] = json.dumps(config_input) report["is_active"] = payload["is_active"] report_id = payload["report_id"] db_session.query(Report).filter( Report.report_id == report_id).update(report) try: db_connection().commit() except Exception as ex: Logger().exception() return payload else: return None
def invoke_lambda(function_name, auth_token, body, trigger_type=constants.TriggerType.ASYNC.value): if not body: body = {} body['Authorization'] = auth_token client = boto3.client('lambda') response = client.invoke( FunctionName=function_name, InvocationType='Event' if trigger_type == constants.TriggerType.ASYNC.value else 'RequestResponse', LogType='None', Payload=bytes(json.dumps(body))) response_payload = {"statusCode": 200, "body": ""} if trigger_type == constants.TriggerType.SYNC.value: response_payload = json.loads( response['Payload'].read().decode("utf-8")) Logger().info("Response from sync lambda invocation is - {}".format( response_payload)) if "errorMessage" in response_payload: return ResponseMessage(500, response_payload['errorMessage']) return ResponseMessage(response_payload['statusCode'], None, response_payload['body'])
def send_email(user_list, email_subject, rendered_html): try: session = boto3.Session() ses_client = session.client('ses') ses_client.send_email(Source='*****@*****.**', Destination={ 'ToAddresses': user_list, 'BccAddresses': [ '*****@*****.**', ] }, Message={ 'Subject': { 'Data': email_subject }, 'Body': { 'Text': { 'Data': email_subject }, 'Html': { 'Data': rendered_html } } }) except Exception as e: Logger().exception("Exception occurred sending " + str(email_subject) + " email to: " + str(user_list))
def send_clean_files_email(datasource_id, user_email, full_name, initiated_by): try: if not datasource_id: return "Invalid datasource! Aborting..." db_session = db_connection().get_session() datasource = db_session.query(DataSource).filter( DataSource.datasource_id == datasource_id).first() login_user = db_session.query(LoginUser).filter( LoginUser.domain_id == datasource.domain_id, LoginUser.email == initiated_by).first() admin_user = login_user.first_name + " " + login_user.last_name template_name = "clean_files" template_parameters = { "user_name": full_name, "admin_user": admin_user, "user_first_name": full_name.split(" ")[0] } rendered_html = get_rendered_html(template_name, template_parameters) email_subject = "Please log in to Adya to manage your G Suite data" aws_utils.send_email([user_email], email_subject, rendered_html) return True except Exception as e: Logger().exception("Exception occurred sending clean files email") return False
def add_permissions(auth_token, permissions, owner_email, initiated_by_email, datasource_id, domain_id): is_success = True exception_messages = "" drive_service = gutils.get_gdrive_service(auth_token, owner_email) for permission in permissions: resource_id = permission['resource_id'] role = permission['permission_type'] email = permission['email'] email_type = "user" db_session = db_connection().get_session() existing_group = db_session.query(DomainUser).filter( and_(DomainUser.datasource_id == permission['datasource_id'], DomainUser.email == email)).first() if existing_group: email_type = "group" add_permission_object = { "role": role, "type": email_type, "emailAddress": email } request = drive_service.permissions().create(fileId=resource_id, body=add_permission_object, quotaUser= owner_email[0:41], transferOwnership=True if role == 'owner' else False, fields='id, emailAddress, type, kind, displayName') try: # check for existing permission only if action role is of 'owner' response = request.execute() permission['displayName'] = response['displayName'] if 'displayName' in response else "" Logger().info("Add permission response from google is - {}".format(response)) permission['permission_id'] = response['id'] add_new_permission_to_db(permission, resource_id, datasource_id, initiated_by_email, role, domain_id) except Exception as ex: Logger().exception("Exception occurred while adding a new permission") is_success = False if ex.content: content = json.loads(ex.content) exception_messages += content['error']['message'] if content['error']['message'] else "Exception occurred while adding new permission to db" else: exception_messages += "Exception occurred while adding new permission to db" return ResponseMessage(200 if is_success else 500, exception_messages)
def check_if_user_isadmin(auth_token, user_email=None, db_session=None): try: directory_service = get_directory_service(auth_token, user_email, db_session) directory_service.users().get(userKey=user_email).execute() return "" except HttpError as ex: ex_msg = json.loads(ex.content)["error"]["message"] Logger().info( "Could not check if user is admin user, so mark user as non admin - {}" .format(ex_msg)) return ex_msg except Exception as ex: Logger().info( "Could not check if user is admin user, so mark user as non admin - {}" .format(ex.message)) return ex.message
def close_connection(self): try: if self._session: self._session.remove() if self._engine: self._engine.dispose() except: Logger().exception("Exception occurred while closing the db connection")
def process_group_related_activities(datasource_id, actor_email, event): event_name = event['name'] if event_name == 'ADD_GROUP_MEMBER': activity_events_parameters = event['parameters'] group_email = None user_email = None for param in activity_events_parameters: name = param['name'] if name == 'GROUP_EMAIL': group_email = param['value'] elif name == 'USER_EMAIL': user_email = param['value'] user_directory_struct = DirectoryStructure() user_directory_struct.datasource_id = datasource_id user_directory_struct.member_email = user_email user_directory_struct.parent_email = group_email user_directory_struct.member_role = 'MEMBER' user_directory_struct.member_type = 'USER' # TODO : check whether type is group or user db_session = db_connection().get_session() db_session.execute(DirectoryStructure.__table__.insert().prefix_with("IGNORE"). values(db_utils.get_model_values(DirectoryStructure, user_directory_struct))) if user_email: datasource_obj = get_datasource(datasource_id) domain_id = datasource_obj.domain_id exposure_type = utils.check_if_external_user(db_session, domain_id, user_email) if exposure_type == constants.EntityExposureType.EXTERNAL.value: # check if external user present in domain user table existing_user = db_session.query(DomainUser).filter(and_(DomainUser.datasource_id == datasource_id, DomainUser.email == user_email)).first() external_user = None if not existing_user: external_user = DomainUser() external_user.datasource_id = datasource_id external_user.email = user_email external_user.member_type = constants.EntityExposureType.EXTERNAL.value external_user.type = 'USER' # TODO: find the first name and last name of external user external_user.first_name = "" external_user.last_name = "" db_session.add(external_user) user_obj = existing_user if existing_user else external_user payload = {} payload["user"] = json.dumps(user_obj, cls=alchemy_encoder()) policy_params = {'dataSourceId': datasource_id, 'policy_trigger': constants.PolicyTriggerType.NEW_USER.value} Logger().info("new_user : payload : {}".format(payload)) messaging.trigger_post_event(urls.GSUITE_POLICIES_VALIDATE_PATH, constants.INTERNAL_SECRET, policy_params, payload, "gsuite") datasource_obj = get_datasource(datasource_id) tags = {"group_email": group_email, "user_email":user_email} activity_db().add_event(domain_id=datasource_obj.domain_id, connector_type=constants.ConnectorTypes.GSUITE.value, event_type='ADD_GROUP_MEMBER', actor=actor_email, tags=tags) db_connection().commit()
def delete_user_from_group(auth_token, group_email, user_email): directory_service = gutils.get_directory_service(auth_token) Logger().info("Initiating removal of user {} from group {} ...".format(user_email, group_email)) try: response = directory_service.members().delete(groupKey=group_email, memberKey=user_email).execute() return ResponseMessage(200, None, response) except Exception as ex: content = json.loads(ex.content) if ex.content else {} return ResponseMessage(500, None, content)
def delete_alert_for_a_policy(policy_id): delete_response = None if policy_id: db_session = db_connection().get_session() Logger().info("delete alert for policy id: {}".format(policy_id)) delete_response = db_session.query(Alert).filter( Alert.policy_id == policy_id).delete() db_connection().commit() return delete_response
def call_validate_policies_for_admin_user(user_obj, datasource_id): if user_obj and user_obj.is_admin: payload = {} payload["user"] = json.dumps(user_obj, cls=alchemy_encoder()) policy_params = {'dataSourceId': datasource_id, 'policy_trigger': constants.PolicyTriggerType.NEW_USER.value} Logger().info("new_user : payload : {}".format(payload)) messaging.trigger_post_event(urls.GSUITE_POLICIES_VALIDATE_PATH, constants.INTERNAL_SECRET, policy_params, payload, "gsuite")
def request_scanner_data(auth_token, query_params): datasource_id = query_params["dataSourceId"] scanner_id = query_params["scannerId"] db_session = db_connection().get_session() scanner = db_session.query(DatasourceScanners).filter(DatasourceScanners.datasource_id == datasource_id, DatasourceScanners.id == scanner_id).first() if not scanner: return response = None try: response = get_scanner_processor(scanner.scanner_type).query(auth_token, query_params, scanner) except Exception as ex: Logger().exception("Exception occurred while querying scan data for - {} - {} ".format(query_params, ex)) db_session.query(DatasourceScanners).filter(and_(DatasourceScanners.datasource_id == datasource_id, DatasourceScanners.id == scanner_id)). \ update({DatasourceScanners.in_progress: 0}) db_connection().commit() return next_page_token = response["nextPageNumber"] if "nextPageNumber" in response else None if next_page_token: scanner.next_page_token = str(next_page_token) query_params["nextPageNumber"] = scanner.next_page_token messaging.trigger_get_event(urls.GITHUB_SCAN_ENTITIES, auth_token, query_params, "github") else: scanner.next_page_token = "" entities_list = response["payload"] fetched_entities_count = len(entities_list) in_progress = 0 if fetched_entities_count < 1 else 1 db_session.query(DatasourceScanners).filter(and_(DatasourceScanners.datasource_id == datasource_id, DatasourceScanners.id == scanner_id)). \ update({DatasourceScanners.total_count: DatasourceScanners.total_count + fetched_entities_count, DatasourceScanners.query_status: DatasourceScanners.query_status + 1}) if in_progress == 0: db_session.query(DatasourceScanners).filter(and_(DatasourceScanners.datasource_id == datasource_id, DatasourceScanners.id == scanner_id)). \ update({DatasourceScanners.in_progress: in_progress}) db_connection().commit() messaging.trigger_post_event(urls.GITHUB_SCAN_UPDATE, auth_token, query_params, {}, "github") return datasource_metric_column = get_datasource_column(scanner.scanner_type) if datasource_metric_column: db_session.query(DataSource).filter(DataSource.datasource_id == datasource_id). \ update({datasource_metric_column: datasource_metric_column + fetched_entities_count}) db_connection().commit() sent_member_count = 0 batch_size = response["batchSize"] if "batchSize" in response else fetched_entities_count while sent_member_count < fetched_entities_count: scanner_data = {} scanner_data["entities"] = entities_list[sent_member_count:sent_member_count + batch_size] #If this is the last set of users, in the process call, send the next page number as empty if fetched_entities_count - sent_member_count <= batch_size and not scanner.next_page_token: query_params["nextPageNumber"] = "" messaging.trigger_post_event(urls.GITHUB_SCAN_ENTITIES, auth_token, query_params, scanner_data, "github") sent_member_count += batch_size
def delete_permissions(auth_token, permissions, owner_email, initiated_by_email, datasource_id): drive_service = gutils.get_gdrive_service(auth_token, owner_email) updated_permissions = {} is_success = True exception_messages = "" for permission in permissions: resource_id = permission['resource_id'] permission_id = permission['permission_id'] retry = 0 while True: retry += 1 try: drive_service.permissions().delete(fileId=resource_id, quotaUser= owner_email[0:41], permissionId=permission_id).execute() if not permission['resource_id'] in updated_permissions: updated_permissions[permission['resource_id']] = [permission] else: updated_permissions[permission['resource_id']].append(permission) break except HttpError as ex: if ex.resp.status == 404: Logger().info("Permission not found : permission - {} : ex - {}".format(permission, ex)) if not permission['resource_id'] in updated_permissions: updated_permissions[permission['resource_id']] = [permission] else: updated_permissions[permission['resource_id']].append(permission) break elif ex.resp.status == 403 and retry < 6: #API limit reached, so retry after few seconds for 5 times sleep_secs = min(64, (2 ** retry)) + (random.randint(0, 1000) / 1000.0) Logger().warn("API limit reached while deleting the permission in gsuite, will retry after {} secs: {}".format(sleep_secs, permission)) time.sleep(sleep_secs) elif ex.resp.status == 500 and retry < 6: sleep_secs = min(64, (2 ** retry)) + (random.randint(0, 1000) / 1000.0) Logger().warn("Backend Error while deleting the permission in gsuite, will retry after {} secs: {}".format( sleep_secs, permission)) time.sleep(sleep_secs) else: Logger().exception("Exception occurred while deleting permissions in gsuite : permission - {}".format(permission)) is_success = False if ex.content: content = json.loads(ex.content) exception_messages += content['error']['message'] if content['error']['message'] else "Exception occurred while deleting permissions in gsuite" break except Exception as ex: Logger().exception("Exception occurred while deleting permissions in gsuite : ex - {}".format(ex)) is_success = False exception_messages += "Exception occurred while deleting permissions in gsuite" break try: Logger().info("updated_permissions to be deleted from db - {}".format(updated_permissions)) delete_resource_permission(initiated_by_email, datasource_id, updated_permissions) except Exception: Logger().exception("Exception occurred while removing permission from db") is_success = False exception_messages += "Exception occurred while removing permission from db" return ResponseMessage(200 if is_success else 500, exception_messages)
def unsubscribed_all_the_previous_subscription(datasource_id): db_session = db_connection().get_session() subscriptions = db_session.query(PushNotificationsSubscription).filter( PushNotificationsSubscription.datasource_id == datasource_id).all() for subscription in subscriptions: unsubscribe_subscription(subscription) Logger().info("unsubscribed all the channel for datasource - {} ".format( datasource_id))
def validate_apps_installed_policy(db_session, auth_token, datasource_id, policy, application): Logger().info("validating_policy : application : {}".format(application)) is_violated = 1 for policy_condition in policy.conditions: if policy_condition.match_type == constants.PolicyMatchType.APP_NAME.value: is_violated = is_violated & check_value_violation( policy_condition, application["display_text"]) elif policy_condition.match_type == constants.PolicyMatchType.APP_RISKINESS.value: is_violated = is_violated & check_value_violation( policy_condition, application["score"]) elif policy_condition.match_type == constants.PolicyMatchType.IS_APP_WHITELISTED.vlaue: is_violated = is_violated & check_value_violation( policy_condition, application["is_whitelisted"]) send_email_action = [] is_reverted = False if is_violated: Logger().info( "Policy \"{}\" is violated, so triggering corresponding actions". format(policy.name)) for action in policy.actions: if action.action_type == constants.PolicyActionType.SEND_EMAIL.value: send_email_action.append(policy) elif action.action_type == constants.PolicyActionType.REVERT.value: remove_app_for_domain(auth_token, application["id"]) is_reverted = True if len(send_email_action) > 0: to_address = json.loads(send_email_action[0].config)["to"] adya_emails.send_app_install_policy_violate_email( to_address, policy, application, is_reverted) payload = {} payload["datasource_id"] = datasource_id payload["name"] = policy.name payload["policy_id"] = policy.policy_id payload["severity"] = policy.severity payload[ "description_template"] = "New app install \"{{display_text}}\" for \"{{user_email}}\" has violated policy \"{{policy_name}}\"" payload["payload"] = application messaging.trigger_post_event(urls.ALERTS_PATH, auth_token, None, payload)
def transfer_ownership(auth_token, old_owner_email, new_owner_email): datatransfer_service = gutils.get_gdrive_datatransfer_service(auth_token) Logger().info("Initiating data transfer...") applicationDataTransfers = get_applicationDataTransfers_for_gdrive(datatransfer_service) directory_service = gutils.get_directory_service(auth_token) old_user_id = directory_service.users().get(userKey=old_owner_email).execute() old_user_id = old_user_id.get('id') new_user_id = directory_service.users().get(userKey=new_owner_email).execute() new_user_id = new_user_id.get('id') transfersResource = {"oldOwnerUserId": old_user_id, "newOwnerUserId": new_user_id, "applicationDataTransfers": applicationDataTransfers} try: response = datatransfer_service.transfers().insert(body=transfersResource).execute() Logger().info(str(response)) return ResponseMessage(200, None, response) except Exception as ex: content = json.loads(ex.content) if ex.content else {} return ResponseMessage(500, None, content)