def log_action_on_ticket(ticket=None, action=None, user=None, **kwargs): """ Log ticket updates """ if not user: user = User.objects.get(username=settings.GENERAL_CONFIG['bot_user']) log_msg = _get_log_message(ticket, action, user, **kwargs) History.objects.create( date=datetime.now(), ticket=ticket, user=user, action=log_msg, actionType=''.join(word.capitalize() for word in action.split('_')), ticketStatus=ticket.status, ) if ImplementationFactory.instance.is_implemented('KPIServiceBase'): _generates_kpi_infos(ticket, log_msg) Logger.debug( unicode(action), extra={ 'ticket': ticket.id, 'public_id': ticket.publicId, 'user': user.username, 'action': action, } )
def update_waiting(): """ Update waiting answer tickets """ now = int(time()) for ticket in Ticket.objects.filter(status=WAITING): try: if now > int(mktime(ticket.snoozeStart.timetuple()) + ticket.snoozeDuration): Logger.debug( unicode('Updating status for ticket %s ' % (ticket.id)), extra={ 'ticket': ticket.id, } ) _check_auto_unassignation(ticket) ticket.status = ALARM ticket.snoozeStart = None ticket.snoozeDuration = None ticket.previousStatus = WAITING ticket.reportTicket.all().update(status='Attached') ticket.save() database.log_action_on_ticket( ticket=ticket, action='change_status', previous_value=ticket.previousStatus, new_value=ticket.status ) except (AttributeError, ValueError) as ex: Logger.debug(unicode('Error while updating ticket %d : %s' % (ticket.id, ex)))
def update_paused(): """ Update paused tickets """ now = int(time()) for ticket in Ticket.objects.filter(status=PAUSED): try: if now > int(mktime(ticket.pauseStart.timetuple()) + ticket.pauseDuration): Logger.debug( str('Updating status for ticket %s ' % (ticket.id)), extra={ 'ticket': ticket.id, } ) if ticket.previousStatus == WAITING and ticket.snoozeDuration and ticket.snoozeStart: ticket.snoozeDuration = ticket.snoozeDuration + (datetime.now() - ticket.pauseStart).seconds ticket.status = ticket.previousStatus ticket.pauseStart = None ticket.pauseDuration = None ticket.previousStatus = PAUSED ticket.save() database.log_action_on_ticket( ticket=ticket, action='change_status', previous_value=ticket.previousStatus, new_value=ticket.status ) except (AttributeError, ValueError) as ex: Logger.debug(unicode('Error while updating ticket %d : %s' % (ticket.id, ex)))
def follow_the_sun(): """ Set tickets to alarm when user is away """ now = int(time()) where = [~Q(status='Open'), ~Q(status='Reopened'), ~Q(status='Paused'), ~Q(status='Closed')] where = reduce(operator.and_, where) for user in User.objects.filter(~Q(username=BOT_USER.username)): if now > mktime((user.last_login + timedelta(hours=24)).timetuple()): Logger.debug( unicode('user %s logged out, set alarm to True' % (user.username)), extra={ 'user': user.username, } ) user.ticketUser.filter(where).update(alarm=True) else: Logger.debug( str('user %s logged in, set alarm to False' % (user.username)), extra={ 'user': user.username, } ) user.ticketUser.filter(where).update(alarm=False)
def timeout(ticket_id=None): """ If ticket timeout , apply action on service (if defendant not internal/VIP) and ticket is not assigned :param int ticket_id: The id of the Cerberus `abuse.models.Ticket` """ try: ticket = Ticket.objects.get(id=ticket_id) except (AttributeError, ObjectDoesNotExist, ValueError): Logger.error(unicode('Ticket %d cannot be found in DB. Skipping...' % (ticket_id))) return if not _check_timeout_ticket_conformance(ticket): return action = ImplementationFactory.instance.get_singleton_of('ActionServiceBase').get_action_for_timeout(ticket) if not action: Logger.error(unicode('Ticket %d service %s: action not found, exiting ...' % (ticket_id, ticket.service.componentType))) return # Maybe customer fixed, closing ticket if ticket.category.name.lower() == 'phishing' and phishing.is_all_down_for_ticket(ticket): Logger.info(unicode('All items are down for ticket %d, closing ticket' % (ticket_id))) close_ticket(ticket, reason=settings.CODENAMES['fixed_customer'], service_blocked=False) return # Getting ip for action ip_addr = _get_ip_for_action(ticket) if not ip_addr: Logger.error(unicode('Error while getting IP for action, exiting')) ticket.status = ticket.previousStatus ticket.status = 'ActionError' database.log_action_on_ticket( ticket=ticket, action='change_status', previous_value=ticket.previousStatus, new_value=ticket.status ) comment = Comment.objects.create(user=BOT_USER, comment='None or multiple ip addresses for this ticket') TicketComment.objects.create(ticket=ticket, comment=comment) database.log_action_on_ticket( ticket=ticket, action='add_comment' ) ticket.save() return # Apply action service_action_job = _apply_timeout_action(ticket, ip_addr, action) if not service_action_job.result: Logger.debug(unicode('Error while executing service action, exiting')) return Logger.info(unicode('All done, sending close notification to provider(s)')) ticket = Ticket.objects.get(id=ticket.id) # Closing ticket close_ticket(ticket, reason=settings.CODENAMES['fixed'], service_blocked=True)
def feedback_to_phishing_service(screenshot_id=None, feedback=None): """ Post phishing feedback for ML and scoring enhancement to Phishing Service :param str screenshot_id: The uuid of the screenshot_id :param bool feedback: Yes or not it's a phishing url """ ImplementationFactory.instance.get_singleton_of('PhishingServiceBase').post_feedback(screenshot_id, feedback) Logger.debug(unicode('Feedback %s sent for %s' % (feedback, screenshot_id)))
def mass_contact(ip_address=None, category=None, campaign_name=None, email_subject=None, email_body=None, user_id=None): """ Try to identify customer based on `ip_address`, creates Cerberus ticket then send email to customer and finally close ticket. The use case is: a trusted provider sent you a list of vulnerable DNS servers (DrDOS amp) for example. To prevent abuse on your network, you notify customer of this vulnerability. :param str ip_address: The IP address :param str category: The category of the abuse :param str campaign_name: The name if the "mass-conctact" campaign :param str email_subject: The subject of the email to send to defendant :param str email_body: The body of the email to send to defendant :param int user_id: The id of the Cerberus `abuse.models.User` who created the campaign """ # Check params _, _, _, values = inspect.getargvalues(inspect.currentframe()) if not all(values.values()): Logger.error(unicode('invalid parameters submitted %s' % str(values))) return try: validate_ipv46_address(ip_address) except (TypeError, ValidationError): Logger.error(unicode('invalid ip addresses submitted')) return # Get Django model objects try: category = Category.objects.get(name=category) user = User.objects.get(id=user_id) except (AttributeError, ObjectDoesNotExist, TypeError): Logger.error(unicode('invalid user or category')) return # Identify service for ip_address try: services = ImplementationFactory.instance.get_singleton_of('CustomerDaoBase').get_services_from_items(ips=[ip_address]) schema.valid_adapter_response('CustomerDaoBase', 'get_services_from_items', services) except CustomerDaoException as ex: Logger.error(unicode('Exception while identifying defendants for ip %s -> %s ' % (ip_address, str(ex)))) raise CustomerDaoException(ex) # Create report/ticket if services: Logger.debug(unicode('creating report/ticket for ip address %s' % (ip_address))) with pglocks.advisory_lock('cerberus_lock'): __create_contact_tickets(services, campaign_name, ip_address, category, email_subject, email_body, user) return True else: Logger.debug(unicode('no service found for ip address %s' % (ip_address))) return False
def __index_report_to_searchservice(parsed_email, filename, reports_id): """ Index a report to the SearchService """ try: Logger.debug(unicode('Pushing email %s document to SearchService' % (filename))) ImplementationFactory.instance.get_singleton_of('SearchServiceBase').index_email( parsed_email, filename, reports_id ) except SearchServiceException as ex: # Not fatal => don't stop current routine Logger.error(unicode('Unable to index mail %s in SearchService -> %s' % (filename, ex)))
def _update_ticket_if_answer(ticket, category, recipient, abuse_report, filename): """ If the email is an answer to a cerberus ticket: - update ticket status - cancel all pending ServiceAction jobs and ticket.timeout jobs - append response to ticket's email thread - save attachments :param `abuse.models.Ticket` ticket: A Cerberus `abuse.models.Ticket` instance :param str category: The category of the answer ('Defendant', 'Plaintiff' or 'Other) :param str recipient: The recipient of the answer :param `worker.parsing.parser.ParsedEmail` abuse_report: The ParsedEmail :param str filename: The filename of the email """ Logger.debug( unicode('New %s answer from %s for ticket %s' % (category, abuse_report.provider, ticket.id)), extra={ 'from': abuse_report.provider, 'action': 'new answer', 'hash': filename, 'ticket': ticket.id, } ) database.log_action_on_ticket( ticket=ticket, action='receive_email', email=abuse_report.provider ) try: if ticket.treatedBy.operator.role.modelsAuthorizations['ticket'].get('unassignedOnAnswer'): ticket.treatedBy = None except (AttributeError, KeyError, ObjectDoesNotExist, ValueError): pass if abuse_report.attachments: _save_attachments( filename, abuse_report.attachments, tickets=[ticket], ) for workflow in TicketAnswerWorkflowFactory.instance.registered_instances: if workflow.identify(ticket, abuse_report, recipient, category) and workflow.apply(ticket, abuse_report, recipient, category): Logger.debug(unicode('Specific workflow %s applied' % (str(workflow.__class__.__name__)))) return
def _genereates_oncreate_kpi(ticket): """ Kpi on ticket creation """ Logger.debug( unicode('new ticket %d' % (ticket.id)), extra={ 'ticket': ticket.id, 'action': 'new ticket', 'public_id': ticket.publicId } ) try: ImplementationFactory.instance.get_singleton_of('KPIServiceBase').new_ticket(ticket) except KPIServiceException as ex: Logger.error(unicode('Error while pushing KPI - %s' % (ex)))
def set_ticket_higher_priority(ticket): """ Set `abuse.models.Ticket` higher priority available through it's `abuse.models.Report`'s `abuse.models.Provider` """ defendant = Defendant.objects.get(customerId=ticket.defendant.customerId) if defendant.details.creationDate >= datetime.now() - timedelta(days=30): ticket.priority = sorted(PRIORITY_LEVEL.items(), key=operator.itemgetter(1))[1][0] # High ticket.save() return priorities = list(set(ticket.reportTicket.all().values_list('provider__priority', flat=True))) for priority, _ in sorted(PRIORITY_LEVEL.items(), key=operator.itemgetter(1)): if priority in priorities: Logger.debug(unicode('set priority %s to ticket %d' % (priority, ticket.id))) ticket.priority = priority ticket.save() return
def log_new_report(report): """ Log report creation """ Logger.debug( unicode('New report %d' % (report.id)), extra={ 'from': report.provider.email, 'action': 'new report', 'report': report.id, } ) if ImplementationFactory.instance.is_implemented('KPIServiceBase'): try: ImplementationFactory.instance.get_singleton_of('KPIServiceBase').new_report(report) except KPIServiceException as ex: Logger.error(unicode('Error while pushing KPI - %s' % (ex)))
def __update_history(defendant, categories, now): """ Update history for given defendant """ Logger.debug(str('Updating history for defendant %s' % (defendant['customerId']))) for category in categories: reports = Report.objects.filter(~Q(status='Archived'), category=category, defendant_id=defendant['id']).count() tickets = Ticket.objects.filter(~Q(status='Closed'), category=category, defendant_id=defendant['id']).count() stats = __get_last_stats(defendant['id'], category) if not len(stats) or reports != stats[0].reports or tickets != stats[0].tickets: Stat.objects.create( defendant_id=defendant['id'], category_id=category, tickets=tickets, reports=reports, date=now, )
def __update_item_status(item, country='FR'): """ Update item status """ if item.itemType != 'URL': return try: Logger.debug(unicode('Checking status for url %s' % (item.rawItem,))) response = ImplementationFactory.instance.get_singleton_of('PhishingServiceBase').ping_url(item.rawItem, country=country) database.insert_url_status( item, response.direct_status, response.proxied_status, response.http_code, response.score, response.is_phishing, ) except PhishingServiceException: pass
def _reinject_validated(report, user): trusted = True ticket = None if all((report.defendant, report.category, report.service)): msg = 'Looking for opened ticket for (%s, %s, %s)' Logger.debug(unicode(msg % (report.defendant.customerId, report.category.name, report.service.name))) ticket = database.search_ticket(report.defendant, report.category, report.service) # Checking specific processing workflow for workflow in ReportWorkflowFactory.instance.registered_instances: if workflow.identify(report, ticket, is_trusted=trusted) and workflow.apply(report, ticket, trusted, False): Logger.debug(unicode('Specific workflow %s applied' % (str(workflow.__class__.__name__)))) return # Create ticket if trusted new_ticket = False if not ticket: ticket = database.create_ticket(report.defendant, report.category, report.service, priority=report.provider.priority) new_ticket = True if ticket: report.ticket = Ticket.objects.get(id=ticket.id) report.status = 'Attached' report.save() database.set_ticket_higher_priority(report.ticket) database.log_action_on_ticket( ticket=ticket, action='attach_report', report=report, new_ticket=new_ticket, user=user ) try: __send_ack(report, lang='EN') except MailerServiceException as ex: raise MailerServiceException(ex)
def _check_auto_unassignation(ticket): history = ticket.ticketHistory.filter(actionType='ChangeStatus').order_by('-date').values_list('ticketStatus', flat=True)[:3] try: unassigned_on_multiple_alarm = ticket.treatedBy.operator.role.modelsAuthorizations['ticket']['unassignedOnMultipleAlarm'] if unassigned_on_multiple_alarm and len(history) == 3 and all([STATUS_SEQUENCE[i] == history[i] for i in xrange(3)]): database.log_action_on_ticket( ticket=ticket, action='change_treatedby', previous_value=ticket.treatedBy ) database.log_action_on_ticket( ticket=ticket, action='update_property', property='escalated', previous_value=ticket.escalated, new_value=True, ) ticket.treatedBy = None ticket.escalated = True Logger.debug(unicode('Unassigning ticket %d because of operator role configuration' % (ticket.id))) except (AttributeError, KeyError, ObjectDoesNotExist, ValueError): pass
def apply(self, report, ticket, is_trusted=False, no_phishtocheck=False): """ Apply specific workflow on given `abuse.models.Report` :param `abuse.models.Report` report: A Cerberus report instance :param `abuse.models.Ticket` ticket: A Cerberus ticket instance :param bool is_trusted: If the report is trusted :param bool no_phishtocheck: if the report does not need PhishToCheck :return: If the workflow is applied :rtype: bool """ from worker import phishing is_there_some_urls = report.reportItemRelatedReport.filter(itemType='URL').exists() all_down = phishing.check_if_all_down(report=report) # Archived report immediatly if all_down: phishing.close_because_all_down(report=report) Logger.debug(unicode('All down phishing workflow applied')) return True if no_phishtocheck: # Just pass return True # All items are clearly phishing ? new_ticket = False if all((is_trusted, is_there_some_urls, are_all_items_phishing(report))): if not ticket: ticket = _create_ticket(report) new_ticket = True _attach_report_to_ticket(report, ticket, new_ticket) phishing.block_url_and_mail(ticket_id=ticket, report_id=report) Logger.debug(unicode('Clearly phishing workflow applied')) return True # Report has to be manually checked if not report.provider.apiKey: # Means it is not a trusted phishing provider report.status = 'PhishToCheck' report.save() utils.push_notification({ 'type': 'new phishToCheck', 'id': report.id, 'message': 'New PhishToCheck report %d' % (report.id), }) Logger.debug(unicode('PhishToCheck workflow applied')) return True else: if not ticket and is_trusted: # Create ticket ticket = _create_ticket(report) new_ticket = True if ticket: _attach_report_to_ticket(report, ticket, new_ticket) if is_there_some_urls: # Block urls phishing.block_url_and_mail(ticket_id=ticket, report_id=report) Logger.debug(unicode('Trusted phishing provider workflow applied')) return True
def __create_contact_tickets(services, campaign_name, ip_address, category, email_subject, email_body, user): # Create fake report report_subject = 'Campaign %s for ip %s' % (campaign_name, ip_address) report_body = 'Campaign: %s\nIP Address: %s\n' % (campaign_name, ip_address) filename = hashlib.sha256(report_body.encode('utf-8')).hexdigest() __save_email(filename, report_body) for data in services: # For identified (service, defendant, items) tuple actions = [] # Create report report = Report.objects.create(**{ 'provider': database.get_or_create_provider('mass_contact'), 'receivedDate': datetime.now(), 'subject': report_subject, 'body': report_body, 'category': category, 'filename': filename, 'status': 'Archived', 'defendant': database.get_or_create_defendant(data['defendant']), 'service': database.get_or_create_service(data['service']), }) database.log_new_report(report) # Create item item_dict = {'itemType': 'IP', 'report_id': report.id, 'rawItem': ip_address} item_dict.update(utils.get_reverses_for_item(ip_address, nature='IP')) ReportItem.objects.create(**item_dict) # Create ticket ticket = database.create_ticket( report.defendant, report.category, report.service, priority=report.provider.priority, attach_new=False, ) database.add_mass_contact_tag(ticket, campaign_name) actions.append({'ticket': ticket, 'action': 'create_masscontact', 'campaign_name': campaign_name}) actions.append({'ticket': ticket, 'action': 'change_treatedby', 'new_value': user.username}) report.ticket = ticket report.save() Logger.debug(unicode( 'ticket %d successfully created for (%s, %s)' % (ticket.id, report.defendant.customerId, report.service.name) )) # Send email to defendant __send_mass_contact_email(ticket, email_subject, email_body) actions.append({'ticket': ticket, 'action': 'send_email', 'email': report.defendant.details.email}) # Close ticket/report ticket.resolution = Resolution.objects.get(codename=settings.CODENAMES['fixed_customer']) ticket.previousStatus = ticket.status ticket.status = 'Closed' ticket.save() actions.append({ 'ticket': ticket, 'action': 'change_status', 'previous_value': ticket.previousStatus, 'new_value': ticket.status, 'close_reason': ticket.resolution.codename }) for action in actions: database.log_action_on_ticket(**action)
def create_from_email(email_content=None, filename=None, lang='EN', send_ack=False): """ Create Cerberus report(s) based on email content If send_ack is True and report is attached to a ticket, then an acknowledgement is sent to the email provider. :param str email_content: The raw email content :param str filename: The name of the raw email file :param str lang: Langage to use if send_ack is True :param bool send_ack: If an acknowledgment have to be sent to provider :raises CustomerDaoException: if exception while identifying defendants from items :raises MailerServiceException: if exception while updating ticket's emails :raises StorageServiceException: if exception while accessing storage """ # This function use a lock/commit_on_succes on db when creating reports # # Huge blocks of code are under transaction because it's important to # rollback if ANYTHING goes wrong in the report creation workflow. # # Concurrent transactions (with multiple workers), on defendant/service creation # can result in unconsistent data, So a pg_lock is used. # # `abuse.models.Defendant` and `abuse.models.Service` HAVE to be unique. if not email_content: Logger.error(unicode('Missing email content')) return if not filename: # Worker have to push email to Storage Service filename = hashlib.sha256(email_content).hexdigest() __save_email(filename, email_content) # Parse email content abuse_report = Parser.parse(email_content) Logger.debug(unicode('New email from %s' % (abuse_report.provider)), extra={'from': abuse_report.provider, 'action': 'new email'}) # Check if provider is not blacklisted if abuse_report.provider in settings.PARSING['providers_to_ignore']: Logger.error(unicode('Provider %s is blacklisted, skipping ...' % (abuse_report.provider))) return # Check if it's an answer to a ticket(s) tickets = ImplementationFactory.instance.get_singleton_of('MailerServiceBase').is_email_ticket_answer(abuse_report) if tickets: for ticket, category, recipient in tickets: if all((ticket, category, recipient)) and not ticket.locked: # OK it's an anwser, updating ticket and exiting _update_ticket_if_answer(ticket, category, recipient, abuse_report, filename) return # Check if items are linked to customer and get corresponding services try: services = ImplementationFactory.instance.get_singleton_of('CustomerDaoBase').get_services_from_items( urls=abuse_report.urls, ips=abuse_report.ips, fqdn=abuse_report.fqdn ) schema.valid_adapter_response('CustomerDaoBase', 'get_services_from_items', services) except CustomerDaoException as ex: Logger.error(unicode('Exception while identifying defendants from items for mail %s -> %s ' % (filename, str(ex)))) raise CustomerDaoException(ex) # Create report(s) with identified services if not services: created_reports = [__create_without_services(abuse_report, filename)] else: with pglocks.advisory_lock('cerberus_lock'): __create_defendants_and_services(services) created_reports = __create_with_services(abuse_report, filename, services) # Upload attachments if abuse_report.attachments: _save_attachments(filename, abuse_report.attachments, reports=created_reports) # Send acknowledgement to provider (only if send_ack = True and report is attached to a ticket) for report in created_reports: if send_ack and report.ticket: try: __send_ack(report, lang=lang) except MailerServiceException as ex: raise MailerServiceException(ex) # Index to SearchService if ImplementationFactory.instance.is_implemented('SearchServiceBase'): __index_report_to_searchservice(abuse_report, filename, [rep.id for rep in created_reports]) Logger.info(unicode('All done successfully for email %s' % (filename)))
def __create_with_services(abuse_report, filename, services): """ Create report(s), ticket(s), item(s), defendant(s), service(s), attachment(s) in Cerberus :param `ParsedEmail` abuse_report: The `ParsedEmail` :param str filename: The filename of the email :param dict services: The identified service(s) (see adapters/dao/customer/abstract.py) :rtype: list :returns: The list of Cerberus `abuse.models.Report` created """ created_reports = [] for data in services: # For identified (service, defendant, items) tuple report = __create_without_services(abuse_report, filename) created_reports.append(report) report.defendant = data['defendant'] report.service = data['service'] report.save() if report.status == 'Archived': # because autoarchive tag continue _, attach_only, no_phishtocheck = __get_attributes_based_on_tags(report, abuse_report.recipients) __insert_items(report.id, data['items']) # The provider or the way we received the report trusted = True if report.provider.trusted or abuse_report.trusted else False # Looking for existing open ticket for same (service, defendant, category) ticket = None if all((report.defendant, report.category, report.service)): msg = 'Looking for opened ticket for (%s, %s, %s)' Logger.debug(unicode(msg % (report.defendant.customerId, report.category.name, report.service.name))) ticket = database.search_ticket(report.defendant, report.category, report.service) # Checking specific processing workflow is_workflow_applied = False for workflow in ReportWorkflowFactory.instance.registered_instances: if workflow.identify(report, ticket, is_trusted=trusted): is_workflow_applied = workflow.apply(report, ticket, trusted, no_phishtocheck) if is_workflow_applied: database.set_report_specificworkflow_tag(report, str(workflow.__class__.__name__)) Logger.debug(unicode('Specific workflow %s applied' % str(workflow.__class__.__name__))) break if is_workflow_applied: continue # If attach report only and no ticket found, continue if not ticket and attach_only: report.status = 'Archived' report.save() continue # Create ticket if trusted new_ticket = False if not ticket and trusted: ticket = database.create_ticket(report.defendant, report.category, report.service, priority=report.provider.priority) new_ticket = True if ticket: report.ticket = Ticket.objects.get(id=ticket.id) report.status = 'Attached' report.save() database.set_ticket_higher_priority(report.ticket) database.log_action_on_ticket( ticket=ticket, action='attach_report', report=report, new_ticket=new_ticket ) return created_reports