def _apply_timeout_action(ticket, ip_addr, action):
Logger.info(unicode('Executing action %s for ticket %d' % (action.name, ticket.id)))
ticket.action = action
database.log_action_on_ticket(
ticket=ticket,
action='set_action',
action_name=action.name
)
ticket.save()
async_job = utils.scheduler.schedule(
scheduled_time=datetime.utcnow() + timedelta(seconds=3),
func='action.apply_action',
kwargs={
'ticket_id': ticket.id,
'action_id': action.id,
'ip_addr': ip_addr,
'user_id': BOT_USER.id,
},
interval=1,
repeat=1,
result_ttl=500,
timeout=3600,
)
Logger.info(unicode('Task has %s job id' % (async_job.id)))
job = ServiceActionJob.objects.create(ip=ip_addr, action=action, asynchronousJobId=async_job.id, creationDate=datetime.now())
ticket.jobs.add(job)
while not async_job.is_finished:
sleep(5)
return async_job
def timeout(ticket_id=None):
"""
If ticket timeout , apply action on service (if defendant not internal/VIP) and ticket is not assigned
:param int ticket_id: The id of the Cerberus `abuse.models.Ticket`
"""
try:
ticket = Ticket.objects.get(id=ticket_id)
except (AttributeError, ObjectDoesNotExist, ValueError):
Logger.error(unicode('Ticket %d cannot be found in DB. Skipping...' % (ticket_id)))
return
if not _check_timeout_ticket_conformance(ticket):
return
action = ImplementationFactory.instance.get_singleton_of('ActionServiceBase').get_action_for_timeout(ticket)
if not action:
Logger.error(unicode('Ticket %d service %s: action not found, exiting ...' % (ticket_id, ticket.service.componentType)))
return
# Maybe customer fixed, closing ticket
if ticket.category.name.lower() == 'phishing' and phishing.is_all_down_for_ticket(ticket):
Logger.info(unicode('All items are down for ticket %d, closing ticket' % (ticket_id)))
close_ticket(ticket, reason=settings.CODENAMES['fixed_customer'], service_blocked=False)
return
# Getting ip for action
ip_addr = _get_ip_for_action(ticket)
if not ip_addr:
Logger.error(unicode('Error while getting IP for action, exiting'))
ticket.status = ticket.previousStatus
ticket.status = 'ActionError'
database.log_action_on_ticket(
ticket=ticket,
action='change_status',
previous_value=ticket.previousStatus,
new_value=ticket.status
)
comment = Comment.objects.create(user=BOT_USER, comment='None or multiple ip addresses for this ticket')
TicketComment.objects.create(ticket=ticket, comment=comment)
database.log_action_on_ticket(
ticket=ticket,
action='add_comment'
)
ticket.save()
return
# Apply action
service_action_job = _apply_timeout_action(ticket, ip_addr, action)
if not service_action_job.result:
Logger.debug(unicode('Error while executing service action, exiting'))
return
Logger.info(unicode('All done, sending close notification to provider(s)'))
ticket = Ticket.objects.get(id=ticket.id)
# Closing ticket
close_ticket(ticket, reason=settings.CODENAMES['fixed'], service_blocked=True)
def __save_email(filename, email):
"""
Push email storage service
:param str filename: The filename of the email
:param str email: The content of the email
"""
with ImplementationFactory.instance.get_instance_of('StorageServiceBase', settings.GENERAL_CONFIG['email_storage_dir']) as cnx:
cnx.write(filename, email)
Logger.info(unicode('Email %s pushed to Storage Service' % (filename)))
def apply_if_no_reply(ticket_id=None, action_id=None, ip_addr=None, resolution_id=None, user_id=None, close=False):
"""
Action if no reply from customer
:param int ticket_id: The id of the Cerberus `Ticket`
:param int action_id: The id of the Cerberus `ServiceAction`
:param str ip_addr: The ip address
:param int resolution_id: The id of the Cerberus `Resolution`
:param int user_id: The id of the Cerberus `User`
:param bool close: If the ticket has to be closed after action
"""
# Checking conformance
if not all((ticket_id, action_id, user_id)):
Logger.error(unicode(
'Invalid parameters [ticket_id=%s, action_id=%s, user_id=%s]' % (ticket_id, action_id, user_id)
))
return
if close and not resolution_id:
Logger.error(unicode('Close requested but no resolution submitted'))
return
if resolution_id and not Resolution.objects.filter(id=resolution_id).exists():
Logger.error(unicode('Ticket resolution %d not found, Skipping...' % (resolution_id)))
return
# Apply action
applied = apply_action(ticket_id, action_id, ip_addr, user_id)
if not applied:
return
# Updating ticket info
ticket = Ticket.objects.get(id=ticket_id)
user = User.objects.get(id=user_id)
ticket.previousStatus = ticket.status
ticket.snoozeDuration = None
ticket.snoozeStart = None
close_reason = None
if close and resolution_id:
__close_ticket(ticket, resolution_id)
close_reason = ticket.resolution.codename
else:
ticket.status = 'Alarm'
ticket.save()
database.log_action_on_ticket(
ticket=ticket,
action='change_status',
user=user,
previous_value=ticket.previousStatus,
new_value=ticket.status,
close_reason=close_reason
)
Logger.info(unicode('Ticket %d processed. Next !' % (ticket_id)))
def close_ticket(ticket, reason=settings.CODENAMES['fixed_customer'], service_blocked=False):
"""
Close ticket and add autoclosed Tag
"""
# Send "case closed" email to already contacted Provider(s)
providers_emails = ContactedProvider.objects.filter(ticket_id=ticket.id).values_list('provider__email', flat=True).distinct()
for email in providers_emails:
try:
validate_email(email.strip())
_send_email(ticket, email, settings.CODENAMES['case_closed'])
ticket.save()
Logger.info(unicode('Mail sent to provider %s' % (email)))
except (AttributeError, TypeError, ValueError, ValidationError):
pass
if service_blocked:
template = settings.CODENAMES['service_blocked']
else:
template = settings.CODENAMES['ticket_closed']
# Send "ticket closed" email to defendant
_send_email(ticket, ticket.defendant.details.email, template, lang=ticket.defendant.details.lang)
if ticket.mailerId:
ImplementationFactory.instance.get_singleton_of('MailerServiceBase').close_thread(ticket)
resolution = Resolution.objects.get(codename=reason)
ticket.resolution = resolution
ticket.previousStatus = ticket.status
ticket.status = 'Closed'
ticket.reportTicket.all().update(status='Archived')
tag_name = settings.TAGS['phishing_autoclosed'] if ticket.category.name.lower() == 'phishing' else settings.TAGS['copyright_autoclosed']
ticket.tags.add(Tag.objects.get(name=tag_name))
ticket.save()
database.log_action_on_ticket(
ticket=ticket,
action='change_status',
previous_value=ticket.previousStatus,
new_value=ticket.status,
close_reason=ticket.resolution.codename
)
database.log_action_on_ticket(
ticket=ticket,
action='add_tag',
tag_name=tag_name
)
def block_url_and_mail(ticket_id=None, report_id=None):
"""
Block url with PhishingService and send mail to defendant
:param int ticket_id: The id of the Cerberus `abuse.models.Ticket`
:param int report_id: The id of the Cerberus `abuse.models.Report`
"""
if not isinstance(ticket_id, Ticket):
try:
ticket = Ticket.objects.get(id=ticket_id)
if not ticket.defendant or not ticket.service:
Logger.error(unicode('Ticket %d has no defendant/service' % (ticket_id)))
return
except (ObjectDoesNotExist, ValueError):
Logger.error(unicode('Ticket %d cannot be found in DB. Skipping...' % (ticket_id)))
return
else:
ticket = ticket_id
if not isinstance(report_id, Report):
try:
report = Report.objects.get(id=report_id)
except (ObjectDoesNotExist, ValueError):
Logger.error(unicode('Report %d cannot be found in DB. Skipping...' % (report_id)))
return
else:
report = report_id
for item in report.reportItemRelatedReport.filter(itemType='URL'):
ImplementationFactory.instance.get_singleton_of('PhishingServiceBase').block_url(item.rawItem, item.report)
database.add_phishing_blocked_tag(report)
__send_email(ticket, report.defendant.details.email, settings.CODENAMES['phishing_blocked'], report.defendant.details.lang)
ticket = Ticket.objects.get(id=ticket.id)
ticket_snooze = settings.GENERAL_CONFIG['phishing']['wait']
if not ticket.status == 'WaitingAnswer' and not ticket.snoozeDuration and not ticket.snoozeStart:
ticket.previousStatus = ticket.status
ticket.status = 'WaitingAnswer'
ticket.snoozeDuration = ticket_snooze
ticket.snoozeStart = datetime.now()
ticket.save()
Logger.info(unicode('Ticket %d now with status WaitingAnswer for %d' % (ticket.id, ticket_snooze)))
def create_ticket_with_threshold():
"""
Automatically creates ticket if there are more than `abuse.models.ReportThreshold.threshold`
new reports created during `abuse.models.ReportThreshold.interval` (days) for same (category/defendant/service)
"""
log_msg = 'threshold : Checking report threshold for category %s, threshold %d, interval %d days'
for thres in ReportThreshold.objects.all():
Logger.info(unicode(log_msg % (thres.category.name, thres.threshold, thres.interval)))
reports = __get_threshold_reports(thres.category, thres.interval)
reports = Counter(reports)
for data, count in reports.iteritems():
nb_tickets = Ticket.objects.filter(
~Q(status='Closed'),
defendant__customerId=data[0],
service__id=data[1],
).count()
if count >= thres.threshold and not nb_tickets:
ticket = __create_threshold_ticket(data, thres)
Logger.info(unicode('threshold: tuple %s match, ticket %s has been created' % (str(data), ticket.id)))
def _create_closed_ticket(report, user):
report.ticket = common.create_ticket(report, attach_new=False)
report.save()
# Add temp proof(s) for mail content
temp_proofs = []
if not report.ticket.proof.count():
temp_proofs = common.get_temp_proofs(report.ticket)
# Send email to Provider
try:
validate_email(report.provider.email.strip())
Logger.info(unicode('Sending email to provider'))
common.send_email(report.ticket, [report.provider.email], settings.CODENAMES['not_managed_ip'])
report.ticket.save()
Logger.info(unicode('Mail sent to provider'))
ImplementationFactory.instance.get_singleton_of('MailerServiceBase').close_thread(report.ticket)
# Delete temp proof(s)
for proof in temp_proofs:
Proof.objects.filter(id=proof.id).delete()
except (AttributeError, TypeError, ValueError, ValidationError):
pass
common.close_ticket(report, resolution_codename=settings.CODENAMES['invalid'], user=user)
Logger.info(unicode('Ticket %d and report %d closed' % (report.ticket.id, report.id)))
def apply_then_close(ticket_id=None, action_id=None, ip_addr=None, resolution_id=None, user_id=None):
"""
Action on service then close
:param int ticket_id: The id of the Cerberus `Ticket`
:param int action_id: The id of the Cerberus `ServiceAction`
:param str ip_addr: The ip address
:param int resolution_id: The id of the Cerberus `Resolution`
:param int user_id: The id of the Cerberus `User`
"""
# Checking conformance
if not all((ticket_id, action_id, resolution_id, user_id)):
msg = 'Invalid parameters submitted [ticket_id=%d, action_id=%s, resolution_id=%s, user_id=%s]'
Logger.error(unicode(msg % (ticket_id, action_id, resolution_id, user_id)))
return
# Apply action
applied = apply_action(ticket_id, action_id, ip_addr, user_id)
if not applied:
return
# Closing ticket and updating ticket info
ticket = Ticket.objects.get(id=ticket_id)
user = User.objects.get(id=user_id)
__close_ticket(ticket, resolution_id)
database.log_action_on_ticket(
ticket=ticket,
action='change_status',
user=user,
previous_value=ticket.previousStatus,
new_value=ticket.status,
close_reason=ticket.resolution.codename
)
ticket.resolution_id = resolution_id
ticket.save()
Logger.info(unicode('Ticket %d processed. Next !' % (ticket_id)))
def archive_if_timeout(report_id=None):
"""
Archived report if not attached
:param int report_id: The report id
"""
if not report_id:
Logger.error(unicode('Invalid parameters submitted [report_id=%d]' % (report_id)))
return
try:
report = Report.objects.get(id=report_id)
except (ObjectDoesNotExist, ValueError):
Logger.error(unicode('Report %d cannot be found in DB. Skipping...' % (report_id)))
return
if report.status != 'New':
Logger.error(unicode('Report %d not New, status : %s , Skipping ...' % (report_id, report.status)))
return
report.ticket = None
report.status = 'Archived'
report.save()
Logger.info(unicode('Report %d successfully archived' % (report_id)))
def check_mass_contact_result(result_campaign_id=None, jobs=None):
"""
Check "mass-contact" campaign jobs's result
:param int result_campaign_id: The id of the `abuse.models.MassContactResult`
:param list jobs: The list of associated Python-Rq jobs id
"""
# Check params
_, _, _, values = inspect.getargvalues(inspect.currentframe())
if not all(values.values()) or not isinstance(jobs, list):
Logger.error(unicode('invalid parameters submitted %s' % str(values)))
return
if not isinstance(result_campaign_id, MassContactResult):
try:
campaign_result = MassContactResult.objects.get(id=result_campaign_id)
except (AttributeError, ObjectDoesNotExist, TypeError, ValueError):
Logger.error(unicode('MassContactResult %d cannot be found in DB. Skipping...' % (result_campaign_id)))
return
result = []
for job_id in jobs:
job = utils.default_queue.fetch_job(job_id)
if not job:
continue
while job.status.lower() == 'queued':
sleep(0.5)
result.append(job.result)
count = Counter(result)
campaign_result.state = 'Done'
campaign_result.matchingCount = count[True]
campaign_result.notMatchingCount = count[False]
campaign_result.failedCount = count[None]
campaign_result.save()
Logger.info(unicode('MassContact campaign %d finished' % (campaign_result.campaign.id)))
def close_because_all_down(report=None, denied_by=None):
"""
Create and close a ticket when all report's items are down
:param `abuse.models.Report` report: A Cerberus `abuse.models.Report` instance
:param int denied_by: The id of the `abuse.models.User` who takes the decision to close the ticket
"""
if not isinstance(report, Report):
try:
report = Report.objects.get(id=report)
except (AttributeError, ObjectDoesNotExist, TypeError, ValueError):
Logger.error(unicode('Report %d cannot be found in DB. Skipping...' % (report)))
return
if not report.ticket:
report.ticket = common.create_ticket(report, denied_by)
report.save()
# Add temp proof(s) for mail content
temp_proofs = []
if not report.ticket.proof.count():
temp_proofs = common.get_temp_proofs(report.ticket)
# Send email to Provider
try:
validate_email(report.provider.email.strip())
Logger.info(unicode('Sending email to provider'))
__send_email(report.ticket, report.provider.email, settings.CODENAMES['no_more_content'])
report.ticket.save()
Logger.info(unicode('Mail sent to provider'))
ImplementationFactory.instance.get_singleton_of('MailerServiceBase').close_thread(report.ticket)
# Delete temp proof(s)
for proof in temp_proofs:
Proof.objects.filter(id=proof.id).delete()
except (AttributeError, TypeError, ValueError, ValidationError):
pass
# Closing ticket and add tags
common.close_ticket(report, resolution_codename=settings.CODENAMES['no_more_content'])
report.ticket.tags.remove(Tag.objects.get(name=settings.TAGS['phishing_autoreopen']))
report.ticket.tags.add(Tag.objects.get(name=settings.TAGS['phishing_autoclosed']))
Logger.info(unicode('Ticket %d and report %d closed' % (report.ticket.id, report.id)))
def create_from_email(email_content=None, filename=None, lang='EN', send_ack=False):
"""
Create Cerberus report(s) based on email content
If send_ack is True and report is attached to a ticket,
then an acknowledgement is sent to the email provider.
:param str email_content: The raw email content
:param str filename: The name of the raw email file
:param str lang: Langage to use if send_ack is True
:param bool send_ack: If an acknowledgment have to be sent to provider
:raises CustomerDaoException: if exception while identifying defendants from items
:raises MailerServiceException: if exception while updating ticket's emails
:raises StorageServiceException: if exception while accessing storage
"""
# This function use a lock/commit_on_succes on db when creating reports
#
# Huge blocks of code are under transaction because it's important to
# rollback if ANYTHING goes wrong in the report creation workflow.
#
# Concurrent transactions (with multiple workers), on defendant/service creation
# can result in unconsistent data, So a pg_lock is used.
#
# `abuse.models.Defendant` and `abuse.models.Service` HAVE to be unique.
if not email_content:
Logger.error(unicode('Missing email content'))
return
if not filename: # Worker have to push email to Storage Service
filename = hashlib.sha256(email_content).hexdigest()
__save_email(filename, email_content)
# Parse email content
abuse_report = Parser.parse(email_content)
Logger.debug(unicode('New email from %s' % (abuse_report.provider)), extra={'from': abuse_report.provider, 'action': 'new email'})
# Check if provider is not blacklisted
if abuse_report.provider in settings.PARSING['providers_to_ignore']:
Logger.error(unicode('Provider %s is blacklisted, skipping ...' % (abuse_report.provider)))
return
# Check if it's an answer to a ticket(s)
tickets = ImplementationFactory.instance.get_singleton_of('MailerServiceBase').is_email_ticket_answer(abuse_report)
if tickets:
for ticket, category, recipient in tickets:
if all((ticket, category, recipient)) and not ticket.locked: # OK it's an anwser, updating ticket and exiting
_update_ticket_if_answer(ticket, category, recipient, abuse_report, filename)
return
# Check if items are linked to customer and get corresponding services
try:
services = ImplementationFactory.instance.get_singleton_of('CustomerDaoBase').get_services_from_items(
urls=abuse_report.urls,
ips=abuse_report.ips,
fqdn=abuse_report.fqdn
)
schema.valid_adapter_response('CustomerDaoBase', 'get_services_from_items', services)
except CustomerDaoException as ex:
Logger.error(unicode('Exception while identifying defendants from items for mail %s -> %s ' % (filename, str(ex))))
raise CustomerDaoException(ex)
# Create report(s) with identified services
if not services:
created_reports = [__create_without_services(abuse_report, filename)]
else:
with pglocks.advisory_lock('cerberus_lock'):
__create_defendants_and_services(services)
created_reports = __create_with_services(abuse_report, filename, services)
# Upload attachments
if abuse_report.attachments:
_save_attachments(filename, abuse_report.attachments, reports=created_reports)
# Send acknowledgement to provider (only if send_ack = True and report is attached to a ticket)
for report in created_reports:
if send_ack and report.ticket:
try:
__send_ack(report, lang=lang)
except MailerServiceException as ex:
raise MailerServiceException(ex)
# Index to SearchService
if ImplementationFactory.instance.is_implemented('SearchServiceBase'):
__index_report_to_searchservice(abuse_report, filename, [rep.id for rep in created_reports])
Logger.info(unicode('All done successfully for email %s' % (filename)))
def apply_action(ticket_id=None, action_id=None, ip_addr=None, user_id=None):
"""
Apply given action on customer service
:param int ticket_id: The id of the Cerberus `Ticket`
:param int action_id: The id of the Cerberus `ServiceAction`
:param int user_id: The id of the Cerberus `User`
:rtype: bool
:returns: if action has been applied
"""
current_job = get_current_job()
# Checking conformance
if not all((ticket_id, action_id, user_id)):
msg = 'Invalid parameters submitted [ticket_id=%d, action_id=%s, user_id=%s]'
Logger.error(unicode(msg % (ticket_id, action_id, user_id)))
return False
# Fetching Django model object
Logger.info(unicode('Starting process ticket %d with params [%d]' % (ticket_id, action_id)))
try:
ticket = Ticket.objects.get(id=ticket_id)
user = User.objects.get(id=user_id)
except (ObjectDoesNotExist, ValueError):
Logger.error(unicode('Ticket %d or user %d cannot be found in DB. Skipping...' % (ticket_id, user_id)))
return False
if ticket.status in ['Closed', 'Answered']:
__cancel_by_status(ticket)
ticket.previousStatus = ticket.status
ticket.status = 'ActionError'
ticket.save()
database.log_action_on_ticket(
ticket=ticket,
action='change_status',
user=user,
previous_value=ticket.previousStatus,
new_value=ticket.status,
)
return False
# Call action service
try:
result = ImplementationFactory.instance.get_singleton_of(
'ActionServiceBase'
).apply_action_on_service(
ticket_id,
action_id,
ip_addr,
user.id
)
_update_job(current_job.id, todo_id=result.todo_id, status=result.status, comment=result.comment)
return True
except ActionServiceException as ex:
Logger.info(unicode('Service Action not apply for ticket %d' % (ticket_id)))
_update_job(current_job.id, status='actionError', comment=str(ex))
ticket.previousStatus = ticket.status
ticket.status = 'ActionError'
ticket.save()
database.log_action_on_ticket(
ticket=ticket,
action='change_status',
user=user,
previous_value=ticket.previousStatus,
new_value=ticket.status,
)
return False
