def test_file_name_from_response(self):
""" test file_name_from_response """
response = requests.Response()
response.headers[
'Content-Disposition'] = 'attachment; filename=2019-09-09-drone_brute_force-austria-geo.csv'
self.assertEqual(utils.file_name_from_response(response),
'2019-09-09-drone_brute_force-austria-geo.csv')
def process_message(self, uid, message):
erroneous = False # If errors occurred this will be set to true.
seen = False
for body in message.body['plain']:
match = re.search(self.parameters.url_regex, str(body.decode('utf-8') if isinstance(body, bytes) else body))
if match:
url = match.group()
# strip leading and trailing spaces, newlines and
# carriage returns
url = url.strip()
self.logger.info("Downloading report from %r.", url)
try:
resp = self.session.get(url=url)
except requests.exceptions.Timeout:
self.logger.error("Request timed out %i times in a row. " %
self.http_timeout_max_tries)
erroneous = True
# The download timed out too often, leave the Loop.
continue
if resp.status_code // 100 != 2:
self.logger.error('HTTP response status code was {}.'
''.format(resp.status_code))
erroneous = True
continue
if not resp.content:
self.logger.warning('Got empty reponse from server.')
else:
self.logger.info("Report downloaded.")
template = self.new_report()
template["feed.url"] = url
template["extra.email_subject"] = message.subject
template["extra.email_from"] = ','.join(x['email'] for x in message.sent_from)
template["extra.email_message_id"] = message.message_id
template["extra.file_name"] = file_name_from_response(resp)
for report in generate_reports(template, io.BytesIO(resp.content),
self.chunk_size,
self.chunk_replicate_header):
self.send_message(report)
seen = True
if not erroneous:
self.logger.info("Email report read.")
else:
if self.parameters.error_procedure == 'pass':
seen = True
else:
self.logger.error("Email report read with above errors, the report was not processed.")
return seen
def process(self):
RT = rt.Rt(self.parameters.uri, self.parameters.user,
self.parameters.password)
if not RT.login():
raise ValueError('Login failed.')
if self.not_older_than_type:
if self.not_older_than_type == 'relative':
self.not_older_than = datetime.now(
) - self.not_older_than_relative
kwargs = {'Created__gt': self.not_older_than.isoformat()}
self.logger.debug('Searching for tickets newer than %r.',
kwargs['Created__gt'])
else:
kwargs = {}
for parameter_name, rt_name in self.parameter_mapping.items():
parameter_value = getattr(self.parameters, parameter_name, None)
if parameter_value:
kwargs[rt_name] = parameter_value
query = RT.search(order='Created', **kwargs)
self.logger.info('%s results on search query.', len(query))
for ticket in query:
ticket_id = int(ticket['id'].split('/')[1])
self.logger.debug('Process ticket %s.', ticket_id)
content = 'attachment'
success = False
if self.parameters.attachment_regex:
for (att_id, att_name, _, _) in RT.get_attachments(ticket_id):
if re.search(self.parameters.attachment_regex, att_name):
self.logger.debug('Found attachment %s: %r.', att_id,
att_name)
success = True
content = 'attachment'
self.extract_files = self.extract_attachment
break
if not success and self.parameters.url_regex:
ticket = RT.get_history(ticket_id)[0]
created = ticket['Created']
urlmatch = re.search(self.parameters.url_regex,
ticket['Content'])
if urlmatch:
content = 'url'
self.extract_files = self.extract_download
url = urlmatch.group(0)
self.logger.debug('Matching URL found %r.', url)
success = True
if not success:
self.logger.info('No matching attachment or URL found.')
continue
report = self.new_report()
if content == 'attachment':
attachment = RT.get_attachment_content(ticket_id, att_id)
created = RT.get_attachment(ticket_id, att_id)['Created']
raw = attachment
else:
resp = self.session.get(url=url)
response_code_class = resp.status_code // 100
if response_code_class != 2:
self.logger.error(
'HTTP response status code for %r was %s. Skipping ticket %d.',
url, resp.status_code, ticket_id)
if response_code_class == 4:
self.logger.debug('Server response: %r.', resp.text)
if self.parameters.set_status:
RT.edit_ticket(ticket_id,
status=self.parameters.set_status)
if self.parameters.take_ticket:
try:
RT.take(ticket_id)
except rt.BadRequest:
self.logger.exception(
"Could not take ticket %s.", ticket_id)
else:
self.logger.info('Skipping now.')
continue
self.logger.info("Report #%d downloaded.", ticket_id)
self.logger.debug("Downloaded content has %d bytes.",
len(resp.content))
if self.extract_download:
raw = resp.content
else:
raw = resp.text
report["extra.file_name"] = file_name_from_response(resp)
report.add("rtir_id", ticket_id)
report.add("time.observation", created + ' UTC', overwrite=True)
"""
On RT 3.8 these fields are only available on the original ticket, not the
first history element as in 4.4
"""
if "Subject" not in ticket:
ticket = RT.get_ticket(ticket_id)
report.add("extra.email_subject", ticket["Subject"])
report.add("extra.ticket_subject", ticket["Subject"])
report.add("extra.email_from", ','.join(ticket["Requestors"]))
report.add("extra.ticket_requestors",
','.join(ticket["Requestors"]))
report.add("extra.ticket_queue", ticket["Queue"])
report.add("extra.ticket_status", ticket["Status"])
report.add("extra.ticket_owner", ticket["Owner"])
if self.extract_files:
try:
unzipped = unzip(raw,
self.extract_files,
return_names=True,
logger=self.logger)
except ValueError:
self.logger.error(
'Could not uncompress the file. Skipping for now.')
continue
for file_name, raw_report in unzipped:
"""
File name priority is:
From the archive (zip, tar.gz)
From the HTTP Response
From the Attachment name
For gz attachments, only the last options works
"""
report_new = report.copy()
report_new.add("raw", raw_report)
report_new.add("extra.file_name",
file_name,
overwrite=True)
if "extra.file_name" not in report_new and att_name.endswith(
'.gz'):
report_new["extra.file_name"] = att_name[:-3]
self.send_message(report_new)
else:
report.add("raw", raw)
self.send_message(report)
if self.parameters.take_ticket:
try:
RT.take(ticket_id)
except rt.BadRequest:
self.logger.exception("Could not take ticket %s.",
ticket_id)
if self.parameters.set_status:
RT.edit_ticket(ticket_id, status=self.parameters.set_status)