def test_file_name_from_response(self): """ test file_name_from_response """ response = requests.Response() response.headers[ 'Content-Disposition'] = 'attachment; filename=2019-09-09-drone_brute_force-austria-geo.csv' self.assertEqual(utils.file_name_from_response(response), '2019-09-09-drone_brute_force-austria-geo.csv')
def process_message(self, uid, message): erroneous = False # If errors occurred this will be set to true. seen = False for body in message.body['plain']: match = re.search(self.parameters.url_regex, str(body.decode('utf-8') if isinstance(body, bytes) else body)) if match: url = match.group() # strip leading and trailing spaces, newlines and # carriage returns url = url.strip() self.logger.info("Downloading report from %r.", url) try: resp = self.session.get(url=url) except requests.exceptions.Timeout: self.logger.error("Request timed out %i times in a row. " % self.http_timeout_max_tries) erroneous = True # The download timed out too often, leave the Loop. continue if resp.status_code // 100 != 2: self.logger.error('HTTP response status code was {}.' ''.format(resp.status_code)) erroneous = True continue if not resp.content: self.logger.warning('Got empty reponse from server.') else: self.logger.info("Report downloaded.") template = self.new_report() template["feed.url"] = url template["extra.email_subject"] = message.subject template["extra.email_from"] = ','.join(x['email'] for x in message.sent_from) template["extra.email_message_id"] = message.message_id template["extra.file_name"] = file_name_from_response(resp) for report in generate_reports(template, io.BytesIO(resp.content), self.chunk_size, self.chunk_replicate_header): self.send_message(report) seen = True if not erroneous: self.logger.info("Email report read.") else: if self.parameters.error_procedure == 'pass': seen = True else: self.logger.error("Email report read with above errors, the report was not processed.") return seen
def process(self): RT = rt.Rt(self.parameters.uri, self.parameters.user, self.parameters.password) if not RT.login(): raise ValueError('Login failed.') if self.not_older_than_type: if self.not_older_than_type == 'relative': self.not_older_than = datetime.now( ) - self.not_older_than_relative kwargs = {'Created__gt': self.not_older_than.isoformat()} self.logger.debug('Searching for tickets newer than %r.', kwargs['Created__gt']) else: kwargs = {} for parameter_name, rt_name in self.parameter_mapping.items(): parameter_value = getattr(self.parameters, parameter_name, None) if parameter_value: kwargs[rt_name] = parameter_value query = RT.search(order='Created', **kwargs) self.logger.info('%s results on search query.', len(query)) for ticket in query: ticket_id = int(ticket['id'].split('/')[1]) self.logger.debug('Process ticket %s.', ticket_id) content = 'attachment' success = False if self.parameters.attachment_regex: for (att_id, att_name, _, _) in RT.get_attachments(ticket_id): if re.search(self.parameters.attachment_regex, att_name): self.logger.debug('Found attachment %s: %r.', att_id, att_name) success = True content = 'attachment' self.extract_files = self.extract_attachment break if not success and self.parameters.url_regex: ticket = RT.get_history(ticket_id)[0] created = ticket['Created'] urlmatch = re.search(self.parameters.url_regex, ticket['Content']) if urlmatch: content = 'url' self.extract_files = self.extract_download url = urlmatch.group(0) self.logger.debug('Matching URL found %r.', url) success = True if not success: self.logger.info('No matching attachment or URL found.') continue report = self.new_report() if content == 'attachment': attachment = RT.get_attachment_content(ticket_id, att_id) created = RT.get_attachment(ticket_id, att_id)['Created'] raw = attachment else: resp = self.session.get(url=url) response_code_class = resp.status_code // 100 if response_code_class != 2: self.logger.error( 'HTTP response status code for %r was %s. Skipping ticket %d.', url, resp.status_code, ticket_id) if response_code_class == 4: self.logger.debug('Server response: %r.', resp.text) if self.parameters.set_status: RT.edit_ticket(ticket_id, status=self.parameters.set_status) if self.parameters.take_ticket: try: RT.take(ticket_id) except rt.BadRequest: self.logger.exception( "Could not take ticket %s.", ticket_id) else: self.logger.info('Skipping now.') continue self.logger.info("Report #%d downloaded.", ticket_id) self.logger.debug("Downloaded content has %d bytes.", len(resp.content)) if self.extract_download: raw = resp.content else: raw = resp.text report["extra.file_name"] = file_name_from_response(resp) report.add("rtir_id", ticket_id) report.add("time.observation", created + ' UTC', overwrite=True) """ On RT 3.8 these fields are only available on the original ticket, not the first history element as in 4.4 """ if "Subject" not in ticket: ticket = RT.get_ticket(ticket_id) report.add("extra.email_subject", ticket["Subject"]) report.add("extra.ticket_subject", ticket["Subject"]) report.add("extra.email_from", ','.join(ticket["Requestors"])) report.add("extra.ticket_requestors", ','.join(ticket["Requestors"])) report.add("extra.ticket_queue", ticket["Queue"]) report.add("extra.ticket_status", ticket["Status"]) report.add("extra.ticket_owner", ticket["Owner"]) if self.extract_files: try: unzipped = unzip(raw, self.extract_files, return_names=True, logger=self.logger) except ValueError: self.logger.error( 'Could not uncompress the file. Skipping for now.') continue for file_name, raw_report in unzipped: """ File name priority is: From the archive (zip, tar.gz) From the HTTP Response From the Attachment name For gz attachments, only the last options works """ report_new = report.copy() report_new.add("raw", raw_report) report_new.add("extra.file_name", file_name, overwrite=True) if "extra.file_name" not in report_new and att_name.endswith( '.gz'): report_new["extra.file_name"] = att_name[:-3] self.send_message(report_new) else: report.add("raw", raw) self.send_message(report) if self.parameters.take_ticket: try: RT.take(ticket_id) except rt.BadRequest: self.logger.exception("Could not take ticket %s.", ticket_id) if self.parameters.set_status: RT.edit_ticket(ticket_id, status=self.parameters.set_status)