async def scan(self, payload: Payload, request: Request) -> WorkerResponse: """ Scan a payload using TRiD """ results: DefaultDict = defaultdict(list) errors: List[Error] = [] unknown_ext: int = 0 with tempfile.NamedTemporaryFile() as temp_file: temp_file.write(payload.content) temp_file.flush() env = os.environ.copy() env['LC_ALL'] = 'C' p = Popen( [self.bin, f"-d:{self.trid_defs}", temp_file.name], stdout=PIPE, stderr=PIPE, env=env, universal_newlines=True, ) trid_results, err = p.communicate() if err: errors.append( Error( error=err, plugin_name=self.plugin_name, payload_id=payload.results.payload_id, )) matches = re.findall(r'^ {0,2}[0-9].*%.*$', trid_results, re.M) warnings = re.findall(r'^Warning: (.*$)', trid_results, re.M) errors.extend([ Error( error=w, plugin_name=self.plugin_name, payload_id=payload.results.payload_id, ) for w in warnings if w not in self.skip_warnings ]) for match in matches: match = match.split() if match: try: ext = match[1].strip('(.)') if not ext: ext = f'UNK{unknown_ext}' unknown_ext += 1 results[ext].append({ 'likely': match[0], 'type': ' '.join(match[2:]) }) except IndexError: continue return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse: """ Return individual result from vtmis-filefeed provider """ extracted: List[ExtractedPayload] = [] errors: List[Error] = [] results: Dict = json.loads(payload.content) if self.download: self.log.info(f'Downloading VTMIS sample sha1: {results["sha1"]}') try: response = requests.get(results['link']) response.raise_for_status() extracted = [ExtractedPayload(response.content)] except Exception as err: errors.append( Error( error= f'Unable to download sample {results["sha1"]}: {err}', plugin_name=self.plugin_name, payload_id=payload.results.payload_id, )) return WorkerResponse(results=results, errors=errors, extracted=extracted)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse: """ Scan payloads using OPSWAT MetaDefender """ errors: List[Error] = [] headers = { 'apikey': self.apikey, 'content-type': 'application/octet-stream', 'filename': payload.results.payload_meta.extra_data.get( 'filename', get_sha1(payload.content).encode()).decode(), } async with aiohttp.ClientSession(raise_for_status=True) as session: async with session.post(self.opswat_url, data=payload.content, headers=headers) as response: content = await response.json() data_id = content['data_id'] results, error = await self._parse_results(data_id) if error: errors.append( Error( error=error, plugin_name=self.plugin_name, payload_id=payload.results.payload_id, )) return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse: extracted: List[ExtractedPayload] = [] errors: List[Error] = [] try: parsed_xml = parseString(payload.content) except ExpatError as err: errors.append( Error( error=f'Unable to parse payload as XML with xdpcarve: {err}', plugin_name=self.plugin_name, payload_id=payload.results.payload_id, ) ) return WorkerResponse(errors=errors) for name in self.elements: dom_element = parsed_xml.getElementsByTagName(name) for dom in dom_element: content = dom.firstChild.nodeValue content = content.rstrip() try: content = base64.b64decode(content) except: pass meta = PayloadMeta(extra_data={'element_name': name}) extracted.append(ExtractedPayload(content, meta)) return WorkerResponse(extracted=extracted, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse: """ Scan payloads using OPSWAT MetaDefender """ errors: List[Error] = [] headers = { 'apikey': self.apikey, 'content-type': 'application/octet-stream', 'filename': payload.results.payload_meta.extra_data.get( 'filename', get_sha1(payload.content)), } response = requests.post(self.opswat_url, data=payload.content, headers=headers) response.raise_for_status() data_id = response.json()['data_id'] results, error = self._parse_results(data_id) if error: errors.append( Error( error=error, plugin_name=self.plugin_name, payload_id=payload.results.payload_id, )) return WorkerResponse(results, errors=errors)
def _parse_results( self, job_id: str ) -> Tuple[Union[Dict, None], Union[List[str], None]]: """ Wait for a scan to complete and then parse the results """ count = 0 errors: List[Error] = [] while count < self.max_attempts: sleep(self.delay) try: url = f'{self.sandbox_url}/report/{job_id}/summary' headers = {'api-key': self.apikey, 'user-agent': self.useragent} response = requests.get(url, headers=headers) response.raise_for_status() result = response.json() if result['state'] not in ('IN_QUEUE', 'IN_PROGRESS'): return result, errors except (JSONDecodeError, KeyError) as err: errors.append( Error( error=err, plugin_name=self.plugin_name, payload_id=payload_id ) ) finally: count += 1 if count >= self.max_attempts: msg = f'Scan did not complete in time -- attempts: {count}' errors.append( Error( error=msg, plugin_name=self.plugin_name, payload_id=payload_id, ) ) return None, errors
async def scan(self, payload: Payload, request: Request) -> WorkerResponse: """ Scan a payload using Exiftool """ errors: List[Error] = [] try: cmd = [self.bin, '-j', '-n', '-'] p = Popen(cmd, stdout=PIPE, stdin=PIPE) out, err = p.communicate(input=payload.content) results = json.loads(out)[0] except Exception as err: errors.append( Error(err, plugin_name=self.plugin_name, payload_id=payload.payload_id)) return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse: extracted: List[ExtractedPayload] = [] errors: List[Error] = [] ole_object = olefile.OleFileIO(payload.content) streams = ole_object.listdir(streams=True) for stream in streams: try: stream_buffer = ole_object.openstream(stream).read() name = ''.join( filter(lambda x: x in string.printable, '_'.join(stream))) if stream_buffer.endswith(b'\x01Ole10Native'): ole_native = oleobj.OleNativeStream(stream_buffer) if ole_native.filename: name = f'{name}_{str(ole_native.filename)}' else: name = f'{name}_olenative' meta = PayloadMeta( should_archive=False, extra_data={ 'index': streams.index(stream), 'name': name }, ) extracted.append(ExtractedPayload(ole_native.data, meta)) else: meta = PayloadMeta( should_archive=False, extra_data={ 'index': streams.index(stream), 'name': name }, ) extracted.append(ExtractedPayload(stream_buffer, meta)) except Exception as err: errors.append( Error( error=str(err), plugin_name=self.plugin_name, payload_id=payload.payload_id, )) return WorkerResponse(extracted=extracted, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse: """ Carve and decompress SWF files from payloads """ extracted: List[ExtractedPayload] = [] errors: List[Error] = [] content = BytesIO(payload.content) content.seek(0) for start, end in self._carve(content): ex, errs = self.decompress(content, start) if ex: extracted.append(ex) for err in errs: errors.append( Error( error=err, plugin_name=self.plugin_name, payload_id=payload.results.payload_id, )) return WorkerResponse(extracted=extracted, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse: """ Scan a payload using Exiftool """ errors: List[Error] = [] try: p = await create_subprocess_exec(self.bin, '-j', '-n', '-', stdout=PIPE, stdin=PIPE, stderr=PIPE) out, err = await p.communicate(input=payload.content) if err: self.log.debug(err) results = json.loads(out)[0] except Exception as err: errors.append( Error(err, plugin_name=self.plugin_name, payload_id=payload.payload_id)) return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse: message_json: Dict[str, str] = {} attachments: List[ExtractedPayload] = [] errors: List[Error] = [] ioc_content: str = '' session = UnicodeDammit(payload.content).unicode_markup message = Parser(policy=policy.default).parsestr(session) try: # Check for invalid date string # https://bugs.python.org/issue30681 message.get('Date') except TypeError: date_header = [d[1] for d in message._headers if d[0] == 'Date'][0] date_header = dtparse(date_header).strftime('%c %z') message.replace_header('Date', date_header) # Create a dict of the SMTP headers for header, value in message.items(): curr_header = header.lower() if curr_header in message_json: message_json[curr_header] += f'\n{value}' else: message_json[curr_header] = value if not self.omit_body: message_json['body'] = self._get_body(message, 'plain') message_json['body_html'] = self._get_body(message, 'html') if self.extract_iocs: for k in self.ioc_keys: if k in message_json: ioc_content += f'\n{message_json[k]}' elif k == 'body' and k not in message_json: b = self._get_body(message, 'plain') if b: ioc_content += b elif k == 'body_html' and k not in message_json: b = self._get_body(message, 'html') if b: ioc_content += b for mailpart in message.iter_attachments(): if mailpart.get_content_type() == 'message/rfc822': for part in mailpart.get_payload(): try: attachment_meta = PayloadMeta( should_archive=self.archive_attachments, extra_data={ 'charset': part.get_content_charset(), 'content-description': part.get('Content-Description'), 'disposition': part.get_content_disposition(), 'filename': part.get_filename(), 'type': part.get_content_type(), }, dispatch_to=['smtp'], ) attachment = ExtractedPayload(part.as_bytes(), attachment_meta) attachments.append(attachment) except Exception as err: errors.append( Error( error=f'Failed rfc822 attachment: {err}', plugin_name=self.plugin_name, payload_id=payload.results.payload_id, )) else: try: attachment_meta = PayloadMeta( should_archive=self.archive_attachments, extra_data={ 'charset': mailpart.get_content_charset(), 'content-description': mailpart.get('Content-Description'), 'disposition': mailpart.get_content_disposition(), 'filename': mailpart.get_filename(), 'type': mailpart.get_content_type(), }, dispatch_to=self.always_dispatch, ) attachment = ExtractedPayload(mailpart.get_content(), attachment_meta) attachments.append(attachment) except Exception as err: errors.append( Error( error=f'Failed extracting attachment: {err}', plugin_name=self.plugin_name, payload_id=payload.results.payload_id, )) if self.extract_iocs: ioc_meta = PayloadMeta(should_archive=False, dispatch_to=['iocextract']) attachments.append(ExtractedPayload(ioc_content.encode(), ioc_meta)) return WorkerResponse(message_json, errors=errors, extracted=attachments)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse: """ Decompress a payload payload.results.payload_meta: - passwords - archiver """ if len(payload.content) > self.maximum_size: raise StoqPluginException( f'Compressed file too large: {len(payload.content)} > {self.maximum_size}' ) archiver = None mimetype = None errors: List[Error] = [] results: Dict = {} extracted: List[ExtractedPayload] = [] passwords: List[str] = payload.results.payload_meta.extra_data.get( 'passwords', self.passwords ) # Determine the mimetype of the payload so we can identify the # correct archiver. This should either be based off the payload.results.payload_meta # (useful when payload is passed via dispatching) or via mimetype plugin if 'archiver' in payload.results.payload_meta.extra_data: if payload.results.payload_meta.extra_data['archiver'] in self.ARCHIVE_CMDS: archiver = self.ARCHIVE_CMDS[ payload.results.payload_meta.extra_data['archiver'] ] else: raise StoqPluginException( f"Unknown archive type of {payload.results.payload_meta['archiver']}" ) else: mimetype = payload.results.workers['mimetype']['mimetype'] if mimetype in self.ARCHIVE_MAGIC: archive_type = self.ARCHIVE_MAGIC[mimetype] if archive_type in self.ARCHIVE_CMDS: archiver = self.ARCHIVE_CMDS[archive_type] else: raise StoqPluginException(f'Unknown archive type of {archive_type}') if not archiver: raise StoqPluginException( f'Unable to determine archive type, mimetype: {mimetype}' ) with tempfile.TemporaryDirectory() as extract_dir: fd, archive_file = tempfile.mkstemp(dir=extract_dir) with open(fd, 'xb') as f: f.write(payload.content) f.flush() archive_outdir = tempfile.mkdtemp(dir=extract_dir) for password in passwords: cmd = archiver.replace('%INFILE%', shlex.quote(archive_file)) cmd = cmd.replace('%OUTDIR%', shlex.quote(archive_outdir)) cmd = cmd.replace('%PASSWORD%', shlex.quote(password)) p = Popen( cmd.split(" "), stdout=PIPE, stderr=PIPE, universal_newlines=True ) try: outs, errs = p.communicate(timeout=self.timeout) except TimeoutExpired: p.kill() raise StoqPluginException('Timed out decompressing payload') if p.returncode == 0: break for root, dirs, files in os.walk(archive_outdir): for f in files: path = os.path.join(extract_dir, root, str(f)) if os.path.getsize(path) > self.maximum_size: errors.append( Error( f'Extracted object is too large ({os.path.getsize(path)} > {self.maximum_size})', plugin_name=self.plugin_name, payload_id=payload.results.payload_id, ) ) continue with open(path, "rb") as extracted_file: meta = PayloadMeta(extra_data={'filename': f}) try: data = extracted_file.read() except OSError as err: errors.append( Error( f'Unable to access extracted content: {err}', plugin_name=self.plugin_name, payload_id=payload.results.payload_id, ) ) continue extracted.append(ExtractedPayload(data, meta)) return WorkerResponse(results, extracted=extracted, errors=errors)