async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Scan a payload using TRiD
"""
results: DefaultDict = defaultdict(list)
errors: List[Error] = []
unknown_ext: int = 0
with tempfile.NamedTemporaryFile() as temp_file:
temp_file.write(payload.content)
temp_file.flush()
env = os.environ.copy()
env['LC_ALL'] = 'C'
p = Popen(
[self.bin, f"-d:{self.trid_defs}", temp_file.name],
stdout=PIPE,
stderr=PIPE,
env=env,
universal_newlines=True,
)
trid_results, err = p.communicate()
if err:
errors.append(
Error(
error=err,
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
))
matches = re.findall(r'^ {0,2}[0-9].*%.*$', trid_results, re.M)
warnings = re.findall(r'^Warning: (.*$)', trid_results, re.M)
errors.extend([
Error(
error=w,
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
) for w in warnings if w not in self.skip_warnings
])
for match in matches:
match = match.split()
if match:
try:
ext = match[1].strip('(.)')
if not ext:
ext = f'UNK{unknown_ext}'
unknown_ext += 1
results[ext].append({
'likely': match[0],
'type': ' '.join(match[2:])
})
except IndexError:
continue
return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Return individual result from vtmis-filefeed provider
"""
extracted: List[ExtractedPayload] = []
errors: List[Error] = []
results: Dict = json.loads(payload.content)
if self.download:
self.log.info(f'Downloading VTMIS sample sha1: {results["sha1"]}')
try:
response = requests.get(results['link'])
response.raise_for_status()
extracted = [ExtractedPayload(response.content)]
except Exception as err:
errors.append(
Error(
error=
f'Unable to download sample {results["sha1"]}: {err}',
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
))
return WorkerResponse(results=results,
errors=errors,
extracted=extracted)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Scan payloads using OPSWAT MetaDefender
"""
errors: List[Error] = []
headers = {
'apikey':
self.apikey,
'content-type':
'application/octet-stream',
'filename':
payload.results.payload_meta.extra_data.get(
'filename',
get_sha1(payload.content).encode()).decode(),
}
async with aiohttp.ClientSession(raise_for_status=True) as session:
async with session.post(self.opswat_url,
data=payload.content,
headers=headers) as response:
content = await response.json()
data_id = content['data_id']
results, error = await self._parse_results(data_id)
if error:
errors.append(
Error(
error=error,
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
))
return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
extracted: List[ExtractedPayload] = []
errors: List[Error] = []
try:
parsed_xml = parseString(payload.content)
except ExpatError as err:
errors.append(
Error(
error=f'Unable to parse payload as XML with xdpcarve: {err}',
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
)
)
return WorkerResponse(errors=errors)
for name in self.elements:
dom_element = parsed_xml.getElementsByTagName(name)
for dom in dom_element:
content = dom.firstChild.nodeValue
content = content.rstrip()
try:
content = base64.b64decode(content)
except:
pass
meta = PayloadMeta(extra_data={'element_name': name})
extracted.append(ExtractedPayload(content, meta))
return WorkerResponse(extracted=extracted, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Scan payloads using OPSWAT MetaDefender
"""
errors: List[Error] = []
headers = {
'apikey':
self.apikey,
'content-type':
'application/octet-stream',
'filename':
payload.results.payload_meta.extra_data.get(
'filename', get_sha1(payload.content)),
}
response = requests.post(self.opswat_url,
data=payload.content,
headers=headers)
response.raise_for_status()
data_id = response.json()['data_id']
results, error = self._parse_results(data_id)
if error:
errors.append(
Error(
error=error,
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
))
return WorkerResponse(results, errors=errors)
def _parse_results(
self, job_id: str
) -> Tuple[Union[Dict, None], Union[List[str], None]]:
"""
Wait for a scan to complete and then parse the results
"""
count = 0
errors: List[Error] = []
while count < self.max_attempts:
sleep(self.delay)
try:
url = f'{self.sandbox_url}/report/{job_id}/summary'
headers = {'api-key': self.apikey, 'user-agent': self.useragent}
response = requests.get(url, headers=headers)
response.raise_for_status()
result = response.json()
if result['state'] not in ('IN_QUEUE', 'IN_PROGRESS'):
return result, errors
except (JSONDecodeError, KeyError) as err:
errors.append(
Error(
error=err, plugin_name=self.plugin_name, payload_id=payload_id
)
)
finally:
count += 1
if count >= self.max_attempts:
msg = f'Scan did not complete in time -- attempts: {count}'
errors.append(
Error(
error=msg,
plugin_name=self.plugin_name,
payload_id=payload_id,
)
)
return None, errors
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Scan a payload using Exiftool
"""
errors: List[Error] = []
try:
cmd = [self.bin, '-j', '-n', '-']
p = Popen(cmd, stdout=PIPE, stdin=PIPE)
out, err = p.communicate(input=payload.content)
results = json.loads(out)[0]
except Exception as err:
errors.append(
Error(err,
plugin_name=self.plugin_name,
payload_id=payload.payload_id))
return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
extracted: List[ExtractedPayload] = []
errors: List[Error] = []
ole_object = olefile.OleFileIO(payload.content)
streams = ole_object.listdir(streams=True)
for stream in streams:
try:
stream_buffer = ole_object.openstream(stream).read()
name = ''.join(
filter(lambda x: x in string.printable, '_'.join(stream)))
if stream_buffer.endswith(b'\x01Ole10Native'):
ole_native = oleobj.OleNativeStream(stream_buffer)
if ole_native.filename:
name = f'{name}_{str(ole_native.filename)}'
else:
name = f'{name}_olenative'
meta = PayloadMeta(
should_archive=False,
extra_data={
'index': streams.index(stream),
'name': name
},
)
extracted.append(ExtractedPayload(ole_native.data, meta))
else:
meta = PayloadMeta(
should_archive=False,
extra_data={
'index': streams.index(stream),
'name': name
},
)
extracted.append(ExtractedPayload(stream_buffer, meta))
except Exception as err:
errors.append(
Error(
error=str(err),
plugin_name=self.plugin_name,
payload_id=payload.payload_id,
))
return WorkerResponse(extracted=extracted, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Carve and decompress SWF files from payloads
"""
extracted: List[ExtractedPayload] = []
errors: List[Error] = []
content = BytesIO(payload.content)
content.seek(0)
for start, end in self._carve(content):
ex, errs = self.decompress(content, start)
if ex:
extracted.append(ex)
for err in errs:
errors.append(
Error(
error=err,
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
))
return WorkerResponse(extracted=extracted, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Scan a payload using Exiftool
"""
errors: List[Error] = []
try:
p = await create_subprocess_exec(self.bin,
'-j',
'-n',
'-',
stdout=PIPE,
stdin=PIPE,
stderr=PIPE)
out, err = await p.communicate(input=payload.content)
if err:
self.log.debug(err)
results = json.loads(out)[0]
except Exception as err:
errors.append(
Error(err,
plugin_name=self.plugin_name,
payload_id=payload.payload_id))
return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
message_json: Dict[str, str] = {}
attachments: List[ExtractedPayload] = []
errors: List[Error] = []
ioc_content: str = ''
session = UnicodeDammit(payload.content).unicode_markup
message = Parser(policy=policy.default).parsestr(session)
try:
# Check for invalid date string
# https://bugs.python.org/issue30681
message.get('Date')
except TypeError:
date_header = [d[1] for d in message._headers if d[0] == 'Date'][0]
date_header = dtparse(date_header).strftime('%c %z')
message.replace_header('Date', date_header)
# Create a dict of the SMTP headers
for header, value in message.items():
curr_header = header.lower()
if curr_header in message_json:
message_json[curr_header] += f'\n{value}'
else:
message_json[curr_header] = value
if not self.omit_body:
message_json['body'] = self._get_body(message, 'plain')
message_json['body_html'] = self._get_body(message, 'html')
if self.extract_iocs:
for k in self.ioc_keys:
if k in message_json:
ioc_content += f'\n{message_json[k]}'
elif k == 'body' and k not in message_json:
b = self._get_body(message, 'plain')
if b:
ioc_content += b
elif k == 'body_html' and k not in message_json:
b = self._get_body(message, 'html')
if b:
ioc_content += b
for mailpart in message.iter_attachments():
if mailpart.get_content_type() == 'message/rfc822':
for part in mailpart.get_payload():
try:
attachment_meta = PayloadMeta(
should_archive=self.archive_attachments,
extra_data={
'charset':
part.get_content_charset(),
'content-description':
part.get('Content-Description'),
'disposition':
part.get_content_disposition(),
'filename':
part.get_filename(),
'type':
part.get_content_type(),
},
dispatch_to=['smtp'],
)
attachment = ExtractedPayload(part.as_bytes(),
attachment_meta)
attachments.append(attachment)
except Exception as err:
errors.append(
Error(
error=f'Failed rfc822 attachment: {err}',
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
))
else:
try:
attachment_meta = PayloadMeta(
should_archive=self.archive_attachments,
extra_data={
'charset':
mailpart.get_content_charset(),
'content-description':
mailpart.get('Content-Description'),
'disposition':
mailpart.get_content_disposition(),
'filename':
mailpart.get_filename(),
'type':
mailpart.get_content_type(),
},
dispatch_to=self.always_dispatch,
)
attachment = ExtractedPayload(mailpart.get_content(),
attachment_meta)
attachments.append(attachment)
except Exception as err:
errors.append(
Error(
error=f'Failed extracting attachment: {err}',
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
))
if self.extract_iocs:
ioc_meta = PayloadMeta(should_archive=False,
dispatch_to=['iocextract'])
attachments.append(ExtractedPayload(ioc_content.encode(),
ioc_meta))
return WorkerResponse(message_json,
errors=errors,
extracted=attachments)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Decompress a payload
payload.results.payload_meta:
- passwords
- archiver
"""
if len(payload.content) > self.maximum_size:
raise StoqPluginException(
f'Compressed file too large: {len(payload.content)} > {self.maximum_size}'
)
archiver = None
mimetype = None
errors: List[Error] = []
results: Dict = {}
extracted: List[ExtractedPayload] = []
passwords: List[str] = payload.results.payload_meta.extra_data.get(
'passwords', self.passwords
)
# Determine the mimetype of the payload so we can identify the
# correct archiver. This should either be based off the payload.results.payload_meta
# (useful when payload is passed via dispatching) or via mimetype plugin
if 'archiver' in payload.results.payload_meta.extra_data:
if payload.results.payload_meta.extra_data['archiver'] in self.ARCHIVE_CMDS:
archiver = self.ARCHIVE_CMDS[
payload.results.payload_meta.extra_data['archiver']
]
else:
raise StoqPluginException(
f"Unknown archive type of {payload.results.payload_meta['archiver']}"
)
else:
mimetype = payload.results.workers['mimetype']['mimetype']
if mimetype in self.ARCHIVE_MAGIC:
archive_type = self.ARCHIVE_MAGIC[mimetype]
if archive_type in self.ARCHIVE_CMDS:
archiver = self.ARCHIVE_CMDS[archive_type]
else:
raise StoqPluginException(f'Unknown archive type of {archive_type}')
if not archiver:
raise StoqPluginException(
f'Unable to determine archive type, mimetype: {mimetype}'
)
with tempfile.TemporaryDirectory() as extract_dir:
fd, archive_file = tempfile.mkstemp(dir=extract_dir)
with open(fd, 'xb') as f:
f.write(payload.content)
f.flush()
archive_outdir = tempfile.mkdtemp(dir=extract_dir)
for password in passwords:
cmd = archiver.replace('%INFILE%', shlex.quote(archive_file))
cmd = cmd.replace('%OUTDIR%', shlex.quote(archive_outdir))
cmd = cmd.replace('%PASSWORD%', shlex.quote(password))
p = Popen(
cmd.split(" "), stdout=PIPE, stderr=PIPE, universal_newlines=True
)
try:
outs, errs = p.communicate(timeout=self.timeout)
except TimeoutExpired:
p.kill()
raise StoqPluginException('Timed out decompressing payload')
if p.returncode == 0:
break
for root, dirs, files in os.walk(archive_outdir):
for f in files:
path = os.path.join(extract_dir, root, str(f))
if os.path.getsize(path) > self.maximum_size:
errors.append(
Error(
f'Extracted object is too large ({os.path.getsize(path)} > {self.maximum_size})',
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
)
)
continue
with open(path, "rb") as extracted_file:
meta = PayloadMeta(extra_data={'filename': f})
try:
data = extracted_file.read()
except OSError as err:
errors.append(
Error(
f'Unable to access extracted content: {err}',
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
)
)
continue
extracted.append(ExtractedPayload(data, meta))
return WorkerResponse(results, extracted=extracted, errors=errors)