async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
extracted: List[ExtractedPayload] = []
errors: List[Error] = []
try:
parsed_xml = parseString(payload.content)
except ExpatError as err:
errors.append(
Error(
error=f'Unable to parse payload as XML with xdpcarve: {err}',
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
)
)
return WorkerResponse(errors=errors)
for name in self.elements:
dom_element = parsed_xml.getElementsByTagName(name)
for dom in dom_element:
content = dom.firstChild.nodeValue
content = content.rstrip()
try:
content = base64.b64decode(content)
except:
pass
meta = PayloadMeta(extra_data={'element_name': name})
extracted.append(ExtractedPayload(content, meta))
return WorkerResponse(extracted=extracted, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
return WorkerResponse(
results={
'sha256': sha256(payload.content).hexdigest(),
'md5': md5(payload.content).hexdigest(),
'sha1': sha1(payload.content).hexdigest(),
})
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
results = parse_zip(
BytesIO(payload.content),
decompress_files=self.decompress_files,
derive_deflate_level=self.derive_deflate_level,
)
return WorkerResponse(results)
def scan(self, payload: Payload,
request_meta: RequestMeta) -> WorkerResponse:
"""
Scan a payload using xorsearch
"""
with tempfile.NamedTemporaryFile() as temp_file:
temp_file.write(payload.content)
temp_file.flush()
cmd = [self.bin_path, '-f', self.terms, temp_file.name]
process_results = check_output(cmd).splitlines()
result = {}
for line in process_results:
line = line.decode()
_, _, key, _, pos, hit = line.split(maxsplit=5)
# We are going to skip over hits that are not xor'd
if key != '00':
key = f'0x{key}'
if key not in result:
result[key] = []
result[key].append({
'pos': f'0x{pos.replace("(-1):", "")}',
'match': hit
})
return WorkerResponse(result)
def scan(self, payload: Payload,
request_meta: RequestMeta) -> WorkerResponse:
"""
Scan a payload using pefile
"""
pe = pefile.PE(data=payload.content)
results = {}
results['imphash'] = self.get_imphash(pe)
results['compile_time'] = self.get_compiletime(pe)
results['packer'] = self.get_packer(pe)
results['is_packed'] = self.is_packed(pe)
results['is_exe'] = self.is_exe(pe)
results['is_dll'] = self.is_dll(pe)
results['is_driver'] = self.is_driver(pe)
results['is_valid'] = self.is_valid(pe)
results['is_suspicious'] = self.is_suspicious(pe)
results['machine_type'] = self.get_machinetype(pe)
results['entrypoint'] = self.get_entrypoint(pe)
results['section_count'] = self.get_sectioncount(pe)
results['sections'] = self.get_sections(pe)
results['imports'] = self.get_imports(pe)
results['rich_header'] = self.get_rich_header(pe)
pe.close()
return WorkerResponse(results)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
xorkey: Union[List[int], str, int,
None] = dpath.util.get(payload.dispatch_meta,
'**/xorkey',
default=None)
if not xorkey:
return
elif isinstance(xorkey, str):
xorkey = [int(k.strip()) for k in xorkey.split(',')]
elif isinstance(xorkey, int):
xorkey = [xorkey]
last_rolling_index = len(xorkey) - 1
current_rolling_index = 0
payload_bytes = bytearray(payload.content)
for index in range(payload.results.size):
xor_value = xorkey[current_rolling_index]
payload_bytes[index] ^= xor_value
if current_rolling_index < last_rolling_index:
current_rolling_index += 1
else:
current_rolling_index = 0
payload.results.payload_meta.extra_data['xorkey'] = xorkey
meta = PayloadMeta(extra_data={'xorkey': xorkey})
extracted = [ExtractedPayload(bytes(payload_bytes), meta)]
return WorkerResponse(extracted=extracted)
def scan(self, payload: Payload,
request_meta: RequestMeta) -> WorkerResponse:
"""
Decodes and extracts information from Java Class files
"""
results = {}
try:
content = unpack_class(payload.content)
except ClassUnpackException as err:
raise StoqPluginException(f'Unable to parse payload: {err}')
try:
results = {
'provided': content.get_provides(),
'required': content.get_requires(),
'constants': [],
}
for obj, _, data in content.cpool.pretty_constants():
if len(data) <= 6:
continue
constants = {}
constants['id'] = obj
constants['data'] = data
results['constants'].append(constants)
except Exception as err:
raise StoqPluginException(f'Unable to analyze Java Class {err}')
return WorkerResponse(results)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Scan payloads using OPSWAT MetaDefender
"""
errors: List[Error] = []
headers = {
'apikey':
self.apikey,
'content-type':
'application/octet-stream',
'filename':
payload.results.payload_meta.extra_data.get(
'filename', get_sha1(payload.content)),
}
response = requests.post(self.opswat_url,
data=payload.content,
headers=headers)
response.raise_for_status()
data_id = response.json()['data_id']
results, error = self._parse_results(data_id)
if error:
errors.append(
Error(
error=error,
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
))
return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Scan payloads using OPSWAT MetaDefender
"""
errors: List[Error] = []
headers = {
'apikey':
self.apikey,
'content-type':
'application/octet-stream',
'filename':
payload.results.payload_meta.extra_data.get(
'filename',
get_sha1(payload.content).encode()).decode(),
}
async with aiohttp.ClientSession(raise_for_status=True) as session:
async with session.post(self.opswat_url,
data=payload.content,
headers=headers) as response:
content = await response.json()
data_id = content['data_id']
results, error = await self._parse_results(data_id)
if error:
errors.append(
Error(
error=error,
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
))
return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Return individual result from vtmis-filefeed provider
"""
extracted: List[ExtractedPayload] = []
errors: List[Error] = []
results: Dict = json.loads(payload.content)
if self.download:
self.log.info(f'Downloading VTMIS sample sha1: {results["sha1"]}')
try:
response = requests.get(results['link'])
response.raise_for_status()
extracted = [ExtractedPayload(response.content)]
except Exception as err:
errors.append(
Error(
error=
f'Unable to download sample {results["sha1"]}: {err}',
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
))
return WorkerResponse(results=results,
errors=errors,
extracted=extracted)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
normalize: bool = True
ioctype: str = 'all'
results: Dict = {}
if ioctype == 'all':
for ioc in self.compiled_re:
if self.compiled_re[ioc]:
matches = self.compiled_re[ioc].findall(payload.content.decode())
if matches:
results[ioc] = list(set(matches))
elif self.compiled_re[ioctype]:
matches = self.compiled_re[ioctype].findall(payload.content.decode())
if matches:
results[ioctype] = list(set(matches))
if 'ipv6' in results:
results['ipv6'] = [
address for address in results['ipv6'] if self._validate_ipv6(address)
]
if not results['ipv6']:
results.pop('ipv6')
if normalize:
results = self._normalize(results)
return WorkerResponse(results)
def scan(self, payload: Payload, request_meta: RequestMeta) -> WorkerResponse:
"""
Scan a payload using TRiD
"""
results = {}
with tempfile.NamedTemporaryFile() as temp_file:
temp_file.write(payload.content)
temp_file.flush()
try:
cmd = [self.bin_path, f"-d:{self.trid_defs}", temp_file.name]
trid_results = check_output(cmd).splitlines()
except Exception as err:
raise StoqPluginException('Failed gathering TRiD data')
for line in trid_results[6:]:
if line.startswith('Warning'.encode()):
break
line = line.decode().split()
if line:
ext = line[1].strip('(.)')
results[ext] = {'likely': line[0], 'type': ' '.join(line[2:])}
return WorkerResponse(results)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
results = {
'matches': [
m async for m in self._yara_matches(payload.content,
self.worker_rules)
]
}
return WorkerResponse(results=results)
def scan(self, payload: Payload, request_meta: RequestMeta) -> WorkerResponse:
"""
Upload content to a Tika server for automated text extraction
"""
response = requests.put(self.tika_url, data=payload.content)
response.raise_for_status()
extracted = ExtractedPayload(response.content)
return WorkerResponse(extracted=extracted)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
entropy: float = 0.0
results: Dict[str, float] = {}
if payload.content:
occurences = Counter(bytearray(payload.content))
for bc in occurences.values():
b = float(bc) / len(payload.content)
entropy -= b * math.log(b, 2)
results['entropy'] = entropy
return WorkerResponse(results=results)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
if USE_PYTHON_MAGIC:
magic_scan = magic.Magic(mime=True)
magic_result = magic_scan.from_buffer(payload.content[0:1000])
else:
with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m:
magic_result = m.id_buffer(payload.content[0:1000])
if hasattr(magic_result, 'decode'):
magic_result = magic_result.decode('utf-8')
return WorkerResponse(results={'mimetype': magic_result})
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Scan a payload using TRiD
"""
results: DefaultDict = defaultdict(list)
errors: List[Error] = []
unknown_ext: int = 0
with tempfile.NamedTemporaryFile() as temp_file:
temp_file.write(payload.content)
temp_file.flush()
env = os.environ.copy()
env['LC_ALL'] = 'C'
p = Popen(
[self.bin, f"-d:{self.trid_defs}", temp_file.name],
stdout=PIPE,
stderr=PIPE,
env=env,
universal_newlines=True,
)
trid_results, err = p.communicate()
if err:
errors.append(
Error(
error=err,
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
))
matches = re.findall(r'^ {0,2}[0-9].*%.*$', trid_results, re.M)
warnings = re.findall(r'^Warning: (.*$)', trid_results, re.M)
errors.extend([
Error(
error=w,
plugin_name=self.plugin_name,
payload_id=payload.results.payload_id,
) for w in warnings if w not in self.skip_warnings
])
for match in matches:
match = match.split()
if match:
try:
ext = match[1].strip('(.)')
if not ext:
ext = f'UNK{unknown_ext}'
unknown_ext += 1
results[ext].append({
'likely': match[0],
'type': ' '.join(match[2:])
})
except IndexError:
continue
return WorkerResponse(results, errors=errors)
def scan(self, payload: Payload,
request_meta: RequestMeta) -> WorkerResponse:
extracted = []
errors = []
try:
parsed_xml = parseString(payload.content)
except ExpatError as err:
errors.append(
f'Unable to parse payload as XML with xdpcarve: {err}')
return WorkerResponse(errors=errors)
for name in self.elements:
dom_element = parsed_xml.getElementsByTagName(name)
for dom in dom_element:
content = dom.firstChild.nodeValue
content = content.rstrip()
try:
content = base64.b64decode(content)
except:
pass
meta = PayloadMeta(extra_data={"element_name": name})
extracted.append(ExtractedPayload(content, meta))
return WorkerResponse(extracted=extracted, errors=errors)
def scan(self, payload: Payload,
request_meta: RequestMeta) -> WorkerResponse:
matches = self.worker_rules.match(data=payload.content, timeout=60)
dict_matches = []
for match in matches:
dict_matches.append({
'tags': match.tags,
'namespace': match.namespace,
'rule': match.rule,
'meta': match.meta,
'strings': match.strings[:self.strings_limit],
})
results = {"matches": dict_matches}
return WorkerResponse(results=results)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
extracted: List[ExtractedPayload] = []
tnef_results = TNEF(payload.content)
if tnef_results.attachments:
for tnef_attachment in tnef_results.attachments:
try:
filename = UnicodeDammit(
tnef_attachment.name).unicode_markup
except:
filename = "None"
tnef_meta = PayloadMeta(extra_data={'filename': filename})
extracted.append(
ExtractedPayload(tnef_attachment.data, tnef_meta))
return WorkerResponse(extracted=extracted)
def scan(self, payload: Payload, request_meta: RequestMeta) -> WorkerResponse:
"""
Scan a payload using Exiftool
"""
with tempfile.NamedTemporaryFile() as temp_file:
temp_file.write(payload.content)
temp_file.flush()
try:
cmd = [self.bin_path, '-j', '-n', temp_file.name]
output = run(cmd, stdout=PIPE)
results = json.loads(output.stdout)[0]
except Exception as err:
raise StoqPluginException(f'Failed gathering exif data: {err}')
return WorkerResponse(results)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Scan a payload using Exiftool
"""
errors: List[Error] = []
try:
cmd = [self.bin, '-j', '-n', '-']
p = Popen(cmd, stdout=PIPE, stdin=PIPE)
out, err = p.communicate(input=payload.content)
results = json.loads(out)[0]
except Exception as err:
errors.append(
Error(err,
plugin_name=self.plugin_name,
payload_id=payload.payload_id))
return WorkerResponse(results, errors=errors)
def scan(self, payload: Payload,
request_meta: RequestMeta) -> WorkerResponse:
"""
Carve and decompress SWF files from payloads
"""
extracted = []
errors = []
content = BytesIO(payload.content)
content.seek(0)
for start, end in self._carve(content):
ex, err = self.decompress(content, start)
if ex:
extracted.append(ex)
if err:
errors.extend(err)
return WorkerResponse(extracted=extracted, errors=errors)
def scan(self, payload: Payload, request_meta: RequestMeta) -> WorkerResponse:
"""
Scan payloads using OPSWAT MetaDefender
"""
headers = {
'apikey': self.apikey,
'filename': payload.payload_meta.extra_data.get(
'filename', get_sha1(payload.content)
),
}
response = requests.post(self.opswat_url, data=payload.content, headers=headers)
response.raise_for_status()
data_id = response.json()['data_id']
results, errors = self._parse_results(data_id)
if errors:
errors = [errors]
return WorkerResponse(results, errors=errors)
def scan(self, payload: Payload,
request_meta: RequestMeta) -> WorkerResponse:
"""
Scan a payload using TRiD
"""
results: DefaultDict = defaultdict(list)
errors: List[str] = []
with tempfile.NamedTemporaryFile() as temp_file:
temp_file.write(payload.content)
temp_file.flush()
p = Popen(
[self.bin_path, f"-d:{self.trid_defs}", temp_file.name],
stdout=PIPE,
stderr=PIPE,
env={'LC_ALL': 'C'},
universal_newlines=True,
)
trid_results, err = p.communicate()
if err:
errors.append(err)
unknown_ext = 0
matches = re.findall(r'^ {0,2}[0-9].*%.*$', trid_results, re.M)
warnings = re.findall(r'^Warning: (.*$)', trid_results, re.M)
errors.extend([w for w in warnings])
for match in matches:
match = match.split()
if match:
try:
ext = match[1].strip('(.)')
if not ext:
ext = f'UNK{unknown_ext}'
unknown_ext += 1
results[ext].append({
'likely': match[0],
'type': ' '.join(match[2:])
})
except IndexError:
continue
return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
extracted: List[ExtractedPayload] = []
rtf = rtfobj.RtfObjParser(payload.content)
rtf.parse()
for obj_idx, obj in enumerate(rtf.objects):
if obj.is_ole:
data = obj.oledata
meta = PayloadMeta(extra_data={'index': obj_idx})
elif obj.is_package:
data = obj.olepkgdata
meta = PayloadMeta(extra_data={
'index': obj_idx,
'filename': obj.filename
})
else:
data = obj.rawdata
meta = PayloadMeta(extra_data={'index': obj_idx})
extracted.append(ExtractedPayload(data, meta))
return WorkerResponse(extracted=extracted)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Search VTMIS for sha1 hash of a payload or from results of `iocextract` plugin
"""
results: List[Dict] = []
seen: Set[str] = set()
if 'iocextract' in payload.results.workers:
for key, iocs in payload.results.workers['iocextract'].items():
for ioc in iocs:
if key in self.ENDPOINTS and ioc not in seen:
response = self._query_api(ioc, key)
seen.add(ioc)
results.append(response)
if not results:
sha1 = get_sha1(payload.content)
results = self._query_api(sha1, 'sha1')
return WorkerResponse(results=results)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
extracted: List[ExtractedPayload] = []
errors: List[Error] = []
ole_object = olefile.OleFileIO(payload.content)
streams = ole_object.listdir(streams=True)
for stream in streams:
try:
stream_buffer = ole_object.openstream(stream).read()
name = ''.join(
filter(lambda x: x in string.printable, '_'.join(stream)))
if stream_buffer.endswith(b'\x01Ole10Native'):
ole_native = oleobj.OleNativeStream(stream_buffer)
if ole_native.filename:
name = f'{name}_{str(ole_native.filename)}'
else:
name = f'{name}_olenative'
meta = PayloadMeta(
should_archive=False,
extra_data={
'index': streams.index(stream),
'name': name
},
)
extracted.append(ExtractedPayload(ole_native.data, meta))
else:
meta = PayloadMeta(
should_archive=False,
extra_data={
'index': streams.index(stream),
'name': name
},
)
extracted.append(ExtractedPayload(stream_buffer, meta))
except Exception as err:
errors.append(
Error(
error=str(err),
plugin_name=self.plugin_name,
payload_id=payload.payload_id,
))
return WorkerResponse(extracted=extracted, errors=errors)
def scan(self, payload: Payload,
request_meta: RequestMeta) -> WorkerResponse:
"""
Scan payloads using Falcon Sandbox
"""
errors = None
url = f'{self.sandbox_url}/submit/file'
headers = {'api-key': self.apikey, 'user-agent': self.useragent}
filename = payload.payload_meta.extra_data.get(
'filename', helpers.get_sha1(payload.content))
if isinstance(filename, bytes):
filename = filename.decode()
files = {'file': (filename, payload.content)}
data = {'environment_id': self.environment_id}
response = requests.post(url, data=data, files=files, headers=headers)
response.raise_for_status()
results = response.json()
if self.wait_for_results:
results, errors = self._parse_results(results['job_id'])
return WorkerResponse(results, errors=errors)
async def scan(self, payload: Payload, request: Request) -> WorkerResponse:
"""
Scan a payload using LIEF
"""
filename = payload.results.payload_meta.extra_data.get(
'filename', payload.results.payload_id)
try:
binary = lief.parse(raw=payload.content, name=filename)
except lief.exception as err:
raise StoqPluginException(f'Unable to parse payload: {err}')
if binary is None:
raise StoqPluginException('The file type isn\'t supported by LIEF')
if self.abstract == True:
results = lief.to_json_from_abstract(binary.abstract)
else:
results = lief.to_json(binary)
return WorkerResponse(json.loads(results))
