class UploadSchema(schema.FileSchema):
"""Extends `FileSchema`."""
args = fields.Dict(required=False, default={}, missing={})
extract = fields.Bool(missing=False)
name = fields.Str(missing=None) # Override name
password = fields.Str(missing=None)
class GetSchema(schema.Schema):
"""Extends `Schema`.
Defines the valid schema for get request.
"""
args = fields.Dict(required=False, default={}, missing={})
command = fields.Str(required=False)
format = fields.Str(type=enums.Format, missing=enums.Format.JSON)
output = fields.Bool(required=False, default=True, missing=True)
sha256_digests = fields.List(fields.Str(), required=False)
scale = fields.Str(required=False)
class Commands(scale.Commands): # pylint: disable=too-many-public-methods
def check(self):
strings = shutil.which('radare2')
if not strings:
raise error.CommandWarning("binary 'radare2' not found")
return
@scale.command({
'args': {
'offset': fields.Str(required=True),
'magic_bytes': fields.Str(default=None, missing=None),
'patch': fields.Bool(default=True, missing=True),
'size': fields.Str(required=True),
},
'info':
'this function will carve binaries out of MDMP files'
})
def binary_carver(self, args, file, opts):
sample = {}
with tempfile.TemporaryDirectory(dir=path.abspath(
path.expanduser(
config.snake_config['cache_dir']))) as temp_dir:
# Try and carve
file_path = r2_bin_carver.carve(file.file_path, temp_dir,
args['offset'], args['size'],
args['magic_bytes'])
if not file_path:
raise error.CommandError('failed to carve binary')
if args['patch']:
if not r2_bin_carver.patch(file_path):
raise error.CommandError(
'failed to patch binary, not a valid pe file')
# Get file name
document = db.file_collection.select(file.sha256_digest)
if not document:
raise error.SnakeError("failed to get sample's metadata")
# Create schema and save
name = '{}.{}'.format(document['name'], args['offset'])
file_schema = schema.FileSchema().load({
'name':
name,
'description':
'extracted with radare2 script r2_bin_carver.py'
})
new_file = fs.FileStorage()
new_file.create(file_path)
sample = submitter.submit(file_schema, enums.FileType.FILE,
new_file, file, NAME)
sample = schema.FileSchema().dump(schema.FileSchema().load(
sample)) # Required to clean the above
return sample
def binary_carver_markdown(self, json):
output = md.table_header(('Name', 'SHA256 Digest', 'File Type'))
output += md.table_row(
(json['name'],
md.url(
json['sha256_digest'],
'/#/{}/{}'.format(json['file_type'],
json['sha256_digest'])), json['file_type']))
if not json.keys():
output += md.table_row(('-', '-', '-'))
return output
@scale.command({
'args': {
'bits': fields.Str(default='32', missing='32'),
'technique': fields.Str(required=True)
},
'info': 'scan shellcode for hashed functions'
})
def hash_function_decoder(self, args, file, opts):
# Validate
if args['bits'] not in ['32', '64']:
raise error.CommandError(
'invalid bits provided, currently supported: 32, 64')
if args['technique'] not in r2_hash_func_decoder.TECHNIQUES:
raise error.CommandError(
'invalid technique provided, currently supported: {}'.format(
r2_hash_func_decoder.TECHNIQUES))
scale_dir = path.dirname(__file__)
func_decoder = path.join(scale_dir, 'scripts/r2_hash_func_decoder.py')
hash_db = path.join(scale_dir, 'scripts/r2_hash_func_decoder.db')
# Get the output
proc = subprocess.run([
'python3', '{}'.format(func_decoder), 'analyse', '-f', '{}'.format(
file.file_path), '-d', '{}'.format(hash_db), '{}'.format(
args['technique'])
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if proc.returncode:
raise error.CommandError('failed to execute script')
return {'analysis': str(proc.stdout, encoding='utf-8')}
def hash_function_decoder_plaintext(self, json):
return json['analysis']
class Interface(scale.Interface):
"""
Connect to MalwareBazaar and retrieve the JSON data for the malware hash.
"""
def _malwarebazaar(self, sha256_digest, cache=True):
params = {'query': 'get_info', 'hash': sha256_digest}
document = db.file_collection.select(sha256_digest)
if 'malwarebazaar' not in document or not cache:
try:
response = requests.post(API_ENDPOINT,
data=params,
headers=HEADERS,
proxies=PROXIES,
timeout=10)
except Exception:
raise error.InterfaceWarning(
'failed to connect to MalwareBazaar')
if 'application/json' not in response.headers.get('content-type'):
raise error.InterfaceWarning(
'invalid response received from MalwareBazaar')
data = {'malwarebazaar': response.json()}
db.file_collection.update(sha256_digest, data)
document = db.file_collection.select(sha256_digest)
if not document or 'malwarebazaar' not in document:
raise error.MongoError(
'error adding malwarebazaar into file document %s' %
sha256_digest)
if str(document['malwarebazaar']['query_status']) == 'hash_not_found':
raise error.InterfaceWarning('File not present in MalwareBazaar')
if str(document['malwarebazaar']['query_status']) != 'ok':
raise error.InterfaceWarning('An unexpected error occured')
return document['malwarebazaar']
def check(self):
"""Self check are prerequisits set. API key is needed for upload."""
if not API_KEY:
raise error.InterfaceError(
'config variable \'api_key\' has not been set')
@scale.pull({
'args': {
'cache': fields.Bool(missing=True)
},
'info': 'MalwareBazaar results report'
})
def results(self, args, file, opts):
"""
Parse the JSON and return the specific data part.
"""
j = self._malwarebazaar(file.sha256_digest, cache=args['cache'])
return j['data'][0]
def results_markdown(self, json):
"""
Convert the JSON result data to Markdown.
"""
output = md.h2('General Information')
output += md.paragraph('SHA256 hash: ' + str(json['sha256_hash']))
output += md.paragraph('SHA1 hash: ' + str(json['sha1_hash']))
output += md.paragraph('MD5 hash: ' + str(json['md5_hash']))
output += md.paragraph('File name: ' + str(json['file_name']))
output += md.paragraph('Signature: ' + str(json['signature']))
output += md.paragraph('File size: ' + str(json['file_size']) +
" bytes")
output += md.paragraph('First seen: ' + str(json['first_seen']))
output += md.paragraph('Last seen: ' + str(json['last_seen']))
output += md.paragraph('File type: ' + str(json['file_type']))
output += md.paragraph('MIME type: ' + str(json['file_type_mime']))
output += md.paragraph('imphash: ' + str(json['imphash']))
output += md.paragraph('ssdeep: ' + str(json['ssdeep']))
output += md.paragraph('Delivery Method: ' +
str(json['delivery_method']))
if str(json['reporter']) == "anonymous":
reporter = "*Anonymous*"
else:
reporter = "[@"
reporter += str(json['reporter'])
reporter += "](https://twitter.com/"
reporter += str(json['reporter'])
reporter += ")"
output += md.paragraph('Reporter: ' + reporter)
output += md.h2('Intelligence')
output += md.paragraph('ClamAV: ' +
str(json['intelligence']['clamav']))
output += md.paragraph('Number of downloads: ' +
str(json['intelligence']['downloads']))
output += md.paragraph('Number of uploads: ' +
str(json['intelligence']['uploads']))
output += md.paragraph('Mail intelligence: ' +
str(json['intelligence']['mail']))
output += md.h2('File Information')
if json['file_information']:
for fileinfo in json['file_information']:
output += md.paragraph('Contect: ' + str(fileinfo['context']))
output += md.paragraph('Value: ' + str(fileinfo['value']))
comment = str(json['comment']).replace('\r', '').replace('\n', '<br>')
output += md.paragraph('Comment: ')
output += md.paragraph(comment)
taglist = ''
if not json['tags']:
taglist = 'None '
else:
for tag in json['tags']:
taglist += tag + ','
output += md.paragraph('Tags: ' + taglist[:-1])
return output
@scale.push({'info': 'submit file to MalwareBazaar'})
def submit(self, args, file, opts):
"""
Routine to submit a sample to MalwareBazaar.
"""
document = db.file_collection.select(file.sha256_digest)
with open(file.file_path, "rb") as sample:
try:
tags = []
delivery_method = "other"
data = {'tags': tags, 'delivery_method': delivery_method}
files = {
'json_data': (None, js.dumps(data), 'application/json'),
'file': (document['name'], sample)
}
response = requests.post(API_ENDPOINT,
files=files,
headers=HEADERS,
verify=True,
timeout=10)
except requests.exceptions.RequestException:
raise error.InterfaceError("Failled to connect")
json_response = response.json()
return json_response
@scale.pull({
'args': {
'cache': fields.Bool(missing=True)
},
'info': 'MalwareBazaar general info report'
})
def info(self, args, file, opts):
"""
Retrieve the JSON information for the info block.
"""
j = self._malwarebazaar(file.sha256_digest, cache=args['cache'])
output = j['data'][0]
return output
def info_markdown(self, json):
"""
Parse the JSON info block data to Markdown.
"""
output = md.table_header(('Attribute', 'Value'))
output += md.table_row(
('MB Link',
'https://bazaar.abuse.ch/sample/' + str(json['sha256_hash'])))
if str(json['reporter']) == "anonymous":
reporter = "*Anonymous*"
else:
reporter = "[@"
reporter += str(json['reporter'])
reporter += "](https://twitter.com/"
reporter += str(json['reporter'])
reporter += ")"
output += md.table_row(('Reporter', reporter))
comment = str(json['comment']).partition('\n')[0].rstrip(':\r')
output += md.table_row(('Comment', comment))
taglist = ''
if not json['tags']:
taglist = 'None '
else:
for tag in json['tags']:
taglist += tag + ','
output += md.table_row(('Tags', taglist[:-1]))
output += md.table_row(('ClamAV', str(json['intelligence']['clamav'])))
output += md.table_row(('First seen', str(json['first_seen'])))
output += md.table_row(('Last seen', str(json['last_seen'])))
return output
class Interface(scale.Interface):
def _vt_scan(self, sha256_digest, cache=True):
params = {
'apikey': API_KEY,
'resource': sha256_digest,
'allinfo': 1
}
document = db.file_collection.select(sha256_digest)
if 'vt' not in document or not cache:
try:
response = requests.get('https://www.virustotal.com/vtapi/v2/file/report',
params=params,
headers=HEADERS,
proxies=PROXIES,
timeout=10)
except Exception:
raise error.InterfaceWarning("failed to connect to VirusTotal")
if 'application/json' not in response.headers.get('content-type'):
raise error.InterfaceWarning("invalid response received from VirusTotal")
if 'response_code' not in response.json():
raise error.InterfaceWarning("unknown response from VirusTotal")
data = {'vt': response.json()}
db.file_collection.update(sha256_digest, data)
document = db.file_collection.select(sha256_digest)
if not document or 'vt' not in document:
raise error.MongoError('error adding vt into file document %s' % sha256_digest)
if document['vt']["response_code"] is 0:
raise error.InterfaceWarning("file is not present on VirusTotal")
# Check if we had public key but now its private, if so warn that cache is out of date
# NOTE: we just check for missing info variable
if IS_PRIVATE and 'first_seen' not in document['vt']:
raise error.InterfaceWarning("private key specified but no private api data in cache, please flush vt cache for sample")
return document['vt']
def check(self):
if not API_KEY:
raise error.InterfaceError("config variable 'api_key' has not been set")
@scale.pull({
'args': {
'cache': fields.Bool(missing=True)
},
'info': 'virustotal results report'
})
def results(self, args, file, opts):
j = self._vt_scan(file.sha256_digest, cache=args['cache'])
return j['scans']
def results_markdown(self, json):
scanresults = sorted(json)
score = 0
count = 0
output = 'AV Vendor | Result\r\n'
output += ':--- | :---\r\n'
cleanoutput = ''
for vendor in scanresults:
count += 1
if str(json[vendor]['detected']) == "True":
score += 1
output += vendor + ' | '
output += str(json[vendor]['result']).replace('%', r'\%')
output += ' | \r\n'
else:
cleanoutput += vendor + ' | '
cleanoutput += 'Clean'
cleanoutput += ' | \r\n'
if score < 3:
output = '**Score: ' + str(score) + '/' + str(count) + '**\r\n\r\n' + output
else:
output = '**Score: ' + str(score) + '/' + str(count) + '**\r\n\r\n' + output
output = output + cleanoutput
return output
# TODO: Do It!
# @scale.push({
# 'info': 'submit file to virustotal'
# })
# def submit(self, args, file, opts):
# # TODO: Implement
# pass
if IS_PRIVATE:
@scale.pull({
'args': {
'cache': fields.Bool(missing=True)
},
'info': 'VirusTotal general info report'
})
def info(self, args, file, opts):
j = self._vt_scan(file.sha256_digest, cache=args['cache'])
score = 0
for _, v in j['scans'].items():
if v['detected'] is True:
score += 1
output = {
'vt_link': j['permalink'],
'first_seen': j['first_seen'],
'last_seen': j['last_seen'],
'score': "%i/%i" % (score, len(j['scans'])),
'times_submitted': j['times_submitted'],
'type': j['type']
}
return output
def info_markdown(self, json):
output = md.table_header(('Attribute', 'Value'))
output += md.table_row(('VT Link', json['vt_link']))
output += md.table_row(('First Seen', json['first_seen']))
output += md.table_row(('Last Seen', json['last_seen']))
if int(json['score'].split('/')[0]) < 3:
output += md.table_row(('Score', json['score']))
else:
output += md.table_row(('Score', json['score']))
output += md.table_row(('Times Submitted', str(json['times_submitted'])))
output += md.table_row(('Type', json['type']))
return output
@scale.pull({
'args': {
'cache': fields.Bool(missing=True)
},
'info': 'VirtualTotal submission names'
})
def names(self, args, file, opts):
j = self._vt_scan(file.sha256_digest, cache=args['cache'])
return j['submission_names']
def names_markdown(self, json):
output = '| Submission Names |\r\n'
output += '| :------ |\r\n'
for name in json:
output += '| ' + name + ' |' + '\r\n'
return output
@scale.pull({
'args': {
'cache': fields.Bool(missing=True)
},
'info': 'VirusTotal associates URLs'
})
def urls(self, args, file, opts):
j = self._vt_scan(file.sha256_digest, cache=args['cache'])
return j['ITW_urls']
def urls_markdown(self, json):
output = '| Associated URLs |\r\n'
output += '| :------ |\r\n'
for url in json:
# TODO: Determine some kind of URL sanitiser for use here
output += '| ' + url.replace('http', 'hxxp') + ' |' + '\r\n'
return output
@scale.pull({
'args': {
'cache': fields.Bool(missing=True)
},
'info': 'VirtualTotal submission names'
})
def parents(self, args, file, opts):
j = self._vt_scan(file.sha256_digest, cache=args['cache'])
if 'compressed_parents' in j['additional_info']:
return j['additional_info']['compressed_parents']
return []
def parents_markdown(self, json):
output = '| Compressed Parents |\r\n'
output += '| :------ |\r\n'
for name in json:
output += '| [' + name + '](https://www.virustotal.com/#/file/' + name + '/analysis) |' + '\r\n'
if not json:
output += md.table_row(('-'))
return output
else:
@scale.pull({
'args': {
'cache': fields.Bool(missing=True)
},
'info': 'VirusTotal general info report'
})
def info(self, args, file, opts):
j = self._vt_scan(file.sha256_digest, cache=args['cache'])
score = 0
for _, v in j['scans'].items():
if v['detected'] is True:
score += 1
output = {
'vt_link': j['permalink'],
'score': "%i/%i" % (score, len(j['scans'])),
}
return output
def info_markdown(self, json):
output = md.table_header(('Attribute', 'Value'))
output += md.table_row(('VT Link', json['vt_link']))
if int(json['score'].split('/')[0]) < 3:
output += md.table_row(('Score', json['score']))
else:
output += md.table_row(('Score', json['score']))
return output
class CommandHandler(snake_handler.SnakeHandler):
"""Extends `SnakeHandler`."""
@tornadoparser.use_args({
# 'args': fields.Dict(required=False, default={}, missing={}),
'command':
fields.Str(required=True),
'format':
fields.Str(type=enums.Format, missing=enums.Format.JSON),
'output':
fields.Bool(required=False, default=True, missing=True),
'scale':
fields.Str(required=True),
'sha256_digest':
fields.Str(required=True)
})
async def get(self, data):
# NOTE: Tornado/Marshmallow does not like Dict in args, will have to parse manually
# TODO: Use marshmallow validation
if 'args' in self.request.arguments and self.request.arguments['args']:
data['args'] = json.loads(self.request.arguments['args'][0])
else:
data['args'] = {}
document = await db.async_command_collection.select(
data['sha256_digest'], data['scale'], data['command'],
data['args'])
if not document:
self.write_warning("no output for given data", 404, data)
self.finish()
return
if document['status'] == enums.Status.ERROR:
self.write_warning("%s" % document['output'], 404, data)
self.finish()
return
document = schema.CommandSchema().load(document)
output = None
if document['_output_id']:
output = await db.async_command_output_collection.get(
document['_output_id'])
try:
scale = scale_manager.get_scale(data['scale'])
commands = scale_manager.get_component(
scale, enums.ScaleComponent.COMMANDS)
if data['output']:
document['output'] = commands.snake.format(
data['format'], document['command'], output)
document['format'] = data['format']
except (SnakeError, TypeError) as err:
self.write_warning("%s" % err, 404, data)
self.finish()
return
document = schema.CommandSchema().dump(document)
self.jsonify({'command': document})
self.finish()
@tornadoparser.use_args({
'args':
fields.Dict(required=False, default={}, missing={}),
'asynchronous':
fields.Bool(required=False),
'command':
fields.Str(required=True),
'format':
fields.Str(type=enums.Format, missing=enums.Format.JSON),
'scale':
fields.Str(required=True),
'sha256_digest':
fields.Str(required=True),
'timeout':
fields.Int(required=False)
})
async def post(self, data):
# Check that there is a file for this hash
document = await db.async_file_collection.select(data['sha256_digest'])
if not document:
self.write_warning("no sample for given data", 404, data)
self.finish()
return
# Check scale support
try:
scale = scale_manager.get_scale(data['scale'],
document['file_type'])
commands = scale_manager.get_component(
scale, enums.ScaleComponent.COMMANDS)
cmd = commands.snake.command(data['command'])
except SnakeError as err:
self.write_warning("%s" % err, 404, data)
self.finish()
return
# Validate arguments as to not waste users time, yes this is also done on execution
result, args = validate_args(cmd, data['args'])
if not result:
self.write_warning(args, 422, data)
self.finish()
return
data['args'] = args
# Queue command
try:
document = await route_support.queue_command(data)
except SnakeError as err:
self.write_warning("%s" % err, 500, data)
self.finish()
return
document = schema.CommandSchema().load(document)
output = None
if document['_output_id']:
output = await db.async_command_output_collection.get(
document['_output_id'])
try:
document['output'] = commands.snake.format(data['format'],
document['command'],
output)
document['format'] = data['format']
except SnakeError as err:
self.write_warning("%s" % err, 404, data)
self.finish()
return
# Dump and finish
document = schema.CommandSchema().dump(document)
self.jsonify({"command": document})
self.finish()
class UploadFileSchema(schema.FileSchema):
"""Extends `FileSchema`."""
name = fields.Str(required=False) # Override
extract = fields.Bool(missing=False)
password = fields.Str(missing=None)