def _decompile(self, kwargs):
# NOTE: Using kwargs is lazy but it just makes life easier!
# Online
if self.decomp:
try:
decompilation = self.decomp.start_decompilation(**kwargs)
decompilation.wait_until_finished()
except exceptions.RetdecError as err:
raise error.CommandError("retdec-python error: {}".format(err))
return decompilation.get_hll_code()
# Local
else:
with tempfile.NamedTemporaryFile('rb') as fp:
cmd = [
path.join(self.retdec_dir, 'bin/retdec-decompiler.sh'),
'-o', fp.name
]
if 'sel_decomp_funcs' in kwargs:
cmd += ['--select-functions', kwargs['sel_decomp_funcs']]
if 'sel_decomp_ranges' in kwargs:
cmd += ['--select-ranges', kwargs['sel_decomp_ranges']]
cmd += [kwargs['input_file']]
proc = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if proc.returncode:
raise error.CommandError("retdec error: {}".format(
proc.stderr.decode()))
return fp.read().decode()
def decompile(self, args, file, opts):
kwargs = {'input_file': file.file_path}
name = ''
# Check the mode and ensure the options
if args['mode'] == 'bin':
if 'address_range' in args and args['address_range'] == '':
del args['address_range']
if 'function_name' in args and args['function_name'] == '':
del args['function_name']
if 'address_range' not in args and 'function_name' not in args:
raise error.CommandError(
"'address_range' or 'function_name' must be set")
if 'address_range' in args and 'function_name' in args:
raise error.CommandError(
"'address_range' and 'function_name' are mutually exclusive"
)
if 'address_range' in args:
name = '{}'.format(args['address_range'].strip())
kwargs['sel_decomp_ranges'] = name.strip()
elif 'function_name' in args:
name = '{}'.format(args['function_name'].strip())
kwargs['sel_decomp_funcs'] = name.strip()
else:
raise error.CommandError(
"incorrect mode specified '{}' the following are supported: 'bin'"
.format(args['mode']))
return {'code': self._decompile(kwargs), 'name': name}
def hash_function_decoder(self, args, file, opts):
# Validate
if args['bits'] not in ['32', '64']:
raise error.CommandError(
'invalid bits provided, currently supported: 32, 64')
if args['technique'] not in r2_hash_func_decoder.TECHNIQUES:
raise error.CommandError(
'invalid technique provided, currently supported: {}'.format(
r2_hash_func_decoder.TECHNIQUES))
scale_dir = path.dirname(__file__)
func_decoder = path.join(scale_dir, 'scripts/r2_hash_func_decoder.py')
hash_db = path.join(scale_dir, 'scripts/r2_hash_func_decoder.db')
# Get the output
proc = subprocess.run([
'python3', '{}'.format(func_decoder), 'analyse', '-f', '{}'.format(
file.file_path), '-d', '{}'.format(hash_db), '{}'.format(
args['technique'])
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if proc.returncode:
raise error.CommandError('failed to execute script')
return {'analysis': str(proc.stdout, encoding='utf-8')}
def check(self):
self.trid_path = None
if config.scale_configs['trid']['trid_path']:
if path.exists(config.scale_configs['trid']['trid_path']):
self.trid_path = config.scale_configs['trid']['trid_path']
else:
self.trid_path = shutil.which("trid")
if not self.trid_path:
raise error.CommandError("binary 'trid' not found")
# TODO: Call tridupdate.py to update defs
# self.tridupdate_path = None
# if config.scale_configs['trid']['tridupdate_path']:
# if path.exists(config.scale_configs['trid']['tridupdate_path']):
# self.tridupdate_path = config.scale_configs['trid']['tridupdate_path']
# if not self.tridupdate_path:
# raise error.CommandError("file 'tridupdate.py' not found")
self.triddefs_path = None
if config.scale_configs['trid']['triddefs_path']:
if path.exists(config.scale_configs['trid']['triddefs_path']):
self.triddefs_path = config.scale_configs['trid'][
'triddefs_path']
if not self.triddefs_path:
raise error.CommandError("file 'triddefs.trd' not found")
def check(self):
rule_files = []
# Check rules_path
if not RULES_PATH or RULES_PATH == '':
raise error.CommandError("config variable 'rules_path' not set")
if not os.path.exists(RULES_PATH):
raise error.CommandError(
"'rules_path': '{}' does not exist".format(RULES_PATH))
if not os.listdir(RULES_PATH):
raise error.CommandError(
"'rules_path': '{}' is empty".format(RULES_PATH))
# Loop through the rules folder and append all of the yar[a] files
for dir_path, _dirs, files in os.walk(RULES_PATH):
for f in files:
if f.endswith('.yar') or f.endswith('.yara'):
rule_files.append(os.path.join(dir_path, f))
# Compile each of the new yara files
for rule_file in rule_files:
compiled_rule = os.path.join(
RULES_PATH,
os.path.splitext(os.path.basename(rule_file))[0] + '.yarac')
if not os.path.exists(compiled_rule):
try:
rule = yara.compile(rule_file)
rule.save(compiled_rule)
except Exception as err: # pylint: disable=broad-except
# TODO: Raising a CommandWarning breaks the module load, hence hacking in app_log direct warning temporarily
app_log.warning(
'error with yara module when compiling rule: %s', err)
def check(self):
# Check the Clam Daemon is alive and well
try:
clamd = pyclamd.ClamdAgnostic()
if not clamd.ping():
raise error.CommandError('clamav daemon not running')
except Exception:
raise error.CommandError('clamav daemon not running')
def check(self):
self.vol = None
if config.scale_configs['volatility']['vol_path']:
if path.exists(config.scale_configs['volatility']['vol_path']):
self.vol = config.scale_configs['volatility']['vol_path']
else:
raise error.CommandError(
"binary 'vol.py' not found - 'vol_path' not set")
if not self.vol:
raise error.CommandError("binary 'vol.py' not found")
def hivedump(self, args, file, opts):
cmd = [
'rekall', '-f', file.file_path, 'hivedump', '--hive-offset',
'%s' % (args['hive_offset'])
]
if config.scale_configs['rekall']['repository_path']:
cmd += [
'--repository_path',
config.scale_configs['rekall']['repository_path']
]
if config.scale_configs['rekall']['cache_dir']:
cmd += ['--cache_dir', config.scale_configs['rekall']['cache_dir']]
env = environ.copy()
if 'http_proxy' in config.snake_config.keys():
env['http_proxy'] = config.snake_config['http_proxy']
env['HTTP_PROXY'] = config.snake_config['http_proxy']
env['https_proxy'] = config.snake_config['https_proxy']
env['HTTPS_PROXY'] = config.snake_config['https_proxy']
proc = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
if proc.returncode != 0:
raise error.CommandError(proc.stderr)
return {'hivedump': str(proc.stdout, encoding="utf-8")}
def kdbgscan(self, args, file, opts):
proc = subprocess.run([self.vol, '-f', file.file_path, 'kdbgscan'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if proc.returncode != 0:
raise error.CommandError(proc.stderr)
return {'kdbgscan': str(proc.stdout, encoding="utf-8")}
def run_command(self, file, args, vol_cmd):
if isinstance(vol_cmd, list):
cmd = ['rekall', '-f', file.file_path]
cmd += vol_cmd
else:
cmd = ['rekall', '-f', file.file_path, '%s' % (vol_cmd)]
if config.scale_configs['rekall']['repository_path']:
cmd += [
'--repository_path',
config.scale_configs['rekall']['repository_path']
]
if config.scale_configs['rekall']['cache_dir']:
cmd += ['--cache_dir', config.scale_configs['rekall']['cache_dir']]
env = environ.copy()
if 'http_proxy' in config.snake_config.keys():
env['http_proxy'] = config.snake_config['http_proxy']
env['HTTP_PROXY'] = config.snake_config['http_proxy']
env['https_proxy'] = config.snake_config['https_proxy']
env['HTTPS_PROXY'] = config.snake_config['https_proxy']
proc = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
if proc.returncode != 0:
raise error.CommandError(proc.stderr)
return str(proc.stdout, encoding="utf-8")
def sections(self, args, file, opts):
output = []
with open(file.file_path, 'rb') as f:
try:
for section in elffile.ELFFile(f).iter_sections():
s = {} # pylint: disable=invalid-name
if section.name == '':
s['name'] = '-'
else:
s['name'] = str(section.name)
s['address'] = str(hex(section['sh_addr']))
s['size'] = str(hex(section['sh_size']))
s['offset'] = str(hex(section['sh_offset']))
s['type'] = str(section['sh_type'])[4:]
if str(descriptions.describe_sh_flags(
section['sh_flags'])) == '':
s['flags'] = '-'
else:
s['flags'] = str(
descriptions.describe_sh_flags(
section['sh_flags']))
output += [s]
except exceptions.ELFError as err:
raise error.CommandError(str(err))
return output
def test_command_error():
"""
Test the class CommandError
"""
err = error.CommandError('hello')
assert 'hello' in err.message
assert err.status_code == 500
assert None is err.payload
def get_profile(self, file):
document = db.file_collection.select(file.sha256_digest)
if 'profile' not in document:
self.imageinfo(None, file.sha256_digest) # pylint: disable=no-value-for-parameter
document = db.file_collection.select(file.sha256_digest)
if 'profile' not in document:
raise error.CommandError(
'Unable to automatically determine profile!')
return document['profile']
def check(self):
self.floss_path = None
if config.scale_configs['floss']['floss_path']:
if os.path.exists(config.scale_configs['floss']['floss_path']):
self.floss_path = config.scale_configs['floss']['floss_path']
else:
self.floss = shutil.which("floss")
if not self.floss_path:
raise error.CommandError("binary 'floss' not found")
def check(self):
self.lifer_path = None
if config.scale_configs['lifer']['lifer_path']:
if os.path.exists(config.scale_configs['lifer']['lifer_path']):
self.lifer_path = config.scale_configs['lifer']['lifer_path']
else:
self.lifer = shutil.which("lifer")
if not self.lifer_path:
raise error.CommandError("binary 'lifer' not found")
def check(self):
self.decomp = None
self.retdec_dir = None
if not shutil.which('radare2'):
raise error.CommandError("binary 'radare2' not found")
if config.scale_configs['retdec']['online']:
if not config.scale_configs['retdec']['api_key']:
raise error.CommandError(
"config variable 'api_key' has not been set and is required to query the online retdec"
)
self.decomp = decompiler.Decompiler(
api_key=config.scale_configs['retdec']['api_key'])
else:
if not config.scale_configs['retdec']['retdec_dir']:
raise error.CommandError(
"config variable 'retdec_dir' has not been set and is required to use a local retdec instance"
)
self.retdec_dir = config.scale_configs['retdec']['retdec_dir']
def run_command(self, file, args, vol_cmd):
cmd = [self.vol, '-f', file.file_path, '%s' % (vol_cmd)]
if 'profile' in args and args['profile'] != '':
cmd += ['--profile', args['profile']]
else:
profile = self.get_profile(file)
cmd += ['--profile', profile]
proc = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if proc.returncode != 0:
raise error.CommandError(proc.stderr)
return str(proc.stdout, encoding="utf-8")
def binary_carver(self, args, file, opts):
sample = {}
with tempfile.TemporaryDirectory(dir=path.abspath(
path.expanduser(
config.snake_config['cache_dir']))) as temp_dir:
# Try and carve
file_path = r2_bin_carver.carve(file.file_path, temp_dir,
args['offset'], args['size'],
args['magic_bytes'])
if not file_path:
raise error.CommandError('failed to carve binary')
if args['patch']:
if not r2_bin_carver.patch(file_path):
raise error.CommandError(
'failed to patch binary, not a valid pe file')
# Get file name
document = db.file_collection.select(file.sha256_digest)
if not document:
raise error.SnakeError("failed to get sample's metadata")
# Create schema and save
name = '{}.{}'.format(document['name'], args['offset'])
file_schema = schema.FileSchema().load({
'name':
name,
'description':
'extracted with radare2 script r2_bin_carver.py'
})
new_file = fs.FileStorage()
new_file.create(file_path)
sample = submitter.submit(file_schema, enums.FileType.FILE,
new_file, file, NAME)
sample = schema.FileSchema().dump(schema.FileSchema().load(
sample)) # Required to clean the above
return sample
def hivedump(self, args, file, opts):
cmd = [
self.vol, '-f', file.file_path, 'hivedump', '--hive-offset',
'%s' % (args['hive_offset'])
]
if 'profile' in args and args['profile'] != '':
cmd += ['--profile', args['profile']]
else:
profile = self.get_profile(file)
cmd += ['--profile', profile]
proc = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if proc.returncode != 0:
raise error.CommandError(proc.stderr)
return {'hivedump': str(proc.stdout, encoding="utf-8")}
def extract(self, args, file, opts):
samples = []
with tempfile.TemporaryDirectory(dir=path.abspath(
path.expanduser(
config.snake_config['cache_dir']))) as temp_dir:
# Extract the samples
proc = subprocess.run(
[self.binwalk_path, file.file_path, '-e', '-C', temp_dir],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if not proc:
raise error.CommandError(
"failed to successfully extract from sample")
# Get file name
document = db.file_collection.select(file.sha256_digest)
if not document:
raise error.SnakeError("failed to get sample's metadata")
# There will be one output directory connataining files with the offsets as names
contents = os.listdir(temp_dir)
if not contents:
return []
directory = path.join(temp_dir, contents[0])
for i in os.listdir(directory):
file_path = path.join(directory, i)
name = '{}.{}'.format(document['name'], i)
file_schema = schema.FileSchema().load({
'name':
name,
'description':
'extracted with binwalk'
})
new_file = fs.FileStorage()
new_file.create(file_path)
new_document = submitter.submit(file_schema,
enums.FileType.FILE, new_file,
file, NAME)
new_document = schema.FileSchema().dump(
schema.FileSchema().load(
new_document)) # Required to clean the above
samples += [new_document]
return samples
def imageinfo(self, args, file, opts):
proc = subprocess.run([self.vol, '-f', file.file_path, 'imageinfo'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if proc.returncode != 0:
raise error.CommandError(proc.stderr)
output = str(proc.stdout, encoding="utf-8")
# Try and extract profile
try:
prof = output.split('\n')[0].split(':')[1]
if 'suggestion' not in prof:
if ',' in prof:
prof = prof.split(',')[0]
data = {'profile': prof.strip()}
if not db.file_collection.update(file.sha256_digest, data):
raise error.MongoError(
'Error adding profile into file document %s' %
file.sha256_digest)
except Exception: # noqa pylint: disable=broad-except
pass
return {'imageinfo': output}
def calculate_pehash(exe): # pylint: disable=too-many-locals, too-many-statements
try:
# image characteristics
img_chars = bitstring.BitArray(hex(exe.FILE_HEADER.Characteristics))
# pad to 16 bits
img_chars = bitstring.BitArray(bytes=img_chars.tobytes())
img_chars_xor = img_chars[0:8] ^ img_chars[8:16]
# start to build pehash
pehash_bin = bitstring.BitArray(img_chars_xor)
# subsystem -
sub_chars = bitstring.BitArray(hex(exe.FILE_HEADER.Machine))
# pad to 16 bits
sub_chars = bitstring.BitArray(bytes=sub_chars.tobytes())
sub_chars_xor = sub_chars[0:8] ^ sub_chars[8:16]
pehash_bin.append(sub_chars_xor)
# Stack Commit Size
stk_size = bitstring.BitArray(
hex(exe.OPTIONAL_HEADER.SizeOfStackCommit))
stk_size_bits = stk_size.bin.zfill(32)
# now xor the bits
stk_size = bitstring.BitArray(bin=stk_size_bits)
stk_size_xor = stk_size[8:16] ^ stk_size[16:24] ^ stk_size[24:32]
# pad to 8 bits
stk_size_xor = bitstring.BitArray(bytes=stk_size_xor.tobytes())
pehash_bin.append(stk_size_xor)
# Heap Commit Size
hp_size = bitstring.BitArray(hex(exe.OPTIONAL_HEADER.SizeOfHeapCommit))
hp_size_bits = hp_size.bin.zfill(32)
# now xor the bits
hp_size = bitstring.BitArray(bin=hp_size_bits)
hp_size_xor = hp_size[8:16] ^ hp_size[16:24] ^ hp_size[24:32]
# pad to 8 bits
hp_size_xor = bitstring.BitArray(bytes=hp_size_xor.tobytes())
pehash_bin.append(hp_size_xor)
# Section chars
for section in exe.sections:
# virtual address
sect_va = bitstring.BitArray(hex(section.VirtualAddress))
sect_va = bitstring.BitArray(bytes=sect_va.tobytes())
sect_va_bits = sect_va[8:32]
pehash_bin.append(sect_va_bits)
# rawsize
sect_rs = bitstring.BitArray(hex(section.SizeOfRawData))
sect_rs = bitstring.BitArray(bytes=sect_rs.tobytes())
sect_rs_bits = sect_rs.bin.zfill(32)
sect_rs = bitstring.BitArray(bin=sect_rs_bits)
sect_rs = bitstring.BitArray(bytes=sect_rs.tobytes())
sect_rs_bits = sect_rs[8:32]
pehash_bin.append(sect_rs_bits)
# section chars
sect_chars = bitstring.BitArray(hex(section.Characteristics))
sect_chars = bitstring.BitArray(bytes=sect_chars.tobytes())
sect_chars_xor = sect_chars[16:24] ^ sect_chars[24:32]
pehash_bin.append(sect_chars_xor)
# entropy calulation
address = section.VirtualAddress
size = section.SizeOfRawData
raw = exe.write()[address + size:]
if size == 0:
kolmog = bitstring.BitArray(float=1, length=32)
pehash_bin.append(kolmog[0:8])
continue
bz2_raw = bz2.compress(raw)
bz2_size = len(bz2_raw)
# k = round(bz2_size / size, 5)
k = bz2_size / size
kolmog = bitstring.BitArray(float=k, length=32)
pehash_bin.append(kolmog[0:8])
sha1 = hashlib.sha1()
sha1.update(pehash_bin.tobytes())
return str(sha1.hexdigest())
except Exception as err:
raise error.CommandError(
'An error occurred with calculate_pehash: %s' % err)
def scan(self, args, file, opts):
# pylint: disable=too-many-locals, too-many-branches
compiled_rule_files = []
output = []
if 'rule' in args and args['rule'] != '':
rule = args['rule']
rule = rule.strip('.yar').strip('.yara')
rule += '.yarac'
path = os.path.join(RULES_PATH, rule)
if os.path.exists(path):
compiled_rule_files.append(path)
else:
raise error.CommandError('rule file does not exist')
else:
for _root, _dirs, files in os.walk(RULES_PATH):
for f in files:
if f.endswith('.yarac'):
compiled_rule_files.append(os.path.join(RULES_PATH, f))
# Load each of the compiled yara files
for compiled_rule_file in compiled_rule_files:
try:
rules = yara.load(compiled_rule_file)
matches = rules.match(file.file_path)
except Exception: # noqa pylint: disable=broad-except
continue
# Skip if no rule matches
if not matches:
continue
# If the rule index doesn't exist we are likely using the yara plugin and not yara-python
if matches[0].rule is None:
raise error.CommandWarning(
'incorrect yara python plugin installed')
# Loop through each match and append to output
for match in matches:
try:
if match.rule in config.scale_configs['yara'][
'blacklisted_rules']:
continue
except Exception: # noqa pylint: disable=broad-except
continue
# Strings matches are stored as byte arrays, and whilst they can be converted to utf-8 strings,
# in the case of hex values these are converted to ASCII which is not the desired output.
# e.g:
# b'This program cannot be run in DOS mo' = 'This program cannot be run in DOS mo'
# b'\x40\x410x42' = @A0x42
output += [{
'file':
str(os.path.basename(compiled_rule_file)),
'rule':
str(match.rule),
'hits': [{
'hit': str(x[2])[2:-1],
'offset': str(x[0])
} for x in match.strings],
'description':
str(match.meta['description'])
if 'description' in match.meta else '',
'author':
str(match.meta['author'])
if 'author' in match.meta else '',
}]
return output
def check(self):
exiftool = shutil.which("exiftool")
if not exiftool:
raise error.CommandError("binary 'exiftool' not found")
if PEEPDF_PATH and path.isfile(PEEPDF_PATH):
has_peepdf = True # pylint: disable=invalid-name
else:
if PEEPDF_PATH:
app_log.warning(
"pdf - pdf-parser disabled - optional dependencies not met: 'peepdf' not found"
)
else:
app_log.warning(
"pdf - pdf-parser disabled - optional dependencies not met: 'peepdf_path' not set"
)
has_peepdf = False # pylint: disable=invalid-name
if not (has_pdf_parser or has_pdfid or has_peepdf):
raise error.CommandError("no supported pdf tools installed")
class Commands(scale.Commands):
def check(self):
pass
if has_pdfid:
@scale.command({'info': 'parse the file with pdfid'})
def pdfid(self, args, file, opts):
try:
proc = subprocess.run(["python2", PDFID_PATH, file.file_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except Exception as err:
def check(self):
self.binwalk_path = shutil.which("binwalk")
if not self.binwalk_path:
raise error.CommandError("binary 'binwalk' not found")
def check(self):
if not shutil.which('rekall'):
raise error.CommandError("binary 'rekall' not found")
def check(self):
if not NSRL_PATH:
raise error.CommandError('path to nsrl hashes file is not set')
if not path.isfile(NSRL_PATH):
raise error.CommandError('nsrl hashes file not found')
