def _decompile(self, kwargs): # NOTE: Using kwargs is lazy but it just makes life easier! # Online if self.decomp: try: decompilation = self.decomp.start_decompilation(**kwargs) decompilation.wait_until_finished() except exceptions.RetdecError as err: raise error.CommandError("retdec-python error: {}".format(err)) return decompilation.get_hll_code() # Local else: with tempfile.NamedTemporaryFile('rb') as fp: cmd = [ path.join(self.retdec_dir, 'bin/retdec-decompiler.sh'), '-o', fp.name ] if 'sel_decomp_funcs' in kwargs: cmd += ['--select-functions', kwargs['sel_decomp_funcs']] if 'sel_decomp_ranges' in kwargs: cmd += ['--select-ranges', kwargs['sel_decomp_ranges']] cmd += [kwargs['input_file']] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if proc.returncode: raise error.CommandError("retdec error: {}".format( proc.stderr.decode())) return fp.read().decode()
def decompile(self, args, file, opts): kwargs = {'input_file': file.file_path} name = '' # Check the mode and ensure the options if args['mode'] == 'bin': if 'address_range' in args and args['address_range'] == '': del args['address_range'] if 'function_name' in args and args['function_name'] == '': del args['function_name'] if 'address_range' not in args and 'function_name' not in args: raise error.CommandError( "'address_range' or 'function_name' must be set") if 'address_range' in args and 'function_name' in args: raise error.CommandError( "'address_range' and 'function_name' are mutually exclusive" ) if 'address_range' in args: name = '{}'.format(args['address_range'].strip()) kwargs['sel_decomp_ranges'] = name.strip() elif 'function_name' in args: name = '{}'.format(args['function_name'].strip()) kwargs['sel_decomp_funcs'] = name.strip() else: raise error.CommandError( "incorrect mode specified '{}' the following are supported: 'bin'" .format(args['mode'])) return {'code': self._decompile(kwargs), 'name': name}
def hash_function_decoder(self, args, file, opts): # Validate if args['bits'] not in ['32', '64']: raise error.CommandError( 'invalid bits provided, currently supported: 32, 64') if args['technique'] not in r2_hash_func_decoder.TECHNIQUES: raise error.CommandError( 'invalid technique provided, currently supported: {}'.format( r2_hash_func_decoder.TECHNIQUES)) scale_dir = path.dirname(__file__) func_decoder = path.join(scale_dir, 'scripts/r2_hash_func_decoder.py') hash_db = path.join(scale_dir, 'scripts/r2_hash_func_decoder.db') # Get the output proc = subprocess.run([ 'python3', '{}'.format(func_decoder), 'analyse', '-f', '{}'.format( file.file_path), '-d', '{}'.format(hash_db), '{}'.format( args['technique']) ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if proc.returncode: raise error.CommandError('failed to execute script') return {'analysis': str(proc.stdout, encoding='utf-8')}
def check(self): self.trid_path = None if config.scale_configs['trid']['trid_path']: if path.exists(config.scale_configs['trid']['trid_path']): self.trid_path = config.scale_configs['trid']['trid_path'] else: self.trid_path = shutil.which("trid") if not self.trid_path: raise error.CommandError("binary 'trid' not found") # TODO: Call tridupdate.py to update defs # self.tridupdate_path = None # if config.scale_configs['trid']['tridupdate_path']: # if path.exists(config.scale_configs['trid']['tridupdate_path']): # self.tridupdate_path = config.scale_configs['trid']['tridupdate_path'] # if not self.tridupdate_path: # raise error.CommandError("file 'tridupdate.py' not found") self.triddefs_path = None if config.scale_configs['trid']['triddefs_path']: if path.exists(config.scale_configs['trid']['triddefs_path']): self.triddefs_path = config.scale_configs['trid'][ 'triddefs_path'] if not self.triddefs_path: raise error.CommandError("file 'triddefs.trd' not found")
def check(self): rule_files = [] # Check rules_path if not RULES_PATH or RULES_PATH == '': raise error.CommandError("config variable 'rules_path' not set") if not os.path.exists(RULES_PATH): raise error.CommandError( "'rules_path': '{}' does not exist".format(RULES_PATH)) if not os.listdir(RULES_PATH): raise error.CommandError( "'rules_path': '{}' is empty".format(RULES_PATH)) # Loop through the rules folder and append all of the yar[a] files for dir_path, _dirs, files in os.walk(RULES_PATH): for f in files: if f.endswith('.yar') or f.endswith('.yara'): rule_files.append(os.path.join(dir_path, f)) # Compile each of the new yara files for rule_file in rule_files: compiled_rule = os.path.join( RULES_PATH, os.path.splitext(os.path.basename(rule_file))[0] + '.yarac') if not os.path.exists(compiled_rule): try: rule = yara.compile(rule_file) rule.save(compiled_rule) except Exception as err: # pylint: disable=broad-except # TODO: Raising a CommandWarning breaks the module load, hence hacking in app_log direct warning temporarily app_log.warning( 'error with yara module when compiling rule: %s', err)
def check(self): # Check the Clam Daemon is alive and well try: clamd = pyclamd.ClamdAgnostic() if not clamd.ping(): raise error.CommandError('clamav daemon not running') except Exception: raise error.CommandError('clamav daemon not running')
def check(self): self.vol = None if config.scale_configs['volatility']['vol_path']: if path.exists(config.scale_configs['volatility']['vol_path']): self.vol = config.scale_configs['volatility']['vol_path'] else: raise error.CommandError( "binary 'vol.py' not found - 'vol_path' not set") if not self.vol: raise error.CommandError("binary 'vol.py' not found")
def hivedump(self, args, file, opts): cmd = [ 'rekall', '-f', file.file_path, 'hivedump', '--hive-offset', '%s' % (args['hive_offset']) ] if config.scale_configs['rekall']['repository_path']: cmd += [ '--repository_path', config.scale_configs['rekall']['repository_path'] ] if config.scale_configs['rekall']['cache_dir']: cmd += ['--cache_dir', config.scale_configs['rekall']['cache_dir']] env = environ.copy() if 'http_proxy' in config.snake_config.keys(): env['http_proxy'] = config.snake_config['http_proxy'] env['HTTP_PROXY'] = config.snake_config['http_proxy'] env['https_proxy'] = config.snake_config['https_proxy'] env['HTTPS_PROXY'] = config.snake_config['https_proxy'] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) if proc.returncode != 0: raise error.CommandError(proc.stderr) return {'hivedump': str(proc.stdout, encoding="utf-8")}
def kdbgscan(self, args, file, opts): proc = subprocess.run([self.vol, '-f', file.file_path, 'kdbgscan'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if proc.returncode != 0: raise error.CommandError(proc.stderr) return {'kdbgscan': str(proc.stdout, encoding="utf-8")}
def run_command(self, file, args, vol_cmd): if isinstance(vol_cmd, list): cmd = ['rekall', '-f', file.file_path] cmd += vol_cmd else: cmd = ['rekall', '-f', file.file_path, '%s' % (vol_cmd)] if config.scale_configs['rekall']['repository_path']: cmd += [ '--repository_path', config.scale_configs['rekall']['repository_path'] ] if config.scale_configs['rekall']['cache_dir']: cmd += ['--cache_dir', config.scale_configs['rekall']['cache_dir']] env = environ.copy() if 'http_proxy' in config.snake_config.keys(): env['http_proxy'] = config.snake_config['http_proxy'] env['HTTP_PROXY'] = config.snake_config['http_proxy'] env['https_proxy'] = config.snake_config['https_proxy'] env['HTTPS_PROXY'] = config.snake_config['https_proxy'] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) if proc.returncode != 0: raise error.CommandError(proc.stderr) return str(proc.stdout, encoding="utf-8")
def sections(self, args, file, opts): output = [] with open(file.file_path, 'rb') as f: try: for section in elffile.ELFFile(f).iter_sections(): s = {} # pylint: disable=invalid-name if section.name == '': s['name'] = '-' else: s['name'] = str(section.name) s['address'] = str(hex(section['sh_addr'])) s['size'] = str(hex(section['sh_size'])) s['offset'] = str(hex(section['sh_offset'])) s['type'] = str(section['sh_type'])[4:] if str(descriptions.describe_sh_flags( section['sh_flags'])) == '': s['flags'] = '-' else: s['flags'] = str( descriptions.describe_sh_flags( section['sh_flags'])) output += [s] except exceptions.ELFError as err: raise error.CommandError(str(err)) return output
def test_command_error(): """ Test the class CommandError """ err = error.CommandError('hello') assert 'hello' in err.message assert err.status_code == 500 assert None is err.payload
def get_profile(self, file): document = db.file_collection.select(file.sha256_digest) if 'profile' not in document: self.imageinfo(None, file.sha256_digest) # pylint: disable=no-value-for-parameter document = db.file_collection.select(file.sha256_digest) if 'profile' not in document: raise error.CommandError( 'Unable to automatically determine profile!') return document['profile']
def check(self): self.floss_path = None if config.scale_configs['floss']['floss_path']: if os.path.exists(config.scale_configs['floss']['floss_path']): self.floss_path = config.scale_configs['floss']['floss_path'] else: self.floss = shutil.which("floss") if not self.floss_path: raise error.CommandError("binary 'floss' not found")
def check(self): self.lifer_path = None if config.scale_configs['lifer']['lifer_path']: if os.path.exists(config.scale_configs['lifer']['lifer_path']): self.lifer_path = config.scale_configs['lifer']['lifer_path'] else: self.lifer = shutil.which("lifer") if not self.lifer_path: raise error.CommandError("binary 'lifer' not found")
def check(self): self.decomp = None self.retdec_dir = None if not shutil.which('radare2'): raise error.CommandError("binary 'radare2' not found") if config.scale_configs['retdec']['online']: if not config.scale_configs['retdec']['api_key']: raise error.CommandError( "config variable 'api_key' has not been set and is required to query the online retdec" ) self.decomp = decompiler.Decompiler( api_key=config.scale_configs['retdec']['api_key']) else: if not config.scale_configs['retdec']['retdec_dir']: raise error.CommandError( "config variable 'retdec_dir' has not been set and is required to use a local retdec instance" ) self.retdec_dir = config.scale_configs['retdec']['retdec_dir']
def run_command(self, file, args, vol_cmd): cmd = [self.vol, '-f', file.file_path, '%s' % (vol_cmd)] if 'profile' in args and args['profile'] != '': cmd += ['--profile', args['profile']] else: profile = self.get_profile(file) cmd += ['--profile', profile] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if proc.returncode != 0: raise error.CommandError(proc.stderr) return str(proc.stdout, encoding="utf-8")
def binary_carver(self, args, file, opts): sample = {} with tempfile.TemporaryDirectory(dir=path.abspath( path.expanduser( config.snake_config['cache_dir']))) as temp_dir: # Try and carve file_path = r2_bin_carver.carve(file.file_path, temp_dir, args['offset'], args['size'], args['magic_bytes']) if not file_path: raise error.CommandError('failed to carve binary') if args['patch']: if not r2_bin_carver.patch(file_path): raise error.CommandError( 'failed to patch binary, not a valid pe file') # Get file name document = db.file_collection.select(file.sha256_digest) if not document: raise error.SnakeError("failed to get sample's metadata") # Create schema and save name = '{}.{}'.format(document['name'], args['offset']) file_schema = schema.FileSchema().load({ 'name': name, 'description': 'extracted with radare2 script r2_bin_carver.py' }) new_file = fs.FileStorage() new_file.create(file_path) sample = submitter.submit(file_schema, enums.FileType.FILE, new_file, file, NAME) sample = schema.FileSchema().dump(schema.FileSchema().load( sample)) # Required to clean the above return sample
def hivedump(self, args, file, opts): cmd = [ self.vol, '-f', file.file_path, 'hivedump', '--hive-offset', '%s' % (args['hive_offset']) ] if 'profile' in args and args['profile'] != '': cmd += ['--profile', args['profile']] else: profile = self.get_profile(file) cmd += ['--profile', profile] proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if proc.returncode != 0: raise error.CommandError(proc.stderr) return {'hivedump': str(proc.stdout, encoding="utf-8")}
def extract(self, args, file, opts): samples = [] with tempfile.TemporaryDirectory(dir=path.abspath( path.expanduser( config.snake_config['cache_dir']))) as temp_dir: # Extract the samples proc = subprocess.run( [self.binwalk_path, file.file_path, '-e', '-C', temp_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if not proc: raise error.CommandError( "failed to successfully extract from sample") # Get file name document = db.file_collection.select(file.sha256_digest) if not document: raise error.SnakeError("failed to get sample's metadata") # There will be one output directory connataining files with the offsets as names contents = os.listdir(temp_dir) if not contents: return [] directory = path.join(temp_dir, contents[0]) for i in os.listdir(directory): file_path = path.join(directory, i) name = '{}.{}'.format(document['name'], i) file_schema = schema.FileSchema().load({ 'name': name, 'description': 'extracted with binwalk' }) new_file = fs.FileStorage() new_file.create(file_path) new_document = submitter.submit(file_schema, enums.FileType.FILE, new_file, file, NAME) new_document = schema.FileSchema().dump( schema.FileSchema().load( new_document)) # Required to clean the above samples += [new_document] return samples
def imageinfo(self, args, file, opts): proc = subprocess.run([self.vol, '-f', file.file_path, 'imageinfo'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if proc.returncode != 0: raise error.CommandError(proc.stderr) output = str(proc.stdout, encoding="utf-8") # Try and extract profile try: prof = output.split('\n')[0].split(':')[1] if 'suggestion' not in prof: if ',' in prof: prof = prof.split(',')[0] data = {'profile': prof.strip()} if not db.file_collection.update(file.sha256_digest, data): raise error.MongoError( 'Error adding profile into file document %s' % file.sha256_digest) except Exception: # noqa pylint: disable=broad-except pass return {'imageinfo': output}
def calculate_pehash(exe): # pylint: disable=too-many-locals, too-many-statements try: # image characteristics img_chars = bitstring.BitArray(hex(exe.FILE_HEADER.Characteristics)) # pad to 16 bits img_chars = bitstring.BitArray(bytes=img_chars.tobytes()) img_chars_xor = img_chars[0:8] ^ img_chars[8:16] # start to build pehash pehash_bin = bitstring.BitArray(img_chars_xor) # subsystem - sub_chars = bitstring.BitArray(hex(exe.FILE_HEADER.Machine)) # pad to 16 bits sub_chars = bitstring.BitArray(bytes=sub_chars.tobytes()) sub_chars_xor = sub_chars[0:8] ^ sub_chars[8:16] pehash_bin.append(sub_chars_xor) # Stack Commit Size stk_size = bitstring.BitArray( hex(exe.OPTIONAL_HEADER.SizeOfStackCommit)) stk_size_bits = stk_size.bin.zfill(32) # now xor the bits stk_size = bitstring.BitArray(bin=stk_size_bits) stk_size_xor = stk_size[8:16] ^ stk_size[16:24] ^ stk_size[24:32] # pad to 8 bits stk_size_xor = bitstring.BitArray(bytes=stk_size_xor.tobytes()) pehash_bin.append(stk_size_xor) # Heap Commit Size hp_size = bitstring.BitArray(hex(exe.OPTIONAL_HEADER.SizeOfHeapCommit)) hp_size_bits = hp_size.bin.zfill(32) # now xor the bits hp_size = bitstring.BitArray(bin=hp_size_bits) hp_size_xor = hp_size[8:16] ^ hp_size[16:24] ^ hp_size[24:32] # pad to 8 bits hp_size_xor = bitstring.BitArray(bytes=hp_size_xor.tobytes()) pehash_bin.append(hp_size_xor) # Section chars for section in exe.sections: # virtual address sect_va = bitstring.BitArray(hex(section.VirtualAddress)) sect_va = bitstring.BitArray(bytes=sect_va.tobytes()) sect_va_bits = sect_va[8:32] pehash_bin.append(sect_va_bits) # rawsize sect_rs = bitstring.BitArray(hex(section.SizeOfRawData)) sect_rs = bitstring.BitArray(bytes=sect_rs.tobytes()) sect_rs_bits = sect_rs.bin.zfill(32) sect_rs = bitstring.BitArray(bin=sect_rs_bits) sect_rs = bitstring.BitArray(bytes=sect_rs.tobytes()) sect_rs_bits = sect_rs[8:32] pehash_bin.append(sect_rs_bits) # section chars sect_chars = bitstring.BitArray(hex(section.Characteristics)) sect_chars = bitstring.BitArray(bytes=sect_chars.tobytes()) sect_chars_xor = sect_chars[16:24] ^ sect_chars[24:32] pehash_bin.append(sect_chars_xor) # entropy calulation address = section.VirtualAddress size = section.SizeOfRawData raw = exe.write()[address + size:] if size == 0: kolmog = bitstring.BitArray(float=1, length=32) pehash_bin.append(kolmog[0:8]) continue bz2_raw = bz2.compress(raw) bz2_size = len(bz2_raw) # k = round(bz2_size / size, 5) k = bz2_size / size kolmog = bitstring.BitArray(float=k, length=32) pehash_bin.append(kolmog[0:8]) sha1 = hashlib.sha1() sha1.update(pehash_bin.tobytes()) return str(sha1.hexdigest()) except Exception as err: raise error.CommandError( 'An error occurred with calculate_pehash: %s' % err)
def scan(self, args, file, opts): # pylint: disable=too-many-locals, too-many-branches compiled_rule_files = [] output = [] if 'rule' in args and args['rule'] != '': rule = args['rule'] rule = rule.strip('.yar').strip('.yara') rule += '.yarac' path = os.path.join(RULES_PATH, rule) if os.path.exists(path): compiled_rule_files.append(path) else: raise error.CommandError('rule file does not exist') else: for _root, _dirs, files in os.walk(RULES_PATH): for f in files: if f.endswith('.yarac'): compiled_rule_files.append(os.path.join(RULES_PATH, f)) # Load each of the compiled yara files for compiled_rule_file in compiled_rule_files: try: rules = yara.load(compiled_rule_file) matches = rules.match(file.file_path) except Exception: # noqa pylint: disable=broad-except continue # Skip if no rule matches if not matches: continue # If the rule index doesn't exist we are likely using the yara plugin and not yara-python if matches[0].rule is None: raise error.CommandWarning( 'incorrect yara python plugin installed') # Loop through each match and append to output for match in matches: try: if match.rule in config.scale_configs['yara'][ 'blacklisted_rules']: continue except Exception: # noqa pylint: disable=broad-except continue # Strings matches are stored as byte arrays, and whilst they can be converted to utf-8 strings, # in the case of hex values these are converted to ASCII which is not the desired output. # e.g: # b'This program cannot be run in DOS mo' = 'This program cannot be run in DOS mo' # b'\x40\x410x42' = @A0x42 output += [{ 'file': str(os.path.basename(compiled_rule_file)), 'rule': str(match.rule), 'hits': [{ 'hit': str(x[2])[2:-1], 'offset': str(x[0]) } for x in match.strings], 'description': str(match.meta['description']) if 'description' in match.meta else '', 'author': str(match.meta['author']) if 'author' in match.meta else '', }] return output
def check(self): exiftool = shutil.which("exiftool") if not exiftool: raise error.CommandError("binary 'exiftool' not found")
if PEEPDF_PATH and path.isfile(PEEPDF_PATH): has_peepdf = True # pylint: disable=invalid-name else: if PEEPDF_PATH: app_log.warning( "pdf - pdf-parser disabled - optional dependencies not met: 'peepdf' not found" ) else: app_log.warning( "pdf - pdf-parser disabled - optional dependencies not met: 'peepdf_path' not set" ) has_peepdf = False # pylint: disable=invalid-name if not (has_pdf_parser or has_pdfid or has_peepdf): raise error.CommandError("no supported pdf tools installed") class Commands(scale.Commands): def check(self): pass if has_pdfid: @scale.command({'info': 'parse the file with pdfid'}) def pdfid(self, args, file, opts): try: proc = subprocess.run(["python2", PDFID_PATH, file.file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) except Exception as err:
def check(self): self.binwalk_path = shutil.which("binwalk") if not self.binwalk_path: raise error.CommandError("binary 'binwalk' not found")
def check(self): if not shutil.which('rekall'): raise error.CommandError("binary 'rekall' not found")
def check(self): if not NSRL_PATH: raise error.CommandError('path to nsrl hashes file is not set') if not path.isfile(NSRL_PATH): raise error.CommandError('nsrl hashes file not found')