class EthDetectorTest(unittest.TestCase): """ Subclasses must assign this class variable to the class for the detector """ DETECTOR_CLASS = None def setUp(self): self.mevm = ManticoreEVM() self.mevm.register_plugin(KeepOnlyIfStorageChanges()) self.mevm.verbosity(0) self.worksp = self.mevm.workspace def tearDown(self): self.mevm = None shutil.rmtree(self.worksp) def _test(self, name, should_find, use_ctor_sym_arg=False): """ Tests DetectInvalid over the consensys benchmark suit """ mevm = self.mevm dir = os.path.join(THIS_DIR, "contracts", "detectors") filepath = os.path.join(dir, f"{name}.sol") if use_ctor_sym_arg: ctor_arg = (mevm.make_symbolic_value(), ) else: ctor_arg = () self.mevm.register_detector(self.DETECTOR_CLASS()) with self.mevm.kill_timeout(240): mevm.multi_tx_analysis( filepath, contract_name="DetectThis", args=ctor_arg, crytic_compile_args={"solc_working_dir": dir}, ) expected_findings = set( ((finding, at_init) for finding, at_init in should_find)) actual_findings = set( ((finding, at_init) for _addr, _pc, finding, at_init in mevm.global_findings)) self.assertEqual(expected_findings, actual_findings)
def manticore_verifier( source_code, contract_name, maxfail=None, maxt=3, maxcov=100, deployer=None, senders=None, psender=None, propre=r"crytic_.*", compile_args=None, outputspace_url=None, timeout=100, ): """ Verify solidity properties The results are dumped to stdout and to the workspace folder. $manticore-verifier property.sol --contract TestToken --smt.solver yices --maxt 4 # Owner account: 0xf3c67ffb8ab4cdd4d3243ad247d0641cd24af939 # Contract account: 0x6f4b51ac2eb017600e9263085cfa06f831132c72 # Sender_0 account: 0x97528a0c7c6592772231fd581e5b42125c1a2ff4 # PSender account: 0x97528a0c7c6592772231fd581e5b42125c1a2ff4 # Found 2 properties: crytic_test_must_revert, crytic_test_balance # Exploration will stop when some of the following happens: # * 4 human transaction sent # * Code coverage is greater than 100% meassured on target contract # * No more coverage was gained in the last transaction # * At least 2 different properties where found to be breakable. (1 for fail fast) # * 240 seconds pass # Starting exploration... Transactions done: 0. States: 1, RT Coverage: 0.0%, Failing properties: 0/2 Transactions done: 1. States: 2, RT Coverage: 55.43%, Failing properties: 0/2 Transactions done: 2. States: 8, RT Coverage: 80.48%, Failing properties: 1/2 Transactions done: 3. States: 30, RT Coverage: 80.48%, Failing properties: 1/2 No coverage progress. Stopping exploration. Coverage obtained 80.48%. (RT + prop) +-------------------------+------------+ | Property Named | Status | +-------------------------+------------+ | crytic_test_balance | failed (0) | | crytic_test_must_revert | passed | +-------------------------+------------+ Checkout testcases here:./mcore_6jdil7nh :param maxfail: stop after maxfail properties are failing. All if None :param maxcov: Stop after maxcov % coverage is obtained in the main contract :param maxt: Max transaction count to explore :param deployer: (optional) address of account used to deploy the contract :param senders: (optional) a list of calles addresses for the exploration :param psender: (optional) address from where the property is tested :param source_code: A filename or source code :param contract_name: The target contract name defined in the source code :param propre: A regular expression for selecting properties :param outputspace_url: where to put the extended result :param timeout: timeout in seconds :return: """ # Termination condition # Exploration will stop when some of the following happens: # * MAXTX human transaction sent # * Code coverage is greater than MAXCOV meassured on target contract # * No more coverage was gained in the last transaction # * At least MAXFAIL different properties where found to be breakable. (1 for fail fast) # Max transaction count to explore MAXTX = maxt # Max coverage % to get MAXCOV = maxcov # Max different properties fails MAXFAIL = maxfail config.get_group("smt").timeout = 120 config.get_group("smt").memory = 16384 config.get_group("evm").ignore_balance = True config.get_group("evm").oog = "ignore" print("# Welcome to manticore-verifier") # Main manticore manager object m = ManticoreEVM() # avoid all human level tx that are marked as constant (have no effect on the storage) filter_out_human_constants = FilterFunctions(regexp=r".*", depth="human", mutability="constant", include=False) m.register_plugin(filter_out_human_constants) filter_out_human_constants.disable() # Avoid automatically exploring property filter_no_crytic = FilterFunctions(regexp=propre, include=False) m.register_plugin(filter_no_crytic) filter_no_crytic.disable() # Only explore properties (at human level) filter_only_crytic = FilterFunctions(regexp=propre, depth="human", fallback=False, include=True) m.register_plugin(filter_only_crytic) filter_only_crytic.disable() # And now make the contract account to analyze # User accounts. Transactions trying to break the property are send from one # of this senders = (None, ) if senders is None else senders user_accounts = [] for n, address_i in enumerate(senders): user_accounts.append( m.create_account(balance=10**10, address=address_i, name=f"sender_{n}")) # the address used for deployment owner_account = m.create_account(balance=10**10, address=deployer, name="deployer") # the target contract account contract_account = m.solidity_create_contract( source_code, owner=owner_account, contract_name=contract_name, compile_args=compile_args, name="contract_account", ) # the address used for checking porperties checker_account = m.create_account(balance=10**10, address=psender, name="psender") print(f"# Owner account: 0x{int(owner_account):x}") print(f"# Contract account: 0x{int(contract_account):x}") for n, user_account in enumerate(user_accounts): print(f"# Sender_{n} account: 0x{int(user_account):x}") print(f"# PSender account: 0x{int(checker_account):x}") properties = {} md = m.get_metadata(contract_account) for func_hsh in md.function_selectors: func_name = md.get_abi(func_hsh)["name"] if re.match(propre, func_name): properties[func_name] = [] print( f"# Found {len(properties)} properties: {', '.join(properties.keys())}" ) if not properties: print("I am sorry I had to run the init bytecode for this.\n" "Good Bye.") return MAXFAIL = len(properties) if MAXFAIL is None else MAXFAIL tx_num = 0 # transactions count current_coverage = None # obtained coverge % new_coverage = 0.0 print(f"""# Exploration will stop when some of the following happens: # * {MAXTX} human transaction sent # * Code coverage is greater than {MAXCOV}% meassured on target contract # * No more coverage was gained in the last transaction # * At least {MAXFAIL} different properties where found to be breakable. (1 for fail fast) # * {timeout} seconds pass""") print("# Starting exploration...") print( f"Transactions done: {tx_num}. States: {m.count_ready_states()}, RT Coverage: {0.00}%, " f"Failing properties: 0/{len(properties)}") with m.kill_timeout(timeout=timeout): while not m.is_killed(): # check if we found a way to break more than MAXFAIL properties broken_properties = sum( int(len(x) != 0) for x in properties.values()) if broken_properties >= MAXFAIL: print( f"Found {broken_properties}/{len(properties)} failing properties. Stopping exploration." ) break # check if we sent more than MAXTX transaction if tx_num >= MAXTX: print(f"Max number of transactions reached ({tx_num})") break tx_num += 1 # check if we got enough coverage new_coverage = m.global_coverage(contract_account) if new_coverage >= MAXCOV: print( f"Current coverage({new_coverage}%) is greater than max allowed ({MAXCOV}%). Stopping exploration." ) break # check if we have made coverage progress in the last transaction if current_coverage == new_coverage: print(f"No coverage progress. Stopping exploration.") break current_coverage = new_coverage # Make sure we didn't time out before starting first transaction if m.is_killed(): print("Cancelled or timeout.") break # Explore all methods but the "crytic_" properties # Note: you may be tempted to get all valid function ids/hashes from the # metadata and to constrain the first 4 bytes of the calldata here. # This wont work because we also want to prevent the contract to call # crytic added methods as internal transactions filter_no_crytic.enable() # filter out crytic_porperties filter_out_human_constants.enable() # Exclude constant methods filter_only_crytic.disable( ) # Exclude all methods that are not property checks symbolic_data = m.make_symbolic_buffer(320) symbolic_value = m.make_symbolic_value() caller_account = m.make_symbolic_value(160) args = tuple( (caller_account == address_i for address_i in user_accounts)) m.constrain(OR(*args, False)) m.transaction( caller=caller_account, address=contract_account, value=symbolic_value, data=symbolic_data, ) # check if timeout was requested during the previous transaction if m.is_killed(): print("Cancelled or timeout.") break m.clear_terminated_states() # no interest in reverted states m.take_snapshot() # make a copy of all ready states print( f"Transactions done: {tx_num}. States: {m.count_ready_states()}, " f"RT Coverage: {m.global_coverage(contract_account):3.2f}%, " f"Failing properties: {broken_properties}/{len(properties)}") # check if timeout was requested while we were taking the snapshot if m.is_killed(): print("Cancelled or timeout.") break # And now explore all properties (and only the properties) filter_no_crytic.disable() # Allow crytic_porperties filter_out_human_constants.disable( ) # Allow them to be marked as constants filter_only_crytic.enable( ) # Exclude all methods that are not property checks symbolic_data = m.make_symbolic_buffer(4) m.transaction(caller=checker_account, address=contract_account, value=0, data=symbolic_data) for state in m.all_states: world = state.platform tx = world.human_transactions[-1] md = m.get_metadata(tx.address) """ A is _broken_ if: * is normal property * RETURN False OR: * property name ends with 'revert' * does not REVERT Property is considered to _pass_ otherwise """ N = constrain_to_known_func_ids(state) for func_id in map(bytes, state.solve_n(tx.data[:4], nsolves=N)): func_name = md.get_abi(func_id)["name"] if not func_name.endswith("revert"): # Property does not ends in "revert" # It must RETURN a 1 if tx.return_value == 1: # TODO: test when property STOPs return_data = ABI.deserialize( "bool", tx.return_data) testcase = m.generate_testcase( state, f"property {md.get_func_name(func_id)} is broken", only_if=AND(tx.data[:4] == func_id, return_data == 0), ) if testcase: properties[func_name].append(testcase.num) else: # property name ends in "revert" so it MUST revert if tx.result != "REVERT": testcase = m.generate_testcase( state, f"Some property is broken did not reverted.(MUST REVERTED)", only_if=tx.data[:4] == func_id, ) if testcase: properties[func_name].append(testcase.num) m.clear_terminated_states( ) # no interest in reverted states for now! m.goto_snapshot() else: print("Cancelled or timeout.") m.clear_terminated_states() m.clear_ready_states() m.clear_snapshot() if m.is_killed(): print("Exploration ended by CTRL+C or timeout") print(f"Coverage obtained {new_coverage:3.2f}%. (RT + prop)") x = PrettyTable() x.field_names = ["Property Named", "Status"] for name, testcases in sorted(properties.items()): result = "passed" if testcases: result = f"failed ({testcases[0]})" x.add_row((name, result)) print(x) m.clear_ready_states() workspace = os.path.abspath(m.workspace)[len(os.getcwd()) + 1:] print(f"Checkout testcases here:./{workspace}")
class Analyzer: def __init__(self, args): self._args = args self._main_evm = None self._test_cases = [] self._all_mutants = [] self._main_contract_results = [] self._conc_testcases = [] def _find_test_cases(self): print('Finding test cases ...') self._main_evm = ManticoreEVM(workspace_url=self._args.workspace) if self._args.quick_mode: self._args.avoid_constant = True self._args.exclude_all = True self._args.only_alive_testcases = True consts_evm = config.get_group("evm") consts_evm.oog = "ignore" consts.skip_reverts = True if consts.skip_reverts: self._main_evm.register_plugin(SkipRevertBasicBlocks()) if consts.explore_balance: self._main_evm.register_plugin(KeepOnlyIfStorageChanges()) if self._args.limit_loops: self._main_evm.register_plugin(LoopDepthLimiter()) with self._main_evm.kill_timeout(): self._main_evm.multi_tx_analysis( self._args.argv[0], contract_name=self._args.contract, tx_limit=self._args.txlimit, tx_use_coverage=not self._args.txnocoverage, tx_send_ether=not self._args.txnoether, tx_account=self._args.txaccount, tx_preconstrain=self._args.txpreconstrain, compile_args=vars(self._args), ) self._test_cases = list(self._main_evm.all_states) def _concretizing_testcases(self): for test_case_number in range(len(self._test_cases)): test_case = self._test_cases[test_case_number] print(f'Concretizing testcase {test_case_number + 1}') conc_txs = [] blockchain = test_case.platform self._main_evm.fix_unsound_symbolication(test_case) human_transactions = list(blockchain.human_transactions) for sym_tx in human_transactions: try: conc_tx = sym_tx.concretize(test_case) except: break conc_txs.append(conc_tx) if conc_txs: self._conc_testcases.append(conc_txs) def _find_mutants(self): self._all_mutants = sorted(os.listdir(self._args.argv[1])) def _run_test_case_on_mutant(self, mutant_name, conc_txs): mutant_mevm = self._run_test_case_on_contract( os.path.join(self._args.argv[1], mutant_name), conc_txs) mutant_blockchain_state = BlockChainState.create_from_evm(mutant_mevm) # terminate mutant_mevm.kill() mutant_mevm.remove_all() return mutant_blockchain_state def _run_test_cases_on_main_contract(self): for test_case_number in range(len(self._test_cases)): print( f'Start running test case {test_case_number + 1} on main contract' ) conc_txs = self._conc_testcases[test_case_number] mevm = self._run_test_case_on_contract(self._args.argv[0], conc_txs) main_blockchain_state = BlockChainState.create_from_evm(mevm) self._main_contract_results.append(main_blockchain_state) mevm.kill() mevm.remove_all() def _run_testcases_on_single_mutant(self, mutant_name, result_dict): for test_case_number in range(len(self._test_cases)): print( f'Start running test case {test_case_number + 1} on {mutant_name}' ) conc_txs = self._conc_testcases[test_case_number] mutant_blockchain_state = self._run_test_case_on_mutant( mutant_name, conc_txs) main_blockchain_state = self._main_contract_results[ test_case_number] if mutant_blockchain_state != main_blockchain_state: print( f'Mutant {mutant_name} killed by testcase {test_case_number + 1}' ) result_dict[mutant_name] = test_case_number return result_dict[mutant_name] = None @staticmethod def get_mutants_in_batch(all_mutants): mutants_num = len(all_mutants) number_of_batch = min(int(mutants_num / 10) + 1, 10) mutants_batch = [[] for _ in range(number_of_batch)] for i in range(mutants_num): mutants_batch[i % number_of_batch].append(all_mutants[i]) return mutants_batch def _run_test_cases_on_mutants(self): print(f'Running {len(self._test_cases)} testcases on mutants ...') manager = multiprocessing.Manager() result_dict = manager.dict() mutant_batchs = self.get_mutants_in_batch(self._all_mutants) workers = [] for i in range(len(mutant_batchs)): p = multiprocessing.Process( target=self._run_test_cases_on_mutants_batch, args=(mutant_batchs[i], result_dict)) workers.append(p) p.start() for p in workers: p.join() self._results = result_dict def _run_test_cases_on_mutants_batch(self, mutant_list, result_dict): for mutant in mutant_list: try: self._run_testcases_on_single_mutant(mutant, result_dict) except Exception as e: print(e) print( 'exception on running test case on mutant. continuing ...') def run(self): try: self._find_mutants() with record_manticore_time(): self._find_test_cases() self._concretizing_testcases() with record_project_time(): self._run_test_cases_on_main_contract() self._run_test_cases_on_mutants() self._print_result() global manticore_run_time, project_run_time print('manticore run time: ' + str(manticore_run_time)) print('project run time: ' + str(project_run_time)) if not self._args.no_testcases: self._main_evm.finalize( only_alive_states=self._args.only_alive_testcases) else: self._main_evm.kill() for plugin in list(self._main_evm.plugins): self._main_evm.unregister_plugin(plugin) finally: clean_dir() def _print_result(self): print('Write result in result.txt') f = open(f'result.txt', 'w') f.write('Not killed mutants:\n\n') not_killed_mutants = [ mutant_name for mutant_name, test_case_number in self._results.items() if test_case_number is None ] for mutant_name in not_killed_mutants: f.write(f'{mutant_name}\n') f.write('\n') f.write('Selected test cases\n\n') f2 = open('test_cases.txt', 'w') f2.write('All test cases:\n\n') selected_test_case_number = set([ test_case_number for test_case_number in self._results.values() if test_case_number is not None ]) for test_case_number in range(len(self._test_cases)): test_case = self._main_contract_results[test_case_number] is_selected = test_case_number in selected_test_case_number f2.write( f'------------------------- Test case {test_case_number+1} -------------------------\n\n' ) if is_selected: f2.write(f'* SELECTED\n\n') f.write( f'------------------------- Test case {test_case_number+1} -------------------------\n\n' ) f.write(str(test_case.transaction_state)) f.write('\n\n') f2.write(str(test_case.transaction_state)) f2.write('\n\n') f.close() f2.close() print('') print(f'Number of mutants: {len(self._all_mutants)}') print( f'Number of killed mutants: {len(self._all_mutants) - len(not_killed_mutants)}' ) def _run_test_case_on_contract(self, contract_code, conc_txs): m2 = ManticoreEVM() owner_account = m2.create_account( balance=10**10, name="owner", address=self._main_evm.accounts.get('owner').address) attacker_account = m2.create_account( balance=10**10, name="attacker", address=self._main_evm.accounts.get('attacker').address) try: call_args = get_argument_from_create_transaction( self._main_evm, conc_txs[0]) create_value = m2.make_symbolic_value() m2.constrain(create_value == conc_txs[0].value) contract_account = solidity_create_contract_with_zero_price( m2, contract_code, owner=owner_account, args=call_args, balance=create_value, gas=0, ) except Exception as e: return m2 for conc_tx in conc_txs[1:]: try: m2.transaction( caller=conc_tx.caller, address=contract_account, value=conc_tx.value, data=conc_tx. data, # data has all needed metadata like function id ([:4]) and argument passed to function gas=0, price=0) except Exception as e: return m2 return m2