class EthDetectorTest(unittest.TestCase):
"""
Subclasses must assign this class variable to the class for the detector
"""
DETECTOR_CLASS = None
def setUp(self):
self.mevm = ManticoreEVM()
self.mevm.register_plugin(KeepOnlyIfStorageChanges())
self.mevm.verbosity(0)
self.worksp = self.mevm.workspace
def tearDown(self):
self.mevm = None
shutil.rmtree(self.worksp)
def _test(self, name, should_find, use_ctor_sym_arg=False):
"""
Tests DetectInvalid over the consensys benchmark suit
"""
mevm = self.mevm
dir = os.path.join(THIS_DIR, "contracts", "detectors")
filepath = os.path.join(dir, f"{name}.sol")
if use_ctor_sym_arg:
ctor_arg = (mevm.make_symbolic_value(), )
else:
ctor_arg = ()
self.mevm.register_detector(self.DETECTOR_CLASS())
with self.mevm.kill_timeout(240):
mevm.multi_tx_analysis(
filepath,
contract_name="DetectThis",
args=ctor_arg,
crytic_compile_args={"solc_working_dir": dir},
)
expected_findings = set(
((finding, at_init) for finding, at_init in should_find))
actual_findings = set(
((finding, at_init)
for _addr, _pc, finding, at_init in mevm.global_findings))
self.assertEqual(expected_findings, actual_findings)
def manticore_verifier(
source_code,
contract_name,
maxfail=None,
maxt=3,
maxcov=100,
deployer=None,
senders=None,
psender=None,
propre=r"crytic_.*",
compile_args=None,
outputspace_url=None,
timeout=100,
):
""" Verify solidity properties
The results are dumped to stdout and to the workspace folder.
$manticore-verifier property.sol --contract TestToken --smt.solver yices --maxt 4
# Owner account: 0xf3c67ffb8ab4cdd4d3243ad247d0641cd24af939
# Contract account: 0x6f4b51ac2eb017600e9263085cfa06f831132c72
# Sender_0 account: 0x97528a0c7c6592772231fd581e5b42125c1a2ff4
# PSender account: 0x97528a0c7c6592772231fd581e5b42125c1a2ff4
# Found 2 properties: crytic_test_must_revert, crytic_test_balance
# Exploration will stop when some of the following happens:
# * 4 human transaction sent
# * Code coverage is greater than 100% meassured on target contract
# * No more coverage was gained in the last transaction
# * At least 2 different properties where found to be breakable. (1 for fail fast)
# * 240 seconds pass
# Starting exploration...
Transactions done: 0. States: 1, RT Coverage: 0.0%, Failing properties: 0/2
Transactions done: 1. States: 2, RT Coverage: 55.43%, Failing properties: 0/2
Transactions done: 2. States: 8, RT Coverage: 80.48%, Failing properties: 1/2
Transactions done: 3. States: 30, RT Coverage: 80.48%, Failing properties: 1/2
No coverage progress. Stopping exploration.
Coverage obtained 80.48%. (RT + prop)
+-------------------------+------------+
| Property Named | Status |
+-------------------------+------------+
| crytic_test_balance | failed (0) |
| crytic_test_must_revert | passed |
+-------------------------+------------+
Checkout testcases here:./mcore_6jdil7nh
:param maxfail: stop after maxfail properties are failing. All if None
:param maxcov: Stop after maxcov % coverage is obtained in the main contract
:param maxt: Max transaction count to explore
:param deployer: (optional) address of account used to deploy the contract
:param senders: (optional) a list of calles addresses for the exploration
:param psender: (optional) address from where the property is tested
:param source_code: A filename or source code
:param contract_name: The target contract name defined in the source code
:param propre: A regular expression for selecting properties
:param outputspace_url: where to put the extended result
:param timeout: timeout in seconds
:return:
"""
# Termination condition
# Exploration will stop when some of the following happens:
# * MAXTX human transaction sent
# * Code coverage is greater than MAXCOV meassured on target contract
# * No more coverage was gained in the last transaction
# * At least MAXFAIL different properties where found to be breakable. (1 for fail fast)
# Max transaction count to explore
MAXTX = maxt
# Max coverage % to get
MAXCOV = maxcov
# Max different properties fails
MAXFAIL = maxfail
config.get_group("smt").timeout = 120
config.get_group("smt").memory = 16384
config.get_group("evm").ignore_balance = True
config.get_group("evm").oog = "ignore"
print("# Welcome to manticore-verifier")
# Main manticore manager object
m = ManticoreEVM()
# avoid all human level tx that are marked as constant (have no effect on the storage)
filter_out_human_constants = FilterFunctions(regexp=r".*",
depth="human",
mutability="constant",
include=False)
m.register_plugin(filter_out_human_constants)
filter_out_human_constants.disable()
# Avoid automatically exploring property
filter_no_crytic = FilterFunctions(regexp=propre, include=False)
m.register_plugin(filter_no_crytic)
filter_no_crytic.disable()
# Only explore properties (at human level)
filter_only_crytic = FilterFunctions(regexp=propre,
depth="human",
fallback=False,
include=True)
m.register_plugin(filter_only_crytic)
filter_only_crytic.disable()
# And now make the contract account to analyze
# User accounts. Transactions trying to break the property are send from one
# of this
senders = (None, ) if senders is None else senders
user_accounts = []
for n, address_i in enumerate(senders):
user_accounts.append(
m.create_account(balance=10**10,
address=address_i,
name=f"sender_{n}"))
# the address used for deployment
owner_account = m.create_account(balance=10**10,
address=deployer,
name="deployer")
# the target contract account
contract_account = m.solidity_create_contract(
source_code,
owner=owner_account,
contract_name=contract_name,
compile_args=compile_args,
name="contract_account",
)
# the address used for checking porperties
checker_account = m.create_account(balance=10**10,
address=psender,
name="psender")
print(f"# Owner account: 0x{int(owner_account):x}")
print(f"# Contract account: 0x{int(contract_account):x}")
for n, user_account in enumerate(user_accounts):
print(f"# Sender_{n} account: 0x{int(user_account):x}")
print(f"# PSender account: 0x{int(checker_account):x}")
properties = {}
md = m.get_metadata(contract_account)
for func_hsh in md.function_selectors:
func_name = md.get_abi(func_hsh)["name"]
if re.match(propre, func_name):
properties[func_name] = []
print(
f"# Found {len(properties)} properties: {', '.join(properties.keys())}"
)
if not properties:
print("I am sorry I had to run the init bytecode for this.\n"
"Good Bye.")
return
MAXFAIL = len(properties) if MAXFAIL is None else MAXFAIL
tx_num = 0 # transactions count
current_coverage = None # obtained coverge %
new_coverage = 0.0
print(f"""# Exploration will stop when some of the following happens:
# * {MAXTX} human transaction sent
# * Code coverage is greater than {MAXCOV}% meassured on target contract
# * No more coverage was gained in the last transaction
# * At least {MAXFAIL} different properties where found to be breakable. (1 for fail fast)
# * {timeout} seconds pass""")
print("# Starting exploration...")
print(
f"Transactions done: {tx_num}. States: {m.count_ready_states()}, RT Coverage: {0.00}%, "
f"Failing properties: 0/{len(properties)}")
with m.kill_timeout(timeout=timeout):
while not m.is_killed():
# check if we found a way to break more than MAXFAIL properties
broken_properties = sum(
int(len(x) != 0) for x in properties.values())
if broken_properties >= MAXFAIL:
print(
f"Found {broken_properties}/{len(properties)} failing properties. Stopping exploration."
)
break
# check if we sent more than MAXTX transaction
if tx_num >= MAXTX:
print(f"Max number of transactions reached ({tx_num})")
break
tx_num += 1
# check if we got enough coverage
new_coverage = m.global_coverage(contract_account)
if new_coverage >= MAXCOV:
print(
f"Current coverage({new_coverage}%) is greater than max allowed ({MAXCOV}%). Stopping exploration."
)
break
# check if we have made coverage progress in the last transaction
if current_coverage == new_coverage:
print(f"No coverage progress. Stopping exploration.")
break
current_coverage = new_coverage
# Make sure we didn't time out before starting first transaction
if m.is_killed():
print("Cancelled or timeout.")
break
# Explore all methods but the "crytic_" properties
# Note: you may be tempted to get all valid function ids/hashes from the
# metadata and to constrain the first 4 bytes of the calldata here.
# This wont work because we also want to prevent the contract to call
# crytic added methods as internal transactions
filter_no_crytic.enable() # filter out crytic_porperties
filter_out_human_constants.enable() # Exclude constant methods
filter_only_crytic.disable(
) # Exclude all methods that are not property checks
symbolic_data = m.make_symbolic_buffer(320)
symbolic_value = m.make_symbolic_value()
caller_account = m.make_symbolic_value(160)
args = tuple(
(caller_account == address_i for address_i in user_accounts))
m.constrain(OR(*args, False))
m.transaction(
caller=caller_account,
address=contract_account,
value=symbolic_value,
data=symbolic_data,
)
# check if timeout was requested during the previous transaction
if m.is_killed():
print("Cancelled or timeout.")
break
m.clear_terminated_states() # no interest in reverted states
m.take_snapshot() # make a copy of all ready states
print(
f"Transactions done: {tx_num}. States: {m.count_ready_states()}, "
f"RT Coverage: {m.global_coverage(contract_account):3.2f}%, "
f"Failing properties: {broken_properties}/{len(properties)}")
# check if timeout was requested while we were taking the snapshot
if m.is_killed():
print("Cancelled or timeout.")
break
# And now explore all properties (and only the properties)
filter_no_crytic.disable() # Allow crytic_porperties
filter_out_human_constants.disable(
) # Allow them to be marked as constants
filter_only_crytic.enable(
) # Exclude all methods that are not property checks
symbolic_data = m.make_symbolic_buffer(4)
m.transaction(caller=checker_account,
address=contract_account,
value=0,
data=symbolic_data)
for state in m.all_states:
world = state.platform
tx = world.human_transactions[-1]
md = m.get_metadata(tx.address)
"""
A is _broken_ if:
* is normal property
* RETURN False
OR:
* property name ends with 'revert'
* does not REVERT
Property is considered to _pass_ otherwise
"""
N = constrain_to_known_func_ids(state)
for func_id in map(bytes, state.solve_n(tx.data[:4],
nsolves=N)):
func_name = md.get_abi(func_id)["name"]
if not func_name.endswith("revert"):
# Property does not ends in "revert"
# It must RETURN a 1
if tx.return_value == 1:
# TODO: test when property STOPs
return_data = ABI.deserialize(
"bool", tx.return_data)
testcase = m.generate_testcase(
state,
f"property {md.get_func_name(func_id)} is broken",
only_if=AND(tx.data[:4] == func_id,
return_data == 0),
)
if testcase:
properties[func_name].append(testcase.num)
else:
# property name ends in "revert" so it MUST revert
if tx.result != "REVERT":
testcase = m.generate_testcase(
state,
f"Some property is broken did not reverted.(MUST REVERTED)",
only_if=tx.data[:4] == func_id,
)
if testcase:
properties[func_name].append(testcase.num)
m.clear_terminated_states(
) # no interest in reverted states for now!
m.goto_snapshot()
else:
print("Cancelled or timeout.")
m.clear_terminated_states()
m.clear_ready_states()
m.clear_snapshot()
if m.is_killed():
print("Exploration ended by CTRL+C or timeout")
print(f"Coverage obtained {new_coverage:3.2f}%. (RT + prop)")
x = PrettyTable()
x.field_names = ["Property Named", "Status"]
for name, testcases in sorted(properties.items()):
result = "passed"
if testcases:
result = f"failed ({testcases[0]})"
x.add_row((name, result))
print(x)
m.clear_ready_states()
workspace = os.path.abspath(m.workspace)[len(os.getcwd()) + 1:]
print(f"Checkout testcases here:./{workspace}")
class Analyzer:
def __init__(self, args):
self._args = args
self._main_evm = None
self._test_cases = []
self._all_mutants = []
self._main_contract_results = []
self._conc_testcases = []
def _find_test_cases(self):
print('Finding test cases ...')
self._main_evm = ManticoreEVM(workspace_url=self._args.workspace)
if self._args.quick_mode:
self._args.avoid_constant = True
self._args.exclude_all = True
self._args.only_alive_testcases = True
consts_evm = config.get_group("evm")
consts_evm.oog = "ignore"
consts.skip_reverts = True
if consts.skip_reverts:
self._main_evm.register_plugin(SkipRevertBasicBlocks())
if consts.explore_balance:
self._main_evm.register_plugin(KeepOnlyIfStorageChanges())
if self._args.limit_loops:
self._main_evm.register_plugin(LoopDepthLimiter())
with self._main_evm.kill_timeout():
self._main_evm.multi_tx_analysis(
self._args.argv[0],
contract_name=self._args.contract,
tx_limit=self._args.txlimit,
tx_use_coverage=not self._args.txnocoverage,
tx_send_ether=not self._args.txnoether,
tx_account=self._args.txaccount,
tx_preconstrain=self._args.txpreconstrain,
compile_args=vars(self._args),
)
self._test_cases = list(self._main_evm.all_states)
def _concretizing_testcases(self):
for test_case_number in range(len(self._test_cases)):
test_case = self._test_cases[test_case_number]
print(f'Concretizing testcase {test_case_number + 1}')
conc_txs = []
blockchain = test_case.platform
self._main_evm.fix_unsound_symbolication(test_case)
human_transactions = list(blockchain.human_transactions)
for sym_tx in human_transactions:
try:
conc_tx = sym_tx.concretize(test_case)
except:
break
conc_txs.append(conc_tx)
if conc_txs:
self._conc_testcases.append(conc_txs)
def _find_mutants(self):
self._all_mutants = sorted(os.listdir(self._args.argv[1]))
def _run_test_case_on_mutant(self, mutant_name, conc_txs):
mutant_mevm = self._run_test_case_on_contract(
os.path.join(self._args.argv[1], mutant_name), conc_txs)
mutant_blockchain_state = BlockChainState.create_from_evm(mutant_mevm)
# terminate
mutant_mevm.kill()
mutant_mevm.remove_all()
return mutant_blockchain_state
def _run_test_cases_on_main_contract(self):
for test_case_number in range(len(self._test_cases)):
print(
f'Start running test case {test_case_number + 1} on main contract'
)
conc_txs = self._conc_testcases[test_case_number]
mevm = self._run_test_case_on_contract(self._args.argv[0],
conc_txs)
main_blockchain_state = BlockChainState.create_from_evm(mevm)
self._main_contract_results.append(main_blockchain_state)
mevm.kill()
mevm.remove_all()
def _run_testcases_on_single_mutant(self, mutant_name, result_dict):
for test_case_number in range(len(self._test_cases)):
print(
f'Start running test case {test_case_number + 1} on {mutant_name}'
)
conc_txs = self._conc_testcases[test_case_number]
mutant_blockchain_state = self._run_test_case_on_mutant(
mutant_name, conc_txs)
main_blockchain_state = self._main_contract_results[
test_case_number]
if mutant_blockchain_state != main_blockchain_state:
print(
f'Mutant {mutant_name} killed by testcase {test_case_number + 1}'
)
result_dict[mutant_name] = test_case_number
return
result_dict[mutant_name] = None
@staticmethod
def get_mutants_in_batch(all_mutants):
mutants_num = len(all_mutants)
number_of_batch = min(int(mutants_num / 10) + 1, 10)
mutants_batch = [[] for _ in range(number_of_batch)]
for i in range(mutants_num):
mutants_batch[i % number_of_batch].append(all_mutants[i])
return mutants_batch
def _run_test_cases_on_mutants(self):
print(f'Running {len(self._test_cases)} testcases on mutants ...')
manager = multiprocessing.Manager()
result_dict = manager.dict()
mutant_batchs = self.get_mutants_in_batch(self._all_mutants)
workers = []
for i in range(len(mutant_batchs)):
p = multiprocessing.Process(
target=self._run_test_cases_on_mutants_batch,
args=(mutant_batchs[i], result_dict))
workers.append(p)
p.start()
for p in workers:
p.join()
self._results = result_dict
def _run_test_cases_on_mutants_batch(self, mutant_list, result_dict):
for mutant in mutant_list:
try:
self._run_testcases_on_single_mutant(mutant, result_dict)
except Exception as e:
print(e)
print(
'exception on running test case on mutant. continuing ...')
def run(self):
try:
self._find_mutants()
with record_manticore_time():
self._find_test_cases()
self._concretizing_testcases()
with record_project_time():
self._run_test_cases_on_main_contract()
self._run_test_cases_on_mutants()
self._print_result()
global manticore_run_time, project_run_time
print('manticore run time: ' + str(manticore_run_time))
print('project run time: ' + str(project_run_time))
if not self._args.no_testcases:
self._main_evm.finalize(
only_alive_states=self._args.only_alive_testcases)
else:
self._main_evm.kill()
for plugin in list(self._main_evm.plugins):
self._main_evm.unregister_plugin(plugin)
finally:
clean_dir()
def _print_result(self):
print('Write result in result.txt')
f = open(f'result.txt', 'w')
f.write('Not killed mutants:\n\n')
not_killed_mutants = [
mutant_name
for mutant_name, test_case_number in self._results.items()
if test_case_number is None
]
for mutant_name in not_killed_mutants:
f.write(f'{mutant_name}\n')
f.write('\n')
f.write('Selected test cases\n\n')
f2 = open('test_cases.txt', 'w')
f2.write('All test cases:\n\n')
selected_test_case_number = set([
test_case_number for test_case_number in self._results.values()
if test_case_number is not None
])
for test_case_number in range(len(self._test_cases)):
test_case = self._main_contract_results[test_case_number]
is_selected = test_case_number in selected_test_case_number
f2.write(
f'------------------------- Test case {test_case_number+1} -------------------------\n\n'
)
if is_selected:
f2.write(f'* SELECTED\n\n')
f.write(
f'------------------------- Test case {test_case_number+1} -------------------------\n\n'
)
f.write(str(test_case.transaction_state))
f.write('\n\n')
f2.write(str(test_case.transaction_state))
f2.write('\n\n')
f.close()
f2.close()
print('')
print(f'Number of mutants: {len(self._all_mutants)}')
print(
f'Number of killed mutants: {len(self._all_mutants) - len(not_killed_mutants)}'
)
def _run_test_case_on_contract(self, contract_code, conc_txs):
m2 = ManticoreEVM()
owner_account = m2.create_account(
balance=10**10,
name="owner",
address=self._main_evm.accounts.get('owner').address)
attacker_account = m2.create_account(
balance=10**10,
name="attacker",
address=self._main_evm.accounts.get('attacker').address)
try:
call_args = get_argument_from_create_transaction(
self._main_evm, conc_txs[0])
create_value = m2.make_symbolic_value()
m2.constrain(create_value == conc_txs[0].value)
contract_account = solidity_create_contract_with_zero_price(
m2,
contract_code,
owner=owner_account,
args=call_args,
balance=create_value,
gas=0,
)
except Exception as e:
return m2
for conc_tx in conc_txs[1:]:
try:
m2.transaction(
caller=conc_tx.caller,
address=contract_account,
value=conc_tx.value,
data=conc_tx.
data, # data has all needed metadata like function id ([:4]) and argument passed to function
gas=0,
price=0)
except Exception as e:
return m2
return m2
