class ManticoreMultiprocessing(ManticoreBase):
_worker_type = WorkerProcess
def __init__(self, *args, **kwargs):
# This is the global manager that will handle all shared memory access
# See. https://docs.python.org/3/library/multiprocessing.html#multiprocessing.managers.SyncManager
self._manager = SyncManager()
self._manager.start(raise_signal)
# The main manticore lock. Acquire this for accessing shared objects
# THINKME: we use the same lock to access states lists and shared contexts
self._lock = self._manager.Condition()
self._killed = self._manager.Value(bool, False)
self._running = self._manager.Value(bool, False)
# List of state ids of States on storage
self._ready_states = self._manager.list()
self._terminated_states = self._manager.list()
self._busy_states = self._manager.list()
self._killed_states = self._manager.list()
self._shared_context = self._manager.dict()
self._context_value_types = {
list: self._manager.list,
dict: self._manager.dict
}
super().__init__(*args, **kwargs)
class StateTest(unittest.TestCase):
_multiprocess_can_split_ = True
def setUp(self):
if not hasattr(self, 'manager'):
self.manager = SyncManager()
self.manager.start(
lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
dirname = os.path.dirname(__file__)
l = linux.Linux(os.path.join(dirname, 'binaries', 'basic_linux_amd64'))
self.state = State(ConstraintSet(), l)
self.lock = self.manager.Condition()
def test_workspace_save_load(self):
self.state.constraints.add(True)
workspace = Workspace(self.lock, 'mem:')
id_ = workspace.save_state(self.state)
state = workspace.load_state(id_)
# Make sure our memory maps come back through serialization
for left, right in zip(sorted(self.state.mem._maps),
sorted(state.mem._maps)):
self.assertEqual(left.start, right.start)
self.assertEqual(left.end, right.end)
self.assertEqual(left.name, right.name)
# Check constraints
self.assertEqual(str(state.constraints), str(self.state.constraints))
def test_workspace_id_start_with_zero(self):
workspace = Workspace(self.lock, 'mem:')
id_ = workspace.save_state(self.state)
self.assertEquals(id_, 0)
def test_output(self):
out = ManticoreOutput('mem:')
name = 'mytest'
message = 'custom message'
out.save_testcase(self.state, name, message)
workspace = out._store._data
# Make sure names are constructed correctly
for entry, data in workspace.items():
self.assertTrue(entry.startswith(name))
if 'messages' in entry:
self.assertTrue(message in data)
keys = [x.split('.')[1] for x in workspace.keys()]
for key in self.state.platform.generate_workspace_files():
self.assertIn(key, keys)
# Make sure we log everything we should be logging
self.assertIn('smt', keys)
self.assertIn('trace', keys)
self.assertIn('messages', keys)
self.assertIn('input', keys)
self.assertIn('pkl', keys)
class ManticoreBase(Eventful):
def _manticore_single(self):
self._worker_type = WorkerSingle
class FakeLock:
def _nothing(self, *args, **kwargs):
pass
acquire = _nothing
release = _nothing
__enter__ = _nothing
__exit__ = _nothing
notify_all = _nothing
wait = _nothing
def wait_for(self, condition, *args, **kwargs):
if not condition():
raise Exception("Deadlock: Waiting for CTRL+C")
self._lock = FakeLock()
self._killed = ctypes.c_bool(False)
self._running = ctypes.c_bool(False)
self._ready_states = []
self._terminated_states = []
self._busy_states = []
self._killed_states = []
self._shared_context = {}
def _manticore_threading(self):
self._worker_type = WorkerThread
self._lock = threading.Condition()
self._killed = ctypes.c_bool(False)
self._running = ctypes.c_bool(False)
self._ready_states = []
self._terminated_states = []
self._busy_states = []
self._killed_states = []
self._shared_context = {}
def _manticore_multiprocessing(self):
def raise_signal():
signal.signal(signal.SIGINT, signal.SIG_IGN)
self._worker_type = WorkerProcess
# This is the global manager that will handle all shared memory access
# See. https://docs.python.org/3/library/multiprocessing.html#multiprocessing.managers.SyncManager
self._manager = SyncManager()
self._manager.start(raise_signal)
# The main manticore lock. Acquire this for accessing shared objects
# THINKME: we use the same lock to access states lists and shared contexts
self._lock = self._manager.Condition()
self._killed = self._manager.Value(bool, False)
self._running = self._manager.Value(bool, False)
# List of state ids of States on storage
self._ready_states = self._manager.list()
self._terminated_states = self._manager.list()
self._busy_states = self._manager.list()
self._killed_states = self._manager.list()
self._shared_context = self._manager.dict()
self._context_value_types = {
list: self._manager.list,
dict: self._manager.dict
}
# Decorators added first for convenience.
def sync(func: Callable) -> Callable: # type: ignore
"""Synchronization decorator"""
@functools.wraps(func)
def newFunction(self, *args, **kw):
with self._lock:
return func(self, *args, **kw)
return newFunction
def at_running(func: Callable) -> Callable: # type: ignore
"""Allows the decorated method to run only when manticore is actively
exploring states
"""
@functools.wraps(func)
def newFunction(self, *args, **kw):
if not self.is_running():
raise ManticoreError(
f"{func.__name__} only allowed while exploring states")
return func(self, *args, **kw)
return newFunction
def at_not_running(func: Callable) -> Callable: # type: ignore
"""Allows the decorated method to run only when manticore is NOT
exploring states
"""
@functools.wraps(func)
def newFunction(self, *args, **kw):
if self.is_running():
logger.error("Calling at running not allowed")
raise ManticoreError(
f"{func.__name__} only allowed while NOT exploring states")
return func(self, *args, **kw)
return newFunction
def only_from_main_script(func: Callable) -> Callable: # type: ignore
"""Allows the decorated method to run only from the main manticore script
"""
@functools.wraps(func)
def newFunction(self, *args, **kw):
if not self.is_main() or self.is_running():
logger.error(
"Calling from worker or forked process not allowed")
raise ManticoreError(f"{func.__name__} only allowed from main")
return func(self, *args, **kw)
return newFunction
_published_events = {
"run",
"start_worker",
"terminate_worker",
"enqueue_state",
"fork_state",
"load_state",
"terminate_state",
"kill_state",
"execute_instruction",
"terminate_execution",
}
def __init__(self,
initial_state,
workspace_url=None,
outputspace_url=None,
**kwargs):
"""
Manticore symbolically explores program states.
**Manticore phases**
Manticore has multiprocessing capabilities. Several worker processes
could be registered to do concurrent exploration of the READY states.
Manticore can be itself at different phases: STANDBY, RUNNING.
.. code-block:: none
+---------+ +---------+
----->| STANDBY +<------------->+ RUNNING |
+---------+ +----+----+
*Phase STANDBY*
Manticore starts at STANDBY with a single initial state. Here the user
can inspect, modify and generate testcases for the different states. The
workers are paused and not doing any work. Actions: run()
*Phase RUNNING*
At RUNNING the workers consume states from the READY state list and
potentially fork new states or terminate states. A RUNNING manticore can
be stopped back to STANDBY. Actions: stop()
**States and state lists**
A state contains all the information of the running program at a given
moment. State snapshots are saved to the workspace often. Internally
Manticore associates a fresh id with each saved state. The memory copy
of the state is then changed by the emulation of the specific arch.
Stored snapshots are periodically updated using: _save() and _load().
.. code-block:: none
_save +-------------+ _load
State +----------> | WORKSPACE +----------> State
+-------------+
During exploration Manticore spawns a number of temporary states that are
maintained in different lists:
.. code-block:: none
Initial
State
| +-+---{fork}-----+
| | | |
V V V |
+---------+ +---+----+ +------------+
| READY +------->| BUSY +----->| TERMINATED |
+---------+ +---+----+ +------------+
|
| +--------+
+---------------------------->| KILLED |
+--------+
At any given time a state must be at the READY, BUSY, TERMINATED or
KILLED list.
*State list: READY*
The READY list holds all the runnable states. Internally a state is
added to the READY list via method `_put_state(state)`. Workers take
states from the READY list via the `_get_state(wait=True|False)` method.
A worker mainloop will consume states from the READY list and mark them
as BUSYwhile working on them. States in the READY list can go to BUSY or
KILLED
*State list: BUSY*
When a state is selected for exploration from the READY list it is
marked as busy and put in the BUSY list. States being explored will be
constantly modified and only saved back to storage when moved out of
the BUSY list. Hence, when at BUSY the stored copy of the state will be
potentially outdated. States in the BUSY list can go to TERMINATED,
KILLED or they can be {forked} back to READY. The forking process
could involve generating new child states and removing the parent
from all the lists.
*State list: TERMINATED*
TERMINATED contains states that have reached a final condition and raised
TerminateState. Worker's mainloop simply moves the states that requested
termination to the TERMINATED list. This is a final list.
```An inherited Manticore class like ManticoreEVM could internally revive
the states in TERMINATED that pass some condition and move them back to
READY so the user can apply a following transaction.```
*State list: KILLED*
KILLED contains all the READY and BUSY states found at a cancel event.
Manticore supports interactive analysis and has a prominent event system.
A user can stop or cancel the exploration at any time. The unfinished
states caught in this situation are simply moved to their own list for
further user action. This is a final list.
:param initial_state: the initial root `State` object to start from
:param workspace_url: workspace folder name
:param outputspace_url: Folder to place final output. Defaults to workspace
:param kwargs: other kwargs, e.g.
"""
super().__init__()
random.seed(consts.seed)
{
consts.mprocessing.single: self._manticore_single,
consts.mprocessing.threading: self._manticore_threading,
consts.mprocessing.multiprocessing:
self._manticore_multiprocessing,
}[consts.mprocessing]()
if any(not hasattr(self, x) for x in (
"_worker_type",
"_lock",
"_running",
"_killed",
"_ready_states",
"_terminated_states",
"_killed_states",
"_busy_states",
"_shared_context",
)):
raise ManticoreError(
"Need to instantiate one of: ManticoreNative, ManticoreThreads.."
)
# The workspace and the output
# Manticore will use the workspace to save and share temporary states.
# Manticore will use the output to save the final reports.
# By default the output folder and the workspace folder are the same.
# Check type, default to fs:
if isinstance(workspace_url, str):
if ":" not in workspace_url:
workspace_url = f"fs:{workspace_url}"
else:
if workspace_url is not None:
raise TypeError(
f"Invalid workspace type: {type(workspace_url).__name__}")
self._workspace = Workspace(workspace_url)
# reuse the same workspace if not specified
if outputspace_url is None:
outputspace_url = workspace_url
if outputspace_url is None:
outputspace_url = f"fs:{self._workspace.uri}"
self._output = ManticoreOutput(outputspace_url)
# The set of registered plugins
# The callback methods defined in the plugin object will be called when
# the different type of events occur over an exploration.
# Note that each callback will run in a worker process and that some
# careful use of the shared context is needed.
self.plugins = set()
# Set initial root state
if not isinstance(initial_state, StateBase):
raise TypeError(
f"Invalid initial_state type: {type(initial_state).__name__}")
self._put_state(initial_state)
# Workers will use manticore __dict__ So lets spawn them last
self._workers = [
self._worker_type(id=i, manticore=self)
for i in range(consts.procs)
]
self._snapshot = None
self._main_id = os.getpid(), threading.current_thread().ident
def is_main(self):
""" True if called from the main process/script
Note: in "single" mode this is _most likely_ True """
return self._main_id == (os.getpid(), threading.current_thread().ident)
@sync
@only_from_main_script
def take_snapshot(self):
""" Copy/Duplicate/backup all ready states and save it in a snapshot.
If there is a snapshot already saved it will be overrwritten
"""
if self._snapshot is not None:
logger.info("Overwriting a snapshot of the ready states")
snapshot = []
for state_id in self._ready_states:
state = self._load(state_id)
# Re-save the state in case the user changed its data
snapshot.append(self._save(state))
self._snapshot = snapshot
@sync
@only_from_main_script
def goto_snapshot(self):
""" REMOVE current ready states and replace them with the saved states
in a snapshot """
if not self._snapshot:
raise ManticoreError("No snapshot to go to")
self.clear_ready_states()
for state_id in self._snapshot:
self._ready_states.append(state_id)
self._snapshot = None
@sync
@only_from_main_script
def clear_snapshot(self):
""" Remove any saved states """
if self._snapshot:
for state_id in self._snapshot:
self._remove(state_id)
self._snapshot = None
@sync
@at_not_running
def clear_terminated_states(self):
""" Remove all states from the terminated list """
terminated_states_ids = tuple(self._terminated_states)
for state_id in terminated_states_ids:
self._terminated_states.remove(state_id)
self._remove(state_id)
assert self.count_terminated_states() == 0
@sync
@at_not_running
def clear_ready_states(self):
""" Remove all states from the ready list """
ready_states_ids = tuple(self._ready_states)
for state_id in ready_states_ids:
self._ready_states.remove(state_id)
self._remove(state_id)
assert self.count_ready_states() == 0
def __str__(self):
return f"<{str(type(self))[8:-2]}| Alive States: {self.count_ready_states()}; Running States: {self.count_busy_states()} Terminated States: {self.count_terminated_states()} Killed States: {self.count_killed_states()} Started: {self._running.value} Killed: {self._killed.value}>"
@classmethod
def from_saved_state(cls, filename: str, *args, **kwargs):
"""
Creates a Manticore object starting from a serialized state on the disk.
:param filename: File to load the state from
:param args: Arguments forwarded to the Manticore object
:param kwargs: Keyword args forwarded to the Manticore object
:return: An instance of a subclass of ManticoreBase with the given initial state
"""
from ..utils.helpers import PickleSerializer
with open(filename, "rb") as fd:
deserialized = PickleSerializer().deserialize(fd)
return cls(deserialized, *args, **kwargs)
def _fork(self, state, expression, policy="ALL", setstate=None):
"""
Fork state on expression concretizations.
Using policy build a list of solutions for expression.
For the state on each solution setting the new state with setstate
For example if expression is a Bool it may have 2 solutions. True or False.
Parent
(expression = ??)
Child1 Child2
(expression = True) (expression = False)
setstate(True) setstate(False)
The optional setstate() function is supposed to set the concrete value
in the child state.
Parent state is removed from the busy list and the child states are added
to the ready list.
"""
assert isinstance(
expression, Expression), f"{type(expression)} is not an Expression"
if setstate is None:
def setstate(x, y):
pass
# Find a set of solutions for expression
solutions = state.concretize(expression, policy)
if not solutions:
raise ManticoreError("Forking on unfeasible constraint set")
logger.debug("Forking. Policy: %s. Values: %s", policy,
", ".join(f"0x{sol:x}" for sol in solutions))
self._publish("will_fork_state", state, expression, solutions, policy)
# Build and enqueue a state for each solution
children = []
for new_value in solutions:
with state as new_state:
new_state.constrain(expression == new_value)
# and set the PC of the new state to the concrete pc-dest
# (or other register or memory address to concrete)
setstate(new_state, new_value)
# enqueue new_state, assign new state id
new_state_id = self._put_state(new_state)
# maintain a list of children for logging purpose
children.append(new_state_id)
with self._lock:
self._busy_states.remove(state.id)
self._remove(state.id)
state._id = None
self._lock.notify_all()
self._publish("did_fork_state", new_state, expression, new_value,
policy)
logger.debug("Forking current state %r into states %r", state.id,
children)
@staticmethod
@deprecated("Use utils.log.set_verbosity instead.")
def verbosity(level):
""" Sets global verbosity level.
This will activate different logging profiles globally depending
on the provided numeric value
"""
set_verbosity(level)
# State storage
@Eventful.will_did("save_state", can_raise=False)
def _save(self, state, state_id=None):
""" Store or update a state in secondary storage under state_id.
Use a fresh id is None is provided.
:param state: A manticore State
:param state_id: if not None force state_id (overwrite)
:type state_id: int or None
:returns: the state id used
"""
state._id = self._workspace.save_state(state, state_id=state_id)
return state.id
@Eventful.will_did("load_state", can_raise=False)
def _load(self, state_id):
""" Load the state from the secondary storage
:param state_id: a estate id
:type state_id: int
:returns: the state id used
"""
if not hasattr(self, "stcache"):
self.stcache = weakref.WeakValueDictionary()
if state_id in self.stcache:
return self.stcache[state_id]
state = self._workspace.load_state(state_id, delete=False)
state._id = state_id
self.forward_events_from(state, True)
state.manticore = self
self.stcache[state_id] = state
return state
@Eventful.will_did("remove_state", can_raise=False)
def _remove(self, state_id):
""" Remove a state from secondary storage
:param state_id: a estate id
:type state_id: int
"""
if not hasattr(self, "stcache"):
self.stcache = weakref.WeakValueDictionary()
if state_id in self.stcache:
del self.stcache[state_id]
self._workspace.rm_state(state_id)
# Internal support for state lists
def _put_state(self, state):
""" This enqueues the state for exploration.
Serialize and store the state with a fresh state_id. Then add it to
the shared READY states list
+-------+
State +----- >+ READY |
+-------+
"""
state_id = self._save(state, state_id=state.id)
with self._lock:
# Enqueue it in the ready state list for processing
self._ready_states.append(state_id)
self._lock.notify_all()
return state_id
def _get_state(self, wait=False):
""" Dequeue a state form the READY list and add it to the BUSY list """
with self._lock:
# If wait is true do the conditional wait for states
if wait:
# if not more states in the queue, let's wait for some forks
while not self._ready_states and not self._killed.value:
# if a shutdown has been requested then bail
if self.is_killed():
return None # Cancelled operation
# If there are no more READY states and no more BUSY states
# there is no chance we will get any new state so raise
if not self._busy_states:
return None # There are not states
# if there ares actually some workers ready, wait for state forks
logger.debug("Waiting for available states")
self._lock.wait()
if self._killed.value:
return None
# at this point we know there is at least one element
# and we have exclusive access
assert self._ready_states
# make the choice under exclusive access to the shared ready list
# state_id = self._policy.choice(list(self._ready_states)[0])
state_id = random.choice(list(self._ready_states))
# Move from READY to BUSY
self._ready_states.remove(state_id)
self._busy_states.append(state_id)
self._lock.notify_all()
return self._load(state_id)
@sync
def _revive_state(self, state_id):
""" Send a BUSY state back to READY list
+--------+ +------+
| READY +<-------+ BUSY |
+---+----+ +------+
"""
# Move from BUSY to READY
self._busy_states.remove(state_id)
self._ready_states.append(state_id)
self._lock.notify_all()
@sync
def _terminate_state(self, state_id, delete=False):
""" Send a BUSY state to the TERMINATED list or trash it if delete is True
+------+ +------------+
| BUSY +------->+ TERMINATED |
+---+--+ +------------+
|
v
###
###
"""
# wait for a state id to be added to the ready list and remove it
if state_id not in self._busy_states:
raise ManticoreError(
"Can not terminate. State is not being analyzed")
self._busy_states.remove(state_id)
if delete:
self._remove(state_id)
else:
# add the state_id to the terminated list
self._terminated_states.append(state_id)
# wake up everyone waiting for a change in the state lists
self._lock.notify_all()
@sync
def _kill_state(self, state_id, delete=False):
""" Send a BUSY state to the KILLED list or trash it if delete is True
+------+ +--------+
| BUSY +------->+ KILLED |
+---+--+ +--------+
|
v
###
###
"""
# wait for a state id to be added to the ready list and remove it
if state_id not in self._busy_states:
raise ManticoreError(
"Can not even kill it. State is not being analyzed")
self._busy_states.remove(state_id)
if delete:
self._remove(state_id)
else:
# add the state_id to the terminated list
self._killed_states.append(state_id)
# wake up everyone waiting for a change in the state lists
self._lock.notify_all()
@sync
def kill_state(self, state, delete=False):
""" Kill a state.
A state is moved from any list to the kill list or fully
removed from secondary storage
:param state_id: a estate id
:type state_id: int
:param delete: if true remove the state from the secondary storage
:type delete: bool
"""
state_id = state.id
if state_id in self._busy_states:
self._busy_states.remove(state_id)
if state_id in self._terminated_states:
self._terminated_states.remove(state_id)
if state_id in self._ready_states:
self._ready_states.remove(state_id)
if delete:
self._remove(state_id)
else:
# add the state_id to the terminated list
self._killed_states.append(state_id)
@property # type: ignore
@sync
def ready_states(self):
"""
Iterator over ready states.
It supports state changes. State changes will be saved back at each iteration.
The state data change must be done in a loop, e.g. `for state in ready_states: ...`
as we re-save the state when the generator comes back to the function.
This means it is not possible to change the state used by Manticore with `states = list(m.ready_states)`.
"""
_ready_states = self._ready_states
for state_id in _ready_states:
state = self._load(state_id)
yield state
# Re-save the state in case the user changed its data
self._save(state, state_id=state_id)
@property
def running_states(self):
logger.warning(
"manticore.running_states is deprecated! (You probably want manticore.ready_states)"
)
return self.ready_states
@property # type: ignore
@sync
def terminated_states(self):
"""
Iterates over the terminated states.
See also `ready_states`.
"""
for state_id in self._terminated_states:
state = self._load(state_id)
yield state
# Re-save the state in case the user changed its data
self._save(state, state_id=state_id)
@property # type: ignore
@sync
@at_not_running
def killed_states(self):
"""
Iterates over the cancelled/killed states.
See also `ready_states`.
"""
for state_id in self._killed_states:
state = self._load(state_id)
yield state
# Re-save the state in case the user changed its data
self._save(state, state_id=state_id)
@property # type: ignore
@sync
@at_not_running
def _all_states(self):
""" Only allowed at not running.
(At running we can have states at busy)
Returns a tuple with all active state ids.
Notably the "killed" states are not included here.
"""
return tuple(self._ready_states) + tuple(self._terminated_states)
@property # type: ignore
@sync
def all_states(self):
"""
Iterates over the all states (ready and terminated)
It holds a lock so no changes state lists are allowed
Notably the cancelled states are not included here.
See also `ready_states`.
"""
for state_id in self._all_states:
state = self._load(state_id)
yield state
# Re-save the state in case the user changed its data
self._save(state, state_id=state_id)
@sync
def count_states(self):
""" Total states count """
return len(self._all_states)
@sync
def count_all_states(self):
""" Total states count """
return self.count_states()
@sync
def count_ready_states(self):
""" Ready states count """
return len(self._ready_states)
@sync
def count_busy_states(self):
""" Busy states count """
return len(self._busy_states)
@sync
def count_killed_states(self):
""" Cancelled states count """
return len(self._killed_states)
@sync
def count_terminated_states(self):
""" Terminated states count """
return len(self._terminated_states)
def generate_testcase(self,
state,
message: str = "test",
name: str = "test") -> Testcase:
if message == "test" and hasattr(
state, "_terminated_by") and state._terminated_by:
message = str(state._terminated_by)
testcase = self._output.testcase(prefix=name)
with testcase.open_stream("pkl", binary=True) as statef:
PickleSerializer().serialize(state, statef)
# Let the plugins generate a state based report
for p in self.plugins:
p.generate_testcase(state, testcase, message)
logger.info("Generated testcase No. %d - %s", testcase.num, message)
return testcase
@at_not_running
def register_plugin(self, plugin: Plugin):
# Global enumeration of valid events
assert isinstance(plugin, Plugin)
assert plugin not in self.plugins, "Plugin instance already registered"
assert getattr(plugin, "manticore",
None) is None, "Plugin instance already owned"
plugin.manticore = self
self.plugins.add(plugin)
events = Eventful.all_events()
prefix = Eventful.prefixes
all_events = [x + y for x, y in itertools.product(prefix, events)]
for event_name in all_events:
callback_name = f"{event_name}_callback"
callback = getattr(plugin, callback_name, None)
if callback is not None:
self.subscribe(event_name, callback)
# Safety checks
for callback_name in dir(plugin):
if callback_name.endswith("_callback"):
event_name = callback_name[:-9]
if event_name not in all_events:
logger.warning(
"There is no event named %s for callback on plugin %s",
event_name,
type(plugin).__name__,
)
for event_name in all_events:
for plugin_method_name in dir(plugin):
if event_name in plugin_method_name:
if not plugin_method_name.endswith("_callback"):
if (plugin_method_name.startswith("on_")
or plugin_method_name.startswith("will_")
or plugin_method_name.startswith("did_")):
logger.warning(
"Plugin methods named '%s()' should end with '_callback' on plugin %s",
plugin_method_name,
type(plugin).__name__,
)
if (plugin_method_name.endswith("_callback")
and not plugin_method_name.startswith("on_")
and not plugin_method_name.startswith("will_")
and not plugin_method_name.startswith("did_")):
logger.warning(
"Plugin methods named '%s()' should start with 'on_', 'will_' or 'did_' on plugin %s",
plugin_method_name,
type(plugin).__name__,
)
plugin.on_register()
return plugin
@at_not_running
def unregister_plugin(self, plugin):
""" Removes a plugin from manticore.
No events should be sent to it after
"""
assert plugin in self.plugins, "Plugin instance not registered"
plugin.on_unregister()
self.plugins.remove(plugin)
plugin.manticore = None
def subscribe(self, name, callback):
""" Register a callback to an event"""
from types import MethodType
if not isinstance(callback, MethodType):
callback = MethodType(callback, self)
super().subscribe(name, callback)
@property # type: ignore
@at_not_running
def context(self):
""" Convenient access to shared context. We maintain a local copy of the
share context during the time manticore is not running.
This local context is copied to the shared context when a run starts
and copied back when a run finishes
"""
return self._shared_context
@contextmanager
def locked_context(self, key=None, value_type=list):
"""
A context manager that provides safe parallel access to the global
Manticore context. This should be used to access the global Manticore
context when parallel analysis is activated. Code within the `with` block
is executed atomically, so access of shared variables should occur within.
Example use::
with m.locked_context() as context:
visited['visited'].append(state.cpu.PC)
Optionally, parameters can specify a key and type for the object paired to this key.::
with m.locked_context('feature_list', list) as feature_list:
feature_list.append(1)
Note: If standard (non-proxy) list or dict objects are contained in a
referent, modifications to those mutable values will not be propagated
through the manager because the proxy has no way of knowing when the
values contained within are modified. However, storing a value in a
container proxy (which triggers a __setitem__ on the proxy object) does
propagate through the manager and so to effectively modify such an item,
one could re-assign the modified value to the container proxy:
:param object key: Storage key
:param value_type: type of value associated with key
:type value_type: list or dict or set
"""
with self._lock:
if key is None:
# If no key is provided we yield the raw shared context under a lock
yield self._shared_context
else:
# if a key is provided we yield the specific value or a fresh one
if value_type not in (list, dict):
raise TypeError("Type must be list or dict")
if hasattr(self, "_context_value_types"):
value_type = self._context_value_types[value_type]
context = self._shared_context
if key not in context:
context[key] = value_type()
yield context[key]
############################################################################
# Public API
@sync
def wait(self, condition):
""" Waits for the condition callable to return True """
self._lock.wait_for(condition)
@sync
def kill(self):
""" Attempt to cancel and kill all the workers.
Workers must terminate
RUNNING, STANDBY -> KILLED
"""
self._publish("will_terminate_execution", self._output)
self._killed.value = True
self._lock.notify_all()
self._publish("did_terminate_execution", self._output)
def terminate(self):
logger.warning(
"manticore.terminate is deprecated (Use manticore.kill)")
self.kill()
@sync
def is_running(self):
""" True if workers are exploring BUSY states or waiting for READY states """
# If there are still states in the BUSY list then the STOP/KILL event
# was not yet answered
# We know that BUSY states can only decrease after a stop is requested
return self._running.value
@sync
def is_killed(self):
""" True if workers are killed. It is safe to join them """
# If there are still states in the BUSY list then the STOP/KILL event
# was not yet answered
# We know that BUSY states can only decrease after a kill is requested
return self._killed.value
@property
def workspace(self):
return self._output.store.uri
@contextmanager
def kill_timeout(self, timeout=None):
""" A convenient context manager that will kill a manticore run after
timeout seconds
"""
if timeout is None:
timeout = consts.timeout
# Run forever is timeout is negative
if timeout <= 0:
try:
yield
finally:
return
# THINKME kill grabs the lock. Is npt this a deadlock hazard?
timer = threading.Timer(timeout, self.kill)
timer.start()
try:
yield
finally:
timer.cancel()
@at_not_running
def run(self):
"""
Runs analysis.
"""
# Delete state cache
# The cached version of a state may get out of sync if a worker in a
# different process modifies the state
self.stcache = weakref.WeakValueDictionary()
# Lazy process start. At the first run() the workers are not forked.
# This actually starts the worker procs/threads
if self.subscribe:
# User subscription to events is disabled from now on
self.subscribe = None
self._publish("will_run", self.ready_states)
self._running.value = True
# start all the workers!
for w in self._workers:
w.start()
# Main process. Lets just wait and capture CTRL+C at main
with WithKeyboardInterruptAs(self.kill):
with self._lock:
while (self._busy_states
or self._ready_states) and not self._killed.value:
self._lock.wait()
# Join all the workers!
for w in self._workers:
w.join()
with self._lock:
assert not self._busy_states and not self._ready_states or self._killed.value
if self.is_killed():
logger.debug(
"Killed. Moving all remaining ready states to killed list")
# move all READY to KILLED:
while self._ready_states:
self._killed_states.append(self._ready_states.pop())
self._running.value = False
self._publish("did_run")
assert not self.is_running()
@sync
@at_not_running
def remove_all(self):
"""
Deletes all streams from storage and clean state lists
"""
for state_id in self._all_states:
self._remove(state_id)
del self._ready_states[:]
del self._busy_states[:]
del self._terminated_states[:]
del self._killed_states[:]
def finalize(self):
"""
Generate a report testcase for every state in the system and remove
all temporary files/streams from the workspace
"""
self.kill()
for state in self.all_states:
self.generate_testcase(state)
self.remove_all()
############################################################################
############################################################################
############################################################################
############################################################################
############################################################################
############################################################################
def save_run_data(self):
with self._output.save_stream("command.sh") as f:
f.write(" ".join(map(shlex.quote, sys.argv)))
with self._output.save_stream("manticore.yml") as f:
config.save(f)
logger.info("Results in %s", self._output.store.uri)
def run(self, shuffle=False):
"""
Runs all experiments. Blocks until all experiment are finished.
"""
# Setup multiprocessing logging
manager = SyncManager()
manager.start(lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
result_log_queue = manager.Queue()
self.cancel_experiments = manager.Value('b', 0)
self.interrupt_condition = manager.Condition()
listener = multiprocessing.Process(target=result_log_listener,
args=(result_log_queue, setup_result_logger, self.result_log_name,))
listener.start()
# Setup callback throttling
result_lock = Lock()
callback_lock = Lock()
self.callback_disabled = False
def call_callback(experiment_id=None, pause=0):
with callback_lock:
with result_lock:
if pause:
output_status(prefix='C')
self.last_callback = datetime.now()
self.save_results()
sys.stdout.write('Results saved. ')
sys.stdout.flush()
if not self.callback_disabled:
sys.stdout.write('Digesting results ... ')
sys.stdout.flush()
try:
self.update_callback(experiment_id)
except Exception as ex: # pylint: disable=W
sys.stdout.write('errored with {}\n\n\n'.format(ex.__class__))
self.callback_disabled = True
else:
sys.stdout.write('done\n')
else:
sys.stdout.write('Digestion disabled, due to previous exception.\n')
self.last_callback = datetime.now()
# setup process pool
self.jobs_total = len(self.experiments)
start_time = datetime.now()
print("Using up to %i CPUs %s" %
(self.cpu_limit,
'with numpy multi-threading disabled' if os.environ.get('OMP_NUM_THREADS', None) == '1' else ''))
with multiprocessing.Pool(self.cpu_limit) as pool:
# print status function
def output_status(prefix='F'):
progress = self.jobs_finished / self.jobs_total
elapsed_time = datetime.now() - start_time
errors = "" if self.jobs_errored == 0 else " %i ERRORED, " % self.jobs_errored
sys.stdout.write(
("%s %s: %i jobs, %i finished, %i queued," + errors + " %.0f%%, ~remaining: %s\n") %
(
prefix,
datetime.now().strftime('%c'),
self.jobs_total,
self.jobs_finished,
self.jobs_total - self.jobs_finished,
progress * 100,
timedelta(seconds=(elapsed_time * (1 - progress) / progress).total_seconds() // 15 * 15)
if progress > 0 else '???',
)
)
# define callbacks, they are run within the main process, but in separate threads
def update_status(result=None):
from pandas import DataFrame
with result_lock:
self.jobs_finished += 1
output_status()
if not result:
return
row = {}
experiment = self.experiments[result.experiment_id]
row.update({
'experiment_id': result.experiment_id,
'experiment_hash': experiment.hash,
'experiment': experiment.__class__.__name__,
})
row.update(experiment.parameters._asdict())
row.update(result._asdict())
self.results = self.results.append(DataFrame([row]), sort=True)
# If there is already a callback waiting, we will replace it and therefore cancel it
if self.next_callback and self.next_callback.is_alive():
sleep(0) # let other threads run first
if callback_lock.acquire(blocking=False):
self.next_callback.cancel()
callback_lock.release()
else:
# the callback is currently waiting for the result_lock
return
# Schedule callback either immediately (0) or after the pause expired
pause = max(0.0, self.update_callback_min_pause - (datetime.now() - self.last_callback).total_seconds())
self.next_callback = Timer(
pause,
call_callback,
args=[result.experiment_id, pause],
)
self.next_callback.start()
def update_status_error(exception):
if isinstance(exception, ExperimentCanceledException):
return
print('Experiment exception: ', exception, file=sys.stderr)
traceback.print_exception(type(exception), exception, exception.__traceback__, file=sys.stderr)
self.jobs_errored += 1
self.exceptions.append(exception)
update_status()
# randomize order
experiments = list(self.experiments.values())
if shuffle:
random.seed(0xdeadbeef)
random.shuffle(experiments)
# filter loaded experiments
if not self.results.empty:
known_hashes = [ex.hash for ex in experiments]
len_before = len(experiments)
loaded_experiment_hashes = self.results.loc[:, ['experiment_hash']].values[:, 0]
experiments = [ex for ex in experiments if ex.hash not in loaded_experiment_hashes]
if loaded_experiment_hashes.size:
print('Continuing from %s' % self.results_file)
self.jobs_finished = len_before - len(experiments)
# check for experiments with results that we don't know
unknown_experiments = self.results.loc[~self.results['experiment_hash'].isin(known_hashes)]
if not unknown_experiments.empty:
print('@' * 80)
print('Results file %s contains %i results that are not in the study\'s' %
(self.results_file, len(unknown_experiments)))
print('experiment definition. Did you delete experiments from your study?')
print('@' * 80)
# experiment execution
for experiment in experiments:
# Assign experiment to GPU (if used) : might be replaced by more sophisticated load balancer
if self.gpu_limit > 0:
gpu_num = self.gpu_counter % self.gpu_limit
experiment.assign_to_gpu(gpu_num)
self.gpu_counter += 1
# Add experiment to execution queue
pool.apply_async(
experiment.execute,
(result_log_queue, self.result_log_name, self.cancel_experiments, self.interrupt_condition),
callback=update_status,
error_callback=update_status_error,
)
def signal_handler(_sig, _frame):
self.num_int += 1
if self.num_int > 1:
print("Killing all processes.")
sys.exit(1)
print(
"\rPerforming graceful shutdown... (Press CTRL-C again to force. This might result in data loss.)")
with self.interrupt_condition:
self.cancel_experiments.value = 1
self.interrupt_condition.notify_all()
signal.signal(signal.SIGINT, signal_handler)
# show status, then block until we're ready
output_status()
pool.close()
pool.join()
if self.next_callback and self.next_callback.is_alive():
with callback_lock:
self.next_callback.cancel()
call_callback()
# quit logger
result_log_queue.put(None) # trigger listener to quit
listener.join()
# check if we got any exceptions as results
if self.exceptions:
raise FailedExperimentsException(self.exceptions)
class Executor(Eventful):
'''
The executor guides the execution of a single state, handles state forking
and selection, maintains run statistics and handles all exceptional
conditions (system calls, memory faults, concretization, etc.)
'''
_published_events = {'enqueue_state', 'generate_testcase', 'fork_state', 'load_state', 'terminate_state'}
def __init__(self, initial=None, store=None, policy='random', context=None, **kwargs):
super().__init__(**kwargs)
# Signals / Callbacks handlers will be invoked potentially at different
# worker processes. State provides a local context to save data.
self.subscribe('did_load_state', self._register_state_callbacks)
# This is the global manager that will handle all shared memory access among workers
self.manager = SyncManager()
self.manager.start(lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
# The main executor lock. Acquire this for accessing shared objects
self._lock = self.manager.Condition()
# Shutdown Event
self._shutdown = self.manager.Event()
# States on storage. Shared dict state name -> state stats
self._states = self.manager.list()
# Number of currently running workers. Initially no running workers
self._running = self.manager.Value('i', 0)
self._workspace = Workspace(self._lock, store)
# Executor wide shared context
if context is None:
context = {}
self._shared_context = self.manager.dict(context)
# scheduling priority policy (wip)
# Set policy
policies = {'random': Random,
'uncovered': Uncovered,
'branchlimited': BranchLimited,
}
self._policy = policies[policy](self)
assert isinstance(self._policy, Policy)
if self.load_workspace():
if initial is not None:
logger.error("Ignoring initial state")
else:
if initial is not None:
self.add(initial)
def __del__(self):
self.manager.shutdown()
@contextmanager
def locked_context(self, key=None, default=dict):
''' Executor context is a shared memory object. All workers share this.
It needs a lock. Its used like this:
with executor.context() as context:
visited = context['visited']
visited.append(state.cpu.PC)
context['visited'] = visited
'''
assert default in (list, dict, set)
with self._lock:
if key is None:
yield self._shared_context
else:
sub_context = self._shared_context.get(key, None)
if sub_context is None:
sub_context = default()
yield sub_context
self._shared_context[key] = sub_context
def _register_state_callbacks(self, state, state_id):
'''
Install forwarding callbacks in state so the events can go up.
Going up, we prepend state in the arguments.
'''
# Forward all state signals
self.forward_events_from(state, True)
def enqueue(self, state):
'''
Enqueue state.
Save state on storage, assigns an id to it, then add it to the
priority queue
'''
# save the state to secondary storage
state_id = self._workspace.save_state(state)
self.put(state_id)
self._publish('did_enqueue_state', state_id, state)
return state_id
def load_workspace(self):
# Browse and load states in a workspace in case we are trying to
# continue from paused run
loaded_state_ids = self._workspace.try_loading_workspace()
if not loaded_state_ids:
return False
for id in loaded_state_ids:
self._states.append(id)
return True
###############################################
# Synchronization helpers
@sync
def _notify_start_run(self):
# notify siblings we are about to start a run()
self._running.value += 1
@sync
def _notify_stop_run(self):
# notify siblings we are about to stop this run()
self._running.value -= 1
if self._running is None or self._running.value < 0:
raise SystemExit
self._lock.notify_all()
################################################
# Public API
@property
def running(self):
''' Report an estimate of how many workers are currently running '''
return self._running.value
def shutdown(self):
''' This will stop all workers '''
self._shutdown.set()
def is_shutdown(self):
''' Returns True if shutdown was requested '''
return self._shutdown.is_set()
###############################################
# Priority queue
@sync
def put(self, state_id):
''' Enqueue it for processing '''
self._states.append(state_id)
self._lock.notify_all()
return state_id
@sync
def get(self):
''' Dequeue a state with the max priority '''
# A shutdown has been requested
if self.is_shutdown():
return None
# if not more states in the queue, let's wait for some forks
while len(self._states) == 0:
# if no worker is running, bail out
if self.running == 0:
return None
# if a shutdown has been requested, bail out
if self.is_shutdown():
return None
# if there ares actually some workers running, wait for state forks
logger.debug("Waiting for available states")
self._lock.wait()
state_id = self._policy.choice(list(self._states))
if state_id is None:
return None
del self._states[self._states.index(state_id)]
return state_id
def list(self):
''' Returns the list of states ids currently queued '''
return list(self._states)
def generate_testcase(self, state, message='Testcase generated'):
'''
Simply announce that we're going to generate a testcase. Actual generation
should be handled by the driver class (such as :class:`~manticore.Manticore`)
:param state: The state to generate information about
:param message: Accompanying message
'''
# broadcast test generation. This is the time for other modules
# to output whatever helps to understand this testcase
self._publish('will_generate_testcase', state, 'test', message)
def fork(self, state, expression, policy='ALL', setstate=None):
'''
Fork state on expression concretizations.
Using policy build a list of solutions for expression.
For the state on each solution setting the new state with setstate
For example if expression is a Bool it may have 2 solutions. True or False.
Parent
(expression = ??)
Child1 Child2
(expression = True) (expression = True)
setstate(True) setstate(False)
The optional setstate() function is supposed to set the concrete value
in the child state.
'''
assert isinstance(expression, Expression)
if setstate is None:
setstate = lambda x, y: None
# Find a set of solutions for expression
solutions = state.concretize(expression, policy)
if not solutions:
raise ExecutorError("Forking on unfeasible constraint set")
if len(solutions) == 1:
setstate(state, solutions[0])
return state
logger.info("Forking. Policy: %s. Values: %s",
policy,
', '.join(f'0x{sol:x}' for sol in solutions))
self._publish('will_fork_state', state, expression, solutions, policy)
# Build and enqueue a state for each solution
children = []
for new_value in solutions:
with state as new_state:
new_state.constrain(expression == new_value)
# and set the PC of the new state to the concrete pc-dest
#(or other register or memory address to concrete)
setstate(new_state, new_value)
self._publish('did_fork_state', new_state, expression, new_value, policy)
# enqueue new_state
state_id = self.enqueue(new_state)
# maintain a list of children for logging purpose
children.append(state_id)
logger.info("Forking current state into states %r", children)
return None
def run(self):
'''
Entry point of the Executor; called by workers to start analysis.
'''
# policy_order=self.policy_order
# policy=self.policy
current_state = None
current_state_id = None
with WithKeyboardInterruptAs(self.shutdown):
# notify siblings we are about to start a run
self._notify_start_run()
logger.debug("Starting Manticore Symbolic Emulator Worker (pid %d).", os.getpid())
solver = Z3Solver()
while not self.is_shutdown():
try: # handle fatal errors: exceptions in Manticore
try: # handle external (e.g. solver) errors, and executor control exceptions
# select a suitable state to analyze
if current_state is None:
with self._lock:
# notify siblings we are about to stop this run
self._notify_stop_run()
try:
# Select a single state_id
current_state_id = self.get()
# load selected state from secondary storage
if current_state_id is not None:
self._publish('will_load_state', current_state_id)
current_state = self._workspace.load_state(current_state_id)
self.forward_events_from(current_state, True)
self._publish('did_load_state', current_state, current_state_id)
logger.info("load state %r", current_state_id)
# notify siblings we have a state to play with
finally:
self._notify_start_run()
# If current_state is still None. We are done.
if current_state is None:
logger.debug("No more states in the queue, byte bye!")
break
assert current_state is not None
assert current_state.constraints is current_state.platform.constraints
# Allows to terminate manticore worker on user request
while not self.is_shutdown():
if not current_state.execute():
break
else:
# Notify this worker is done
self._publish('will_terminate_state', current_state, current_state_id, TerminateState('Shutdown'))
current_state = None
# Handling Forking and terminating exceptions
except Concretize as e:
# expression
# policy
# setstate()
logger.debug("Generic state fork on condition")
current_state = self.fork(current_state, e.expression, e.policy, e.setstate)
except TerminateState as e:
# Notify this worker is done
self._publish('will_terminate_state', current_state, current_state_id, e)
logger.debug("Generic terminate state")
if e.testcase:
self.generate_testcase(current_state, str(e))
current_state = None
except SolverException as e:
# raise
import traceback
trace = traceback.format_exc()
logger.error("Exception: %s\n%s", str(e), trace)
# Notify this state is done
self._publish('will_terminate_state', current_state, current_state_id, e)
if solver.check(current_state.constraints):
self.generate_testcase(current_state, "Solver failed" + str(e))
current_state = None
except (Exception, AssertionError) as e:
# raise
import traceback
trace = traceback.format_exc()
logger.error("Exception: %s\n%s", str(e), trace)
# Notify this worker is done
self._publish('will_terminate_state', current_state, current_state_id, e)
current_state = None
logger.setState(None)
assert current_state is None or self.is_shutdown()
# notify siblings we are about to stop this run
self._notify_stop_run()
def main():
# global server
global ticker_condition, platforms
global get_dict, get_condition
# Only works with 3.6+, checking version
version = sys.version.split(' ')[0].split('.')
invalid = False
if int(version[0]) < 3:
invalid = True
elif int(version[0]) == 3:
if int(version[1]) < 6:
invalid = True
if invalid:
print('Invalid version of Python. Requires Python 3.6 or newer')
return
read_config()
manager = SyncManager()
manager.start()
running = manager.Event()
running.set()
platforms = manager.dict()
ticker_condition = manager.Condition()
r_queue = manager.Queue()
r_condition = manager.Condition()
# used for HTTP GET requests and their responses
get_dict = manager.dict()
get_condition = manager.Condition()
# used for HTTP PUT and POST to issue their responses
reply_queue = manager.Queue()
reply_condition = manager.Condition()
# A pool closes, don't want it to close.
reply_list = []
reply_args = (running, reply_queue, reply_condition)
for i in range(config['threads']['return_threads']):
r = Process(target=outbound, args=reply_args, name='Reply_%s' % i)
r.deamon = True
reply_list.append(r)
r.start()
# A pool closes, don't want it to close.
r_list = []
r_args = (running, api_key, platforms, r_queue, r_condition, get_dict,
get_condition, reply_queue, reply_condition, ticker_condition,
platform_lock)
for i in range(config['threads']['api_threads']):
r = Process(target=retriever, args=r_args, name='Retriever_%s' % i)
r.deamon = True
r_list.append(r)
r.start()
ticking_args = (running, platforms, ticker_condition, r_queue, r_condition)
ticking = Process(target=ticker, args=ticking_args, name='Ticker')
ticking.deamon = True
ticking.start()
poking_args = (running, platforms, ticker_condition)
poking = Process(target=poker, args=poking_args, name='Poker')
poking.daemon = True
poking.start()
handler = CreateSyncHTTPHandler(manager)
#server = http.server.HTTPServer( (config['server']['host'], config['server']['port']), MyHTTPHandler)
server = http.server.HTTPServer(
(config['server']['host'], config['server']['port']), handler)
# server.set_manager(manager) # Need another way to pass this in, since HTTPServer doesn't have it...
try:
server.serve_forever()
except KeyboardInterrupt:
print('\nStopping server...')
:return: None
'''
Signal.signal(Signal.SIGINT, Signal.SIG_IGN)
# explicitly starting the manager, and telling it to ignore the interrupt signal and propagate it.
manager.start(mgr_init)
CLIENT_STATUS = manager.dict()
CLIENT_STATUS["Sync_RX_status"] = False
CLIENT_STATUS["keyboard_disconnect"] = False
CLIENT_STATUS["keyboard_disconnect_attemp"] = 0
CLIENT_STATUS["measure_running_now"] = False
# threading condition variables for controlling Sync RX thread activity
Sync_RX_condition = manager.Condition()
@contextlib.contextmanager
def nostdout():
"""
Silence stdoutput of function within a context.
Example:
>>> with nostdout():
>>> f0,Qi,Qr,zfit,modelwise = do_fit(...)
"""
save_stdout = sys.stdout
sys.stdout = io.BytesIO()
yield
sys.stdout = save_stdout
