class Generator:
def __init__(self, args, process_count, aislinn_args):
self.args = args
self.statespace = StateSpace()
self.consts_pool = None
self.process_count = process_count
self.error_messages = []
self.message_sizes = set()
self.send_protocol = aislinn_args.send_protocol
self.send_protocol_eager_threshold = \
aislinn_args.send_protocol_eager_threshold
self.send_protocol_rendezvous_threshold = \
aislinn_args.send_protocol_rendezvous_threshold
self.init_time = None
self.end_time = None
self.deterministic_unallocated_memory = None
self.search = aislinn_args.search
self.max_states = aislinn_args.max_states
self.stdout_mode = aislinn_args.stdout
self.stderr_mode = aislinn_args.stderr
self.debug_state = aislinn_args.debug_state
if aislinn_args.debug_compare_states:
self.debug_compare_states = \
aislinn_args.debug_compare_states.split("~")
logging.info("debug_compare_states=%s", self.debug_compare_states)
else:
self.debug_compare_states = None
self.debug_captured_states = None
self.is_full_statespace = False
self.profile = aislinn_args.profile
self.debug_seq = aislinn_args.debug_seq
self.debug_arc_times = aislinn_args.debug_arc_times
self.workers = [Worker(i,
aislinn_args.workers,
self,
args,
aislinn_args)
for i in xrange(aislinn_args.workers)]
def get_statistics(self):
if self.workers[0].stats_time is None:
return None
charts = []
chart = {"type" : "plot",
"name" : "Queue lengthts",
"data" : []}
for i, worker in enumerate(self.workers):
data = (worker.stats_time,
worker.stats_queue_len,
"Worker {}".format(i))
chart["data"].append(data)
charts.append(chart)
chart = {"type" : "timeline",
"name" : "Controllers utilization",
"data" : []}
for i, worker in enumerate(self.workers):
for j in xrange(self.process_count):
data = (worker.stats_controller_start[j],
worker.stats_controller_stop[j],
"Controller {}".format(j))
assert len(data[0]) == len(data[1])
chart["data"].append(data)
charts.append(chart)
chart = {"type" : "boxplot",
"name" : "Controllers utilization",
"data" : []}
for i, worker in enumerate(self.workers):
for j in xrange(self.process_count):
starts = worker.stats_controller_start[j]
stops = worker.stats_controller_stop[j]
times = []
for start, stop in zip(starts, stops):
times.append(stop - start)
chart["data"].append(times)
charts.append(chart)
chart = {"type" : "bars",
"name" : "Controllers utilization",
"data" : [sum(d) for d in chart["data"]]}
charts.append(chart)
return charts
def add_error_message(self, error_message):
if error_message.name in [e.name for e in self.error_messages]:
return
self.error_messages.append(error_message)
def get_const_ptr(self, id):
if id == consts.MPI_TAG_UB:
return self.consts_pool
raise Exception("Invalid const id")
def sanity_check(self):
for node, gstate in self.queue:
gstate.sanity_check()
def sleeping_neighbours_of(self, worker_id):
for i in xrange(1, len(self.workers)):
worker = self.workers[(i + worker_id) % len(self.workers)]
if not worker.queue and not worker.gcontext:
return worker
return None
def start_workers(self):
for worker in self.workers:
worker.start_controllers()
for worker in self.workers:
worker.connect_controllers()
if not self.workers[0].make_initial_node():
return False
for worker in self.workers[1:]:
if not worker.init_nonfirst_worker():
return False
self.workers[0].start_next_in_queue()
return True
def main_cycle(self):
while True:
controllers = sum((worker.running_controllers()
for worker in self.workers if worker.gcontext),
[])
logging.debug("Running controllers %s", controllers)
if not controllers:
return
controllers = poll_controllers(controllers)
for c in controllers:
worker = self.workers[c.name / self.process_count]
#if worker.stats_time is not None:
# worker.record_process_stop(c.name % self.process_count)
context = worker.gcontext.get_context(
c.name % self.process_count)
logging.debug("Ready controller %s", context)
context.process_run_result(c.finish_async())
worker.continue_in_execution()
if worker.gcontext is None:
worker.start_next_in_queue()
def run(self):
self.init_time = datetime.datetime.now()
try:
if not self.start_workers():
return False
self.main_cycle()
self.memory_leak_check()
self.final_check()
if self.send_protocol == "full" and not self.error_messages:
self.ndsync_check()
self.is_full_statespace = True
except ErrorFound:
logging.debug("ErrorFound catched")
finally:
for worker in self.workers:
worker.kill_controllers()
self.end_time = datetime.datetime.now()
return True
def ndsync_check(self):
self.statespace.inject_dfs_indegree()
message = NdsyncChecker(self, self.statespace).run()
if message:
self.add_error_message(message)
def memory_leak_check(self):
final_nodes = list(self.statespace.all_final_nodes())
allocations = [frozenset(node.allocations)
if node.allocations else frozenset()
for node in final_nodes]
self.deterministic_unallocated_memory = 0
if not allocations:
return
all_allocations = frozenset.union(*allocations)
deterministic = frozenset.intersection(*allocations)
for a in sorted(all_allocations - deterministic):
for node in final_nodes:
if node.allocations and a in node.allocations:
gcontext = GlobalContext(None, node, None, generator=self)
m = errormsg.MemoryLeak(gcontext.get_context(a.pid),
address=a.addr,
size=a.size)
break
else:
assert 0 # This shoud not happen
self.add_error_message(m)
for a in deterministic:
self.deterministic_unallocated_memory += a.size
def final_check(self):
if self.debug_compare_states is not None:
self.debug_compare()
for worker in self.workers:
worker.before_final_check()
for worker in self.workers:
worker.final_check()
def debug_compare(self):
if self.debug_captured_states is None:
self.debug_captured_states = []
logging.info("%s states was captured", len(self.debug_captured_states))
if len(self.debug_captured_states) < 2:
return
gstate1, worker1 = self.debug_captured_states[0]
gstate2, worker2 = self.debug_captured_states[1]
logging.info("Hashes %s, %s",
gstate1.compute_hash(), gstate2.compute_hash())
for i, (s1, s2) in enumerate(zip(gstate1.states, gstate2.states)):
if s1.vg_state and s2.vg_state:
logging.info("Pid %s: hash1=%s hash2=%s",
i, s1.vg_state.hash, s2.vg_state.hash)
if s1.vg_state.hash != s2.vg_state.hash:
controller = worker1.controllers[i]
controller.restore_state(s1.vg_state)
controller.debug_dump_state(s1.vg_state.id)
controller = worker2.controllers[i]
controller.restore_state(s2.vg_state)
controller.debug_dump_state(s2.vg_state.id)
for name in s1.__dict__.keys():
if getattr(s1, name) != getattr(s2, name):
logging.info("stateA: %s %s", name, getattr(s1, name))
logging.info("stateB: %s %s", name, getattr(s2, name))
for gstate, worker in self.debug_captured_states:
gstate.dispose()
def add_node(self, prev, worker, gstate, do_hash=True):
if do_hash:
hash = gstate.compute_hash()
if hash is not None:
node = self.statespace.get_node_by_hash(hash)
if node is not None:
return (node, False)
else:
hash = None
uid = str(self.statespace.nodes_count)
node = Node(uid, hash)
logging.debug("New node %s", node.uid)
if prev:
node.prev = prev
self.statespace.add_node(node)
if self.debug_compare_states is not None \
and node.uid in self.debug_compare_states:
if self.debug_captured_states is None:
self.debug_captured_states = []
logging.debug("Capturing %s", node)
self.debug_captured_states.append((gstate.copy(), worker))
if self.statespace.nodes_count > self.max_states:
logging.info("Maximal number of states reached")
if self.debug_compare_states is not None:
self.debug_compare()
raise ErrorFound()
if self.debug_state == uid:
context = Context(self, node, None)
context.add_error_and_throw(
errormsg.StateCaptured(context, uid=uid))
return (node, True)
def create_report(self, args, version):
for error_message in self.error_messages:
if error_message.node:
if error_message.events is None:
error_message.events = \
self.statespace.events_to_node(error_message.node,
None)
if self.stdout_mode == "capture":
error_message.stdout = \
[self.statespace.stream_to_node(
error_message.node,
None,
STREAM_STDOUT,
pid)
for pid in xrange(self.process_count)]
if self.stderr_mode == "capture":
error_message.stderr = \
[self.statespace.stream_to_node(
error_message.node,
None,
STREAM_STDERR,
pid)
for pid in xrange(self.process_count)]
return Report(self, args, version)
class Generator:
def __init__(self, args, process_count, valgrind_args, aislinn_args):
self.args = args
self.statespace = StateSpace()
self.consts_pool = None
self.process_count = process_count
self.working_queue = deque()
self.error_messages = []
self.message_sizes = set()
self.send_protocol = aislinn_args.send_protocol
self.send_protocol_eager_threshold = aislinn_args.send_protocol_eager_threshold
self.send_protocol_rendezvous_threshold = aislinn_args.send_protocol_rendezvous_threshold
self.init_time = None
self.end_time = None
self.deterministic_unallocated_memory = None
if aislinn_args.stats:
self.statistics_tick = aislinn_args.stats
self.statistics = []
else:
self.statistics_tick = None
self.statistics = None
self.search = aislinn_args.search
self.max_states = aislinn_args.max_states
self.stdout_mode = aislinn_args.stdout
self.stderr_mode = aislinn_args.stderr
self.debug_state = aislinn_args.debug_state
if aislinn_args.debug_compare_states:
self.debug_compare_states = aislinn_args.debug_compare_states.split("~")
logging.info("debug_compare_states=%s", self.debug_compare_states)
else:
self.debug_compare_states = None
self.debug_captured_states = None
self.is_full_statespace = False
self.buffer_manager = BufferManager()
self.controllers = [Controller(self.args) for i in xrange(process_count)]
for i, controller in enumerate(self.controllers):
controller.valgrind_args = valgrind_args
controller.name = i
controller.profile = aislinn_args.profile
self.profile = aislinn_args.profile
if aislinn_args.debug_by_valgrind_tool:
for controller in self.controllers:
controller.debug_by_valgrind_tool = aislinn_args.debug_by_valgrind_tool
self.debug_seq = aislinn_args.debug_seq
def get_statistics(self):
if self.statistics is None:
return None
else:
return (
[
("Length of working queue", "states"),
("All pages", "pages"),
("VA", "va"),
("Active pages", "pages"),
("Sum of buffer sizes", "bytes"),
],
self.statistics,
self.statistics_tick,
)
def record_statistics(self):
pages = 0
active_pages = 0
buffers_size = 0
vas = 0
for controller in self.controllers:
stats = controller.get_stats()
pages += stats["pages"]
vas += stats["vas"]
active_pages += stats["active-pages"]
buffers_size += stats["buffers-size"]
self.statistics.append((len(self.working_queue), pages, vas, active_pages, buffers_size))
def add_error_message(self, error_message):
if error_message.name in [e.name for e in self.error_messages]:
return
self.error_messages.append(error_message)
def get_const_ptr(self, id):
if id == consts.MPI_TAG_UB:
return self.consts_pool
raise Exception("Invalid const id")
def sanity_check(self):
for node, gstate in self.working_queue:
gstate.sanity_check()
def get_controller(self, pid):
return self.controllers[pid]
def start_controllers(self):
# We do actions separately to allow parallel initialization
for controller in self.controllers:
controller.start(capture_syscalls=["write"])
for controller in self.controllers:
controller.connect()
initial_node = Node("init", None)
self.statespace.add_node(initial_node)
self.statespace.initial_node = initial_node
gstate = GlobalState(self.process_count)
gcontext = GlobalContext(self, initial_node, gstate)
# TODO: Do it in parallel
for i in xrange(self.process_count):
context = gcontext.get_context(i)
if not context.initial_run():
return False
gcontext.make_node()
gcontext.add_to_queue(None, False)
return True
def run(self):
self.init_time = datetime.datetime.now()
try:
if not self.start_controllers():
return False
tick = self.statistics_tick
tick_counter = tick
while self.working_queue:
if self.search == "dfs":
node, gstate, action = self.working_queue.pop()
else: # bfs
node, gstate, action = self.working_queue.popleft()
self.expand_node(node, gstate, action)
self.cleanup()
if tick:
tick_counter -= 1
if tick_counter == 0:
tick_counter = tick
self.record_statistics()
if self.statespace.nodes_count > self.max_states:
logging.info("Maximal number of states reached")
if self.debug_compare_states is not None:
self.debug_compare()
return True
self.memory_leak_check()
self.final_check()
if self.send_protocol == "full" and not self.error_messages:
self.ndsync_check()
self.is_full_statespace = True
except ErrorFound:
logging.debug("ErrorFound catched")
finally:
for controller in self.controllers:
controller.kill()
self.end_time = datetime.datetime.now()
return True
def ndsync_check(self):
self.statespace.inject_dfs_indegree()
message = NdsyncChecker(self, self.statespace).run()
if message:
self.add_error_message(message)
def memory_leak_check(self):
final_nodes = list(self.statespace.all_final_nodes())
allocations = [frozenset(node.allocations) if node.allocations else frozenset() for node in final_nodes]
self.deterministic_unallocated_memory = 0
if not allocations:
return
all_allocations = frozenset.union(*allocations)
deterministic = frozenset.intersection(*allocations)
for a in sorted(all_allocations - deterministic):
for node in final_nodes:
if node.allocations and a in node.allocations:
gcontext = GlobalContext(self, node, None)
m = errormsg.MemoryLeak(gcontext.get_context(a.pid), address=a.addr, size=a.size)
break
else:
assert 0 # This shoud not happen
self.add_error_message(m)
for a in deterministic:
self.deterministic_unallocated_memory += a.size
def final_check(self):
if self.debug_compare_states is not None:
self.debug_compare()
for controller in self.controllers:
controller.make_buffers()
self.cleanup()
assert self.buffer_manager.resource_count == 0
for controller in self.controllers:
# Check there is no memory leak
assert controller.states_count == 0
stats = controller.get_stats()
# All pages are active, i.e. we have freed everyhing else
assert stats["pages"] == stats["active-pages"]
assert stats["vas"] <= stats["pages"]
assert stats["buffers-size"] == 0
def debug_compare(self):
if self.debug_captured_states is None:
self.debug_captured_states = []
logging.info("%s states was captured", len(self.debug_captured_states))
if len(self.debug_captured_states) < 2:
return
gstate1 = self.debug_captured_states[0]
gstate2 = self.debug_captured_states[1]
logging.info("Hashes %s, %s", gstate1.compute_hash(), gstate2.compute_hash())
for i, (s1, s2) in enumerate(zip(gstate1.states, gstate2.states)):
if s1.vg_state and s2.vg_state:
logging.info("Pids %s %s %s", i, s1.vg_state.hash, s2.vg_state.hash)
if s1.vg_state.id == s2.vg_state.id:
logging.info("States of rank %s are the same", i)
else:
self._controller.debug_compare(s1.vg_state.id, s2.vg_state.id)
for name in s1.__dict__.keys():
if getattr(s1, name) != getattr(s2, name):
logging.info("stateA: %s %s", name, getattr(s1, name))
logging.info("stateB: %s %s", name, getattr(s2, name))
for gstate in self.debug_captured_states:
gstate.dispose()
self.cleanup()
def cleanup(self):
self.buffer_manager.cleanup()
for controller in self.controllers:
controller.cleanup_states()
def check_collective_requests(self, gcontext):
for state in gcontext.gstate.states:
for r in state.active_requests:
if r.is_collective():
if state.gstate.get_operation_by_cc_id(r.comm.comm_id, r.cc_id).can_be_completed(state):
state.finish_collective_request(r)
return True
return False
def continue_probe(self, gcontext, state, rank):
logging.debug("Continue probe %s", state)
comm_id, source, tag, status_ptr = state.probe_data
pid, request = state.probe_deterministic(comm_id, rank, tag)
context = gcontext.prepare_context(state.pid)
if state.flag_ptr:
context.controller.write_int(state.flag_ptr, 1)
context.controller.write_status(status_ptr, rank, request.tag, request.vg_buffer.size)
context.run()
def fast_expand_state(self, gcontext, state):
if state.status == State.StatusWaitAll and state.are_tested_requests_finished():
context = gcontext.prepare_context(state.pid)
context.close_all_requests()
context.run()
return True
if state.status == State.StatusReady:
logging.debug("Continue ready with flag %s", state)
gcontext.prepare_context(state.pid).run()
return True
if (
state.status == State.StatusWaitSome
and state.waits_for_single_non_null_request()
and state.are_tested_requests_finished()
):
context = gcontext.prepare_context(state.pid)
context.continue_waitsome([state.index_of_first_non_null_request()])
context.run()
return True
if (
state.status == State.StatusWaitAny
and state.waits_for_single_non_null_request()
and state.are_tested_requests_finished()
):
context = gcontext.prepare_context(state.pid)
context.continue_waitany(state.index_of_first_non_null_request())
context.run()
return True
if state.status == State.StatusProbe:
comm_id, source, tag, status_ptr = state.probe_data
rank = state.get_probe_promise(comm_id, source, tag)
if rank is not None:
self.continue_probe(gcontext, state, rank)
return True
elif state.probe_data[1] != consts.MPI_ANY_SOURCE and state.flag_ptr is None:
probed = state.probe_deterministic(comm_id, source, tag)
if probed:
pid, request = probed
rank = request.comm.group.pid_to_rank(pid)
self.continue_probe(gcontext, state, rank)
return True
return False
def poll_controllers(self, gcontext):
controllers = [c for c in self.controllers if c.running]
logging.debug("Running controllers %s", controllers)
if not controllers:
return False
controllers = poll_controllers(controllers)
for c in controllers:
context = gcontext.get_context(c.name)
logging.debug("Ready controller %s", context)
result = c.finish_async()
context.process_run_result(result)
return True
def fast_expand_node(self, gcontext):
modified = False
while True:
matching = gcontext.find_deterministic_match()
if matching:
gcontext.apply_matching(matching)
modified = True
continue
if gcontext.gstate.collective_operations and self.check_collective_requests(gcontext):
modified = True
continue
if self.debug_seq: # DEBUG --debug-seq
if all(not gcontext.is_running(state.pid) for state in gcontext.gstate.states):
for state in gcontext.gstate.states:
if self.fast_expand_state(gcontext, state):
break
else: # Normal run
for state in gcontext.gstate.states:
if not gcontext.is_running(state.pid):
self.fast_expand_state(gcontext, state)
self.buffer_manager.cleanup()
if self.poll_controllers(gcontext):
modified = True
continue
return modified
def expand_waitsome(self, gcontext, state, actions):
indices = state.get_indices_of_tested_and_finished_requests()
if not indices:
return
logging.debug("Expanding waitsome %s", state)
for indices in power_set(indices):
if not indices:
continue # skip empty set
actions.append(ActionWaitSome(state.pid, indices))
def expand_testall(self, gcontext, state, actions):
logging.debug("Expanding testall %s", state)
actions.append(ActionFlag0(state.pid))
if state.are_tested_requests_finished():
actions.append(ActionTestAll(state.pid))
def expand_waitany(self, gcontext, state, actions):
logging.debug("Expanding waitany %s", state)
indices = state.get_indices_of_tested_and_finished_requests()
if not indices:
return
for i in indices:
actions.append(ActionWaitAny(state.pid, i))
def expand_probe(self, gcontext, state, actions):
logging.debug("Expanding probe %s", state)
comm_id, source, tag, status_ptr = state.probe_data
if state.get_probe_promise(comm_id, source, tag) is not None:
return
if state.flag_ptr:
actions.append(ActionFlag0(state.pid))
if source != consts.MPI_ANY_SOURCE:
probed = state.probe_deterministic(comm_id, source, tag)
if probed:
pid, request = probed
rank = request.comm.group.pid_to_rank(pid)
actions.append(ActionProbePromise(state.pid, comm_id, source, tag, rank))
if source != consts.MPI_ANY_SOURCE:
return
for pid, request in state.probe_nondeterministic(comm_id, tag):
rank = request.comm.group.pid_to_rank(pid)
actions.append(ActionProbePromise(state.pid, comm_id, source, tag, rank))
def get_actions(self, gcontext):
actions = [ActionMatching(matching) for matching in gcontext.find_nondeterministic_matches()]
for state in gcontext.gstate.states:
if state.status == State.StatusWaitAny:
self.expand_waitany(gcontext, state, actions)
elif state.status == State.StatusProbe:
self.expand_probe(gcontext, state, actions)
elif state.status == State.StatusWaitSome:
self.expand_waitsome(gcontext, state, actions)
elif state.status == State.StatusTest:
self.expand_testall(gcontext, state, actions)
return actions
def slow_expand(self, gcontext):
actions = self.get_actions(gcontext)
if actions:
for action in actions:
gcontext.add_to_queue(action, True)
return bool(actions)
def add_to_queue(self, node, gstate, action):
self.working_queue.append((node, gstate, action))
def expand_node(self, node, gstate, action):
logging.debug("--------- Expanding node %s %s ------------", node.uid, gstate)
if self.debug_compare_states is not None and node.uid in self.debug_compare_states:
if self.debug_captured_states is None:
self.debug_captured_states = []
self.debug_captured_states.append(gstate.copy())
gcontext = GlobalContext(self, node, gstate)
if action:
action.apply_action(gcontext)
self.fast_expand_node(gcontext)
if not gcontext.make_node():
gstate.dispose()
return
if not self.slow_expand(gcontext):
node = gcontext.node
if any(state.status != State.StatusFinished for state in gstate.states):
active_pids = [state.pid for state in gstate.states if state.status != State.StatusFinished]
gcontext = GlobalContext(self, node, gstate)
message = errormsg.Deadlock(None, gcontext=gcontext, active_pids=active_pids)
gcontext.add_error_and_throw(message)
else:
gstate.mpi_leak_check(self, node)
node.allocations = sum((state.allocations for state in gstate.states), [])
gstate.dispose()
def make_context(self, node, state):
# This function exists to avoid importing GlobalContext in state.py
gcontext = GlobalContext(self, node, state.gstate)
return gcontext.get_context(state.pid)
def add_node(self, prev, gstate, do_hash=True):
if do_hash:
hash = gstate.compute_hash()
if hash is not None:
node = self.statespace.get_node_by_hash(hash)
if node is not None:
return (node, False)
else:
hash = None
uid = str(self.statespace.nodes_count)
node = Node(uid, hash)
logging.debug("New node %s", node.uid)
if prev:
node.prev = prev
self.statespace.add_node(node)
if self.debug_state == uid:
context = Context(self, node, None)
context.add_error_and_throw(errormsg.StateCaptured(context, uid=uid))
return (node, True)
def create_report(self, args, version):
for error_message in self.error_messages:
if error_message.node:
if error_message.events is None:
error_message.events = self.statespace.events_to_node(error_message.node, None)
if self.stdout_mode == "capture":
error_message.stdout = [
self.statespace.stream_to_node(error_message.node, None, STREAM_STDOUT, pid)
for pid in xrange(self.process_count)
]
if self.stderr_mode == "capture":
error_message.stderr = [
self.statespace.stream_to_node(error_message.node, None, STREAM_STDERR, pid)
for pid in xrange(self.process_count)
]
return Report(self, args, version)