def work_repack(self, work, shared_state, ctx=None, **repack_kwargs):
runner = ctx['runner']
current_walk = encode.decode(work)
walk_id, branch_id, current_walk = encode.decode(work)
call = runner.prep_work_call((walk_id, branch_id, current_walk,
repack_kwargs.get('serial_action',
None)))
return call
def load_from_master(log_file, walk_id):
"""
Load a ``Walk`` from a master log file. A master log file provides paths to
trace files which were produced by ``Walk`` running services. This presumes
the trace files are available locally, so if they were produced on a remote
node, you'll need to make them available by whatever method you prefer.
:param str log_file: a path to the master log file
:param int walk_id: a ``walk_id`` which appears in the trace file
"""
with open(log_file, 'r') as f:
# Burn the first line, which identifies the file as a master log
f.readline()
for line in f:
decoded = encode.decode(line)
logs = decoded['logs']
trace_file = logs[1]
if trace_file:
try:
return load_from_trace(trace_file, walk_id)
except (ValueError, OSError):
pass
raise ValueError("Cannot find walk with id %d", walk_id)
def load_from_trace(trace_file, walk_id):
"""
Load a ``Walk`` from a trace file.
:param str trace_file: a path to the trace file
:param int walk_id: a ``walk_id`` which appears in the trace file
"""
found = False
walk_out = walk.Walk()
with open(trace_file, 'r') as f:
for line in f:
decoded = encode.decode(line)
walk_id = decoded['walk_id']
if walk_id != walk_id:
continue
found = True
serial_action = decoded.get('serial_action', None)
walk_segment = decoded['walk']
if serial_action is not None:
walk_out.append(serial_action)
walk_out += walk_segment
if found:
return walk_out
raise ValueError("Cannot find walk with id %d" % walk_id)
def prep_work_call(self, work):
walk_id, branch_id, walk_to_run, serial_action = work
if serial_action is not None:
serial_action = encode.decode(serial_action)
def run(state):
try:
self.run_walk(walk_id,
branch_id,
walk_to_run,
state,
serial_action=serial_action)
except CancelWalk:
self.count_cancel()
except Exception as e:
self.count_error()
self.stats['failed_walk_ids'].add(walk_id)
try:
self._state_supplement[walk_id]['cancel'] = True
except KeyError:
self._state_supplement[walk_id] = {'cancel': True}
central_logger.log_remote_error(e)
raise
return run
def _load_walk(log_file, walk_id):
with open(log_file, 'r') as f:
first_line = f.readline()
try:
first_line = encode.decode(first_line)
except ValueError:
raise ValueError("I couldn't interpret this as a log file: %s" %
log_file)
if 'id' in first_line:
return load_from_master(log_file, walk_id)
return load_from_trace(log_file, walk_id)
def gather_state(self, connection, worker_id, full=False):
"""
Gather ``state`` from a remote service for a given worker. A running
:class:`Walk` is free to mutate its ``state``, and sometimes that is
what really constitutes the "response" or "output" of a quantum of
work.
:param tuple connection: str hostname/ip of the remote service, int
port number
:param int worker_id: id of remote worker, as returned when starting
the work (see :func:`scatter_work`).
"""
if full:
state = rpyc.utils.classic.obtain(
self.clients[connection].gather_full_state(worker_id))
else:
state = rpyc.utils.classic.obtain(
self.clients[connection].gather_state(worker_id))
return encode.decode(state)
def work_repack(self, work, shared_state, ctx=None, **repack_kwargs):
"""
This is called back to repackage each work item sent to the service.
The call is an opportunity to e.g. do some deserialization, wrap the
``Walk`` in a ``WalkRunner``, or anything else the user needs to prep
the work for execution.
:param object work: the work to execute; typically this will be e.g. a
JSONified ``Walk``.
:param object shared_state: a ``state`` copied in for executing the
``Walk``
:param object ctx: the object returned by repack_ctx before
"""
runner = ctx['runner']
current_walk = encode.decode(work)
# walk_state is a single copy of shared_state. The Runner will store a
# copy for retrieval later.
walk_state = copy.copy(shared_state)
walk_id = self.get_walk_id()
call = runner.prep_work_call(current_walk, walk_state)
return call
type=str,
help="Qualname for function used for replay; see %s" %
DEFAULT_REPLAY_FUNC_NAME,
default=DEFAULT_REPLAY_FUNC_NAME)
parser.add_argument('--state',
type=str,
help="state provided as a "
"JSON string, decodable by %s" %
utils.get_class_qualname(encode.decode))
parser.add_argument('walk_id', type=int)
parser.add_argument('--print_state', action='store_true')
args = parser.parse_args()
command = args.command
step = command == 'step'
if args.state:
state = encode.decode(args.state)
else:
state = None
state = replay_walk_by_id(args.log_file,
args.walk_id,
step=step,
replay_func_qualname=args.replay_func,
state=state)
if args.print_state:
print(encode.encode(state))
def test_walk_replay(self):
log_dir = tempfile.mkdtemp()
walk_order = [actions.ActionSingleton1,
actions.ActionSingleton2,
actions.SerialActionSingleton1,
actions.ActionSingleton3]
# See classes.py
expected = [1, 2, 0, 3]
state = {}
walk_count, error_count, _, _, states, logs, _ = \
runner.run_tests(walk_order,
verbose=2,
log_dir=log_dir,
state=state,
gather_states=True)
self.assertEqual(walk_count, 1)
self.assertEqual(error_count, 0)
# Single worker
self.assertEqual(len(states), 1)
# Single walk on that worker
worker_states = states[0]
self.assertEqual(len(worker_states), 1)
state = worker_states[0]
state = state['inner']
self.assertEqual(state, expected)
trace_logs = [log_locs[1] for log_locs in
logs['remote'].values()]
# Maps walk_id->list of stuff
walks = {}
for trace_log in trace_logs:
with open(trace_log, 'r') as f:
for line in f:
record = encode.decode(line)
walk_id = record['walk_id']
current_walk = record['walk']
serial_action = record['serial_action']
if walk_id not in walks:
walks[walk_id] = []
walks[walk_id].append((current_walk, serial_action))
# Checking my assumptions
self.assertEqual(len(walks), 1)
walk_to_replay = walk.Walk()
for segment in list(walks.values())[0]:
walk_segment = segment[0]
serial_action = segment[1]
if serial_action:
walk_to_replay.append(serial_action)
walk_to_replay += walk_segment
# Replay walk directly
state = {}
runner.replay_walk(walk_to_replay, state=state)
self.assertTrue('inner' in state)
self.assertEqual(state['inner'], expected)
# Now let's test the replay lib
state = {}
state = replay.replay_walk_by_id(logs, walk_id, step=False,
state=state)
self.assertTrue('inner' in state)
self.assertEqual(state['inner'], expected)
shutil.rmtree(log_dir)