def eager_dereference(self, ref):
real_ref = self.continuation.resolve_tasklocal_reference_with_ref(ref)
if isinstance(real_ref, SWDataValue):
return map_leaf_values(self.convert_real_to_tasklocal_reference, real_ref.value)
elif isinstance(real_ref, SWURLReference):
value = self.block_store.retrieve_object_for_ref(real_ref, 'json')
dv_ref = SWDataValue(value)
self.continuation.rewrite_reference(ref.index, dv_ref)
return map_leaf_values(self.convert_real_to_tasklocal_reference, value)
else:
self.continuation.mark_as_dereferenced(ref)
raise ReferenceUnavailableException(ref, self.continuation)
def visit_Return(self, node, stack, stack_base):
if node.expr is None:
return None
if stack_base == len(stack):
resume_record = ReturnRR()
stack.append(resume_record)
else:
resume_record = stack[stack_base]
try:
if resume_record.ret is None:
resume_record.ret = ExpressionEvaluatorVisitor(
self.context).visit_and_force_eval(node.expr, stack,
stack_base + 1)
# We must scan through the return value to see if it contains any dereferenced references, and if so, yield so these can be fetched.
eager_derefd_val = map_leaf_values(
self.convert_wrapper_to_eager_dereference, resume_record.ret)
stack.pop()
return eager_derefd_val
except:
raise
def map_leaf_values(f, value):
"""
Recurses over a Skywriting data structure (containing lists, dicts and
primitive leaves), and returns a new structure with the leaves mapped as specified.
"""
if isinstance(value, list):
return map(lambda x: map_leaf_values(f, x), value)
elif isinstance(value, dict):
ret = {}
for (dict_key, dict_value) in value.items():
key = map_leaf_values(f, dict_key)
value = map_leaf_values(f, dict_value)
ret[key] = value
return ret
else:
return f(value)
def eager_dereference_from_map(self, ref, fetches):
# Any fetches needed by a URL reference should have been performed in advance
# (see do_eager_thunks)
real_ref = self.continuation.resolve_tasklocal_reference_with_ref(ref)
if isinstance(real_ref, SWDataValue):
return map_leaf_values(self.convert_real_to_tasklocal_reference, real_ref.value)
elif isinstance(real_ref, SWURLReference):
value = fetches[ref.index]
dv_ref = SWDataValue(value)
self.continuation.rewrite_reference(ref.index, dv_ref)
return map_leaf_values(self.convert_real_to_tasklocal_reference, value)
else:
self.continuation.mark_as_dereferenced(ref)
raise ReferenceUnavailableException(ref, self.continuation)
def eager_dereference(self, ref):
real_ref = self.continuation.resolve_tasklocal_reference_with_ref(ref)
if isinstance(real_ref, SWDataValue):
return map_leaf_values(self.convert_real_to_tasklocal_reference, real_ref.value)
else:
self.continuation.mark_as_dereferenced(ref)
raise ReferenceUnavailableException(ref, self.continuation)
def commit_result(self, block_store, master_proxy):
commit_bindings = {}
for ref in self.additional_refs_to_publish:
commit_bindings[ref.id] = ref
if self.result is None:
if self.save_continuation:
save_cont_uri, _ = self.block_store.store_object(self.continuation, 'pickle', self.get_saved_continuation_object_id())
else:
save_cont_uri = None
master_proxy.commit_task(self.task_id, commit_bindings, save_cont_uri, self.replay_uuid_list)
return
serializable_result = map_leaf_values(self.convert_tasklocal_to_real_reference, self.result)
_, size_hint = block_store.store_object(serializable_result, 'json', self.expected_outputs[0])
if size_hint < 128:
result_ref = SWDataValue(serializable_result)
else:
result_ref = SW2_ConcreteReference(self.expected_outputs[0], SWTaskOutputProvenance(self.original_task_id, 0), size_hint)
result_ref.add_location_hint(self.block_store.netloc)
commit_bindings[self.expected_outputs[0]] = result_ref
if self.save_continuation:
save_cont_uri, size_hint = self.block_store.store_object(self.continuation, 'pickle', self.get_saved_continuation_object_id())
else:
save_cont_uri = None
master_proxy.commit_task(self.task_id, commit_bindings, save_cont_uri, self.replay_uuid_list)
def map_leaf_values(f, value):
"""
Recurses over a Skywriting data structure (containing lists, dicts and
primitive leaves), and returns a new structure with the leaves mapped as specified.
"""
if isinstance(value, list):
return map(lambda x: map_leaf_values(f, x), value)
elif isinstance(value, dict):
ret = {}
for (dict_key, dict_value) in value.items():
key = map_leaf_values(f, dict_key)
value = map_leaf_values(f, dict_value)
ret[key] = value
return ret
else:
return f(value)
def commit_result(self, block_store, master_proxy):
commit_bindings = {}
for ref in self.additional_refs_to_publish:
commit_bindings[ref.id] = ref
if self.result is None:
if self.save_continuation:
save_cont_uri, _ = self.block_store.store_object(self.continuation, 'pickle', self.get_saved_continuation_object_id())
else:
save_cont_uri = None
master_proxy.commit_task(self.task_id, commit_bindings, save_cont_uri, self.replay_uuid_list)
return
serializable_result = map_leaf_values(self.convert_tasklocal_to_real_reference, self.result)
_, size_hint = block_store.store_object(serializable_result, 'json', self.expected_outputs[0])
if size_hint < 128:
result_ref = SWDataValue(self.expected_outputs[0], serializable_result)
else:
result_ref = SW2_ConcreteReference(self.expected_outputs[0], size_hint)
result_ref.add_location_hint(self.block_store.netloc)
commit_bindings[self.expected_outputs[0]] = result_ref
if self.save_continuation:
save_cont_uri, size_hint = self.block_store.store_object(self.continuation, 'pickle', self.get_saved_continuation_object_id())
else:
save_cont_uri = None
master_proxy.commit_task(self.task_id, commit_bindings, save_cont_uri, self.replay_uuid_list)
def do_eager_thunks(self, args):
def resolve_thunks_mapper(leaf):
if isinstance(leaf, SWDereferenceWrapper):
return self.eager_dereference(leaf.ref)
else:
return leaf
return map_leaf_values(resolve_thunks_mapper, args)
def eager_dereference_from_map(self, ref, fetches):
# Any fetches needed by a URL reference should have been performed in advance
# (see do_eager_thunks)
real_ref = self.continuation.resolve_tasklocal_reference_with_ref(ref)
if isinstance(real_ref, SWDataValue):
return map_leaf_values(self.convert_real_to_tasklocal_reference, real_ref.value)
else:
self.continuation.mark_as_dereferenced(ref)
raise ReferenceUnavailableException(ref, self.continuation)
def do_eager_thunks(self, args):
fetches_needed = accumulate_leaf_values(self.check_for_eager_fetch, args)
fetches_needed = filter(lambda x: x is not None, fetches_needed)
fetched_objects = self.block_store.retrieve_objects_for_refs([ref for (id, ref) in fetches_needed], "json")
id_to_result_map = dict(zip([id for (id, ref) in fetches_needed], fetched_objects))
def resolve_thunks_mapper(leaf):
if isinstance(leaf, SWDereferenceWrapper):
return self.eager_dereference_from_map(leaf.ref, id_to_result_map)
else:
return leaf
return map_leaf_values(resolve_thunks_mapper, args)
def do_eager_thunks(self, args):
fetches_needed = accumulate_leaf_values(self.check_for_eager_fetch, args)
fetches_needed = filter(lambda x: x is not None, fetches_needed)
fetched_objects = self.block_store.retrieve_objects_for_refs([ref for (id, ref) in fetches_needed], "json")
id_to_result_map = dict(zip([id for (id, ref) in fetches_needed], fetched_objects))
def resolve_thunks_mapper(leaf):
if isinstance(leaf, SWDereferenceWrapper):
return self.eager_dereference_from_map(leaf.ref, id_to_result_map)
else:
return leaf
return map_leaf_values(resolve_thunks_mapper, args)
def fetch_executor_args(self, inputs):
args_ref = None
parsed_inputs = {}
for local_id, ref in inputs.items():
if local_id == '_args':
args_ref = ref
else:
parsed_inputs[int(local_id)] = ref
exec_args = self.task_executor.block_store.retrieve_object_for_ref(args_ref, 'pickle')
def args_parsing_mapper(leaf):
if isinstance(leaf, SWLocalReference):
return parsed_inputs[leaf.index]
else:
return leaf
parsed_args = map_leaf_values(args_parsing_mapper, exec_args)
return parsed_args
def fetch_executor_args(self, inputs):
args_ref = None
parsed_inputs = {}
for local_id, ref in inputs.items():
if local_id == '_args':
args_ref = ref
else:
parsed_inputs[int(local_id)] = ref
exec_args = self.task_executor.block_store.retrieve_object_for_ref(args_ref, 'pickle')
def args_parsing_mapper(leaf):
if isinstance(leaf, SWLocalReference):
return parsed_inputs[leaf.index]
else:
return leaf
parsed_args = map_leaf_values(args_parsing_mapper, exec_args)
return parsed_args
def spawn_exec_func(self, executor_name, exec_args, num_outputs):
new_task_id = self.create_spawned_task_name()
inputs = {}
args = self.do_eager_thunks(exec_args)
args_id, expected_output_ids = self.create_names_for_exec(executor_name, args, num_outputs)
ret = [self.continuation.create_tasklocal_reference(SW2_FutureReference(expected_output_ids[i])) for i in range(num_outputs)]
def args_check_mapper(leaf):
if isinstance(leaf, SWLocalReference):
real_ref = self.continuation.resolve_tasklocal_reference_with_ref(leaf)
i = len(inputs)
inputs[i] = real_ref
ret = SWLocalReference(i)
return ret
return leaf
transformed_args = map_leaf_values(args_check_mapper, args)
_, size_hint = self.block_store.store_object(transformed_args, 'pickle', args_id)
args_ref = SW2_ConcreteReference(args_id, size_hint)
self.spawn_exec_counter += 1
args_ref.add_location_hint(self.block_store.netloc)
self.maybe_also_publish(args_ref)
inputs['_args'] = args_ref
task_descriptor = {'task_id': new_task_id,
'handler': executor_name,
'dependencies': inputs,
'expected_outputs': expected_output_ids}
self.spawn_list.append(SpawnListEntry(new_task_id, task_descriptor))
if len(self.spawn_list) > 20:
self.spawn_all(self.block_store, self.master_proxy)
self.spawn_list = []
return ret
def visit_Return(self, node, stack, stack_base):
if node.expr is None:
return None
if stack_base == len(stack):
resume_record = ReturnRR()
stack.append(resume_record)
else:
resume_record = stack[stack_base]
try:
if resume_record.ret is None:
resume_record.ret = ExpressionEvaluatorVisitor(self.context).visit_and_force_eval(node.expr, stack, stack_base + 1)
# We must scan through the return value to see if it contains any dereferenced references, and if so, yield so these can be fetched.
eager_derefd_val = map_leaf_values(self.convert_wrapper_to_eager_dereference, resume_record.ret)
stack.pop()
return eager_derefd_val
except:
raise
