def eager_dereference(self, ref): real_ref = self.continuation.resolve_tasklocal_reference_with_ref(ref) if isinstance(real_ref, SWDataValue): return map_leaf_values(self.convert_real_to_tasklocal_reference, real_ref.value) elif isinstance(real_ref, SWURLReference): value = self.block_store.retrieve_object_for_ref(real_ref, 'json') dv_ref = SWDataValue(value) self.continuation.rewrite_reference(ref.index, dv_ref) return map_leaf_values(self.convert_real_to_tasklocal_reference, value) else: self.continuation.mark_as_dereferenced(ref) raise ReferenceUnavailableException(ref, self.continuation)
def visit_Return(self, node, stack, stack_base): if node.expr is None: return None if stack_base == len(stack): resume_record = ReturnRR() stack.append(resume_record) else: resume_record = stack[stack_base] try: if resume_record.ret is None: resume_record.ret = ExpressionEvaluatorVisitor( self.context).visit_and_force_eval(node.expr, stack, stack_base + 1) # We must scan through the return value to see if it contains any dereferenced references, and if so, yield so these can be fetched. eager_derefd_val = map_leaf_values( self.convert_wrapper_to_eager_dereference, resume_record.ret) stack.pop() return eager_derefd_val except: raise
def map_leaf_values(f, value): """ Recurses over a Skywriting data structure (containing lists, dicts and primitive leaves), and returns a new structure with the leaves mapped as specified. """ if isinstance(value, list): return map(lambda x: map_leaf_values(f, x), value) elif isinstance(value, dict): ret = {} for (dict_key, dict_value) in value.items(): key = map_leaf_values(f, dict_key) value = map_leaf_values(f, dict_value) ret[key] = value return ret else: return f(value)
def eager_dereference_from_map(self, ref, fetches): # Any fetches needed by a URL reference should have been performed in advance # (see do_eager_thunks) real_ref = self.continuation.resolve_tasklocal_reference_with_ref(ref) if isinstance(real_ref, SWDataValue): return map_leaf_values(self.convert_real_to_tasklocal_reference, real_ref.value) elif isinstance(real_ref, SWURLReference): value = fetches[ref.index] dv_ref = SWDataValue(value) self.continuation.rewrite_reference(ref.index, dv_ref) return map_leaf_values(self.convert_real_to_tasklocal_reference, value) else: self.continuation.mark_as_dereferenced(ref) raise ReferenceUnavailableException(ref, self.continuation)
def eager_dereference(self, ref): real_ref = self.continuation.resolve_tasklocal_reference_with_ref(ref) if isinstance(real_ref, SWDataValue): return map_leaf_values(self.convert_real_to_tasklocal_reference, real_ref.value) else: self.continuation.mark_as_dereferenced(ref) raise ReferenceUnavailableException(ref, self.continuation)
def commit_result(self, block_store, master_proxy): commit_bindings = {} for ref in self.additional_refs_to_publish: commit_bindings[ref.id] = ref if self.result is None: if self.save_continuation: save_cont_uri, _ = self.block_store.store_object(self.continuation, 'pickle', self.get_saved_continuation_object_id()) else: save_cont_uri = None master_proxy.commit_task(self.task_id, commit_bindings, save_cont_uri, self.replay_uuid_list) return serializable_result = map_leaf_values(self.convert_tasklocal_to_real_reference, self.result) _, size_hint = block_store.store_object(serializable_result, 'json', self.expected_outputs[0]) if size_hint < 128: result_ref = SWDataValue(serializable_result) else: result_ref = SW2_ConcreteReference(self.expected_outputs[0], SWTaskOutputProvenance(self.original_task_id, 0), size_hint) result_ref.add_location_hint(self.block_store.netloc) commit_bindings[self.expected_outputs[0]] = result_ref if self.save_continuation: save_cont_uri, size_hint = self.block_store.store_object(self.continuation, 'pickle', self.get_saved_continuation_object_id()) else: save_cont_uri = None master_proxy.commit_task(self.task_id, commit_bindings, save_cont_uri, self.replay_uuid_list)
def commit_result(self, block_store, master_proxy): commit_bindings = {} for ref in self.additional_refs_to_publish: commit_bindings[ref.id] = ref if self.result is None: if self.save_continuation: save_cont_uri, _ = self.block_store.store_object(self.continuation, 'pickle', self.get_saved_continuation_object_id()) else: save_cont_uri = None master_proxy.commit_task(self.task_id, commit_bindings, save_cont_uri, self.replay_uuid_list) return serializable_result = map_leaf_values(self.convert_tasklocal_to_real_reference, self.result) _, size_hint = block_store.store_object(serializable_result, 'json', self.expected_outputs[0]) if size_hint < 128: result_ref = SWDataValue(self.expected_outputs[0], serializable_result) else: result_ref = SW2_ConcreteReference(self.expected_outputs[0], size_hint) result_ref.add_location_hint(self.block_store.netloc) commit_bindings[self.expected_outputs[0]] = result_ref if self.save_continuation: save_cont_uri, size_hint = self.block_store.store_object(self.continuation, 'pickle', self.get_saved_continuation_object_id()) else: save_cont_uri = None master_proxy.commit_task(self.task_id, commit_bindings, save_cont_uri, self.replay_uuid_list)
def do_eager_thunks(self, args): def resolve_thunks_mapper(leaf): if isinstance(leaf, SWDereferenceWrapper): return self.eager_dereference(leaf.ref) else: return leaf return map_leaf_values(resolve_thunks_mapper, args)
def eager_dereference_from_map(self, ref, fetches): # Any fetches needed by a URL reference should have been performed in advance # (see do_eager_thunks) real_ref = self.continuation.resolve_tasklocal_reference_with_ref(ref) if isinstance(real_ref, SWDataValue): return map_leaf_values(self.convert_real_to_tasklocal_reference, real_ref.value) else: self.continuation.mark_as_dereferenced(ref) raise ReferenceUnavailableException(ref, self.continuation)
def do_eager_thunks(self, args): fetches_needed = accumulate_leaf_values(self.check_for_eager_fetch, args) fetches_needed = filter(lambda x: x is not None, fetches_needed) fetched_objects = self.block_store.retrieve_objects_for_refs([ref for (id, ref) in fetches_needed], "json") id_to_result_map = dict(zip([id for (id, ref) in fetches_needed], fetched_objects)) def resolve_thunks_mapper(leaf): if isinstance(leaf, SWDereferenceWrapper): return self.eager_dereference_from_map(leaf.ref, id_to_result_map) else: return leaf return map_leaf_values(resolve_thunks_mapper, args)
def fetch_executor_args(self, inputs): args_ref = None parsed_inputs = {} for local_id, ref in inputs.items(): if local_id == '_args': args_ref = ref else: parsed_inputs[int(local_id)] = ref exec_args = self.task_executor.block_store.retrieve_object_for_ref(args_ref, 'pickle') def args_parsing_mapper(leaf): if isinstance(leaf, SWLocalReference): return parsed_inputs[leaf.index] else: return leaf parsed_args = map_leaf_values(args_parsing_mapper, exec_args) return parsed_args
def spawn_exec_func(self, executor_name, exec_args, num_outputs): new_task_id = self.create_spawned_task_name() inputs = {} args = self.do_eager_thunks(exec_args) args_id, expected_output_ids = self.create_names_for_exec(executor_name, args, num_outputs) ret = [self.continuation.create_tasklocal_reference(SW2_FutureReference(expected_output_ids[i])) for i in range(num_outputs)] def args_check_mapper(leaf): if isinstance(leaf, SWLocalReference): real_ref = self.continuation.resolve_tasklocal_reference_with_ref(leaf) i = len(inputs) inputs[i] = real_ref ret = SWLocalReference(i) return ret return leaf transformed_args = map_leaf_values(args_check_mapper, args) _, size_hint = self.block_store.store_object(transformed_args, 'pickle', args_id) args_ref = SW2_ConcreteReference(args_id, size_hint) self.spawn_exec_counter += 1 args_ref.add_location_hint(self.block_store.netloc) self.maybe_also_publish(args_ref) inputs['_args'] = args_ref task_descriptor = {'task_id': new_task_id, 'handler': executor_name, 'dependencies': inputs, 'expected_outputs': expected_output_ids} self.spawn_list.append(SpawnListEntry(new_task_id, task_descriptor)) if len(self.spawn_list) > 20: self.spawn_all(self.block_store, self.master_proxy) self.spawn_list = [] return ret
def visit_Return(self, node, stack, stack_base): if node.expr is None: return None if stack_base == len(stack): resume_record = ReturnRR() stack.append(resume_record) else: resume_record = stack[stack_base] try: if resume_record.ret is None: resume_record.ret = ExpressionEvaluatorVisitor(self.context).visit_and_force_eval(node.expr, stack, stack_base + 1) # We must scan through the return value to see if it contains any dereferenced references, and if so, yield so these can be fetched. eager_derefd_val = map_leaf_values(self.convert_wrapper_to_eager_dereference, resume_record.ret) stack.pop() return eager_derefd_val except: raise