def get_ref(self, ref, string_decode, file_decode): runtime_response = self.ciel_runtime.synchronous_request("open_ref", {"ref": ref}) if "error" in runtime_response: raise ReferenceUnavailableException(ref) elif "strdata" in runtime_response: return string_decode(decode_datavalue_string(runtime_response["strdata"])) elif "filename" in runtime_response: with open(runtime_response["filename"], "r") as fp: return file_decode(fp)
def deref_decode(ref, decode_string_callback, decode_file_callback, **kwargs): runtime_response = try_fetch_ref(ref, "open_ref", accept_string=True, **kwargs) try: obj = decode_string_callback(decode_datavalue_string(runtime_response["strdata"])) except KeyError: with open(runtime_response["filename"], "r") as ref_fp: obj = decode_file_callback(ref_fp) current_task.ref_cache[ref.id] = obj return obj
def deref_as_raw_file(ref, may_stream=False, sole_consumer=False, chunk_size=67108864, make_sweetheart=False, must_block=False): if not may_stream: runtime_response = try_fetch_ref(ref, "open_ref", make_sweetheart=make_sweetheart) try: return closing(StringIO(decode_datavalue_string(runtime_response["strdata"]))) except KeyError: return CompleteFile(ref, runtime_response["filename"]) else: runtime_response = try_fetch_ref(ref, "open_ref_async", chunk_size=chunk_size, sole_consumer=sole_consumer, make_sweetheart=make_sweetheart, must_block=must_block) if runtime_response["done"]: return CompleteFile(ref, runtime_response["filename"]) elif runtime_response["blocking"]: return CompleteFile(ref, runtime_response["filename"], chunk_size=chunk_size, must_close=True) else: return StreamingFile(ref, runtime_response["filename"], runtime_response["size"], chunk_size)
source_filename = runtime_response["filename"] user_script_namespace = imp.load_source(str("user_namespace_%s" % entry_dict["py_ref"].id), source_filename) user_script_namespaces[entry_dict["py_ref"].id] = user_script_namespace else: print >>sys.stderr, "SkyPy: Using pre-parsed .py file" if "coro_ref" in entry_dict: print >>sys.stderr, "SkyPy: Resuming" resume_state = soft_cache.try_get_cache([entry_dict["coro_ref"]], "coro") if resume_state is not None: print >>sys.stderr, "SkyPy: Resuming from soft-cached coroutine" else: runtime_response = skypy.fetch_ref(entry_dict["coro_ref"], "open_ref", message_helper, make_sweetheart=True) if "strdata" in runtime_response: fp = StringIO(decode_datavalue_string(runtime_response["strdata"])) else: fp = open(runtime_response["filename"], "r") with fp: resume_state = pickle.load(fp) user_coro = resume_state.coro else: print >>sys.stderr, "Entering at", entry_dict["entry_point"], "args", entry_dict["entry_args"] persistent_state = skypy.PersistentState(export_json=entry_dict["export_json"], extra_outputs=entry_dict["extra_outputs"], py_ref=entry_dict["py_ref"], is_fixed=entry_dict["run_fixed"]) user_coro = stackless.coroutine() user_coro.bind(skypy.start_script, user_script_namespace.__dict__[entry_dict["entry_point"]], entry_dict["entry_args"]) resume_state = skypy.ResumeState(persistent_state, user_coro)
def from_safe_dict(in_dict): try: in_dict["strdata"] = decode_datavalue_string(in_dict["strdata"]) except KeyError: pass return FileOrString(**in_dict)
def ref_from_safe_string(string, id): if len(string) < 1024: return SWDataValue(id, value=string) else: return ref_from_string(decode_datavalue_string(string), id)