def get_ref(self, ref, string_decode, file_decode):
runtime_response = self.ciel_runtime.synchronous_request("open_ref", {"ref": ref})
if "error" in runtime_response:
raise ReferenceUnavailableException(ref)
elif "strdata" in runtime_response:
return string_decode(decode_datavalue_string(runtime_response["strdata"]))
elif "filename" in runtime_response:
with open(runtime_response["filename"], "r") as fp:
return file_decode(fp)
def deref_decode(ref, decode_string_callback, decode_file_callback, **kwargs):
runtime_response = try_fetch_ref(ref, "open_ref", accept_string=True, **kwargs)
try:
obj = decode_string_callback(decode_datavalue_string(runtime_response["strdata"]))
except KeyError:
with open(runtime_response["filename"], "r") as ref_fp:
obj = decode_file_callback(ref_fp)
current_task.ref_cache[ref.id] = obj
return obj
def deref_as_raw_file(ref, may_stream=False, sole_consumer=False, chunk_size=67108864, make_sweetheart=False, must_block=False):
if not may_stream:
runtime_response = try_fetch_ref(ref, "open_ref", make_sweetheart=make_sweetheart)
try:
return closing(StringIO(decode_datavalue_string(runtime_response["strdata"])))
except KeyError:
return CompleteFile(ref, runtime_response["filename"])
else:
runtime_response = try_fetch_ref(ref, "open_ref_async", chunk_size=chunk_size, sole_consumer=sole_consumer, make_sweetheart=make_sweetheart, must_block=must_block)
if runtime_response["done"]:
return CompleteFile(ref, runtime_response["filename"])
elif runtime_response["blocking"]:
return CompleteFile(ref, runtime_response["filename"], chunk_size=chunk_size, must_close=True)
else:
return StreamingFile(ref, runtime_response["filename"], runtime_response["size"], chunk_size)
source_filename = runtime_response["filename"]
user_script_namespace = imp.load_source(str("user_namespace_%s" % entry_dict["py_ref"].id), source_filename)
user_script_namespaces[entry_dict["py_ref"].id] = user_script_namespace
else:
print >>sys.stderr, "SkyPy: Using pre-parsed .py file"
if "coro_ref" in entry_dict:
print >>sys.stderr, "SkyPy: Resuming"
resume_state = soft_cache.try_get_cache([entry_dict["coro_ref"]], "coro")
if resume_state is not None:
print >>sys.stderr, "SkyPy: Resuming from soft-cached coroutine"
else:
runtime_response = skypy.fetch_ref(entry_dict["coro_ref"], "open_ref", message_helper, make_sweetheart=True)
if "strdata" in runtime_response:
fp = StringIO(decode_datavalue_string(runtime_response["strdata"]))
else:
fp = open(runtime_response["filename"], "r")
with fp:
resume_state = pickle.load(fp)
user_coro = resume_state.coro
else:
print >>sys.stderr, "Entering at", entry_dict["entry_point"], "args", entry_dict["entry_args"]
persistent_state = skypy.PersistentState(export_json=entry_dict["export_json"],
extra_outputs=entry_dict["extra_outputs"],
py_ref=entry_dict["py_ref"],
is_fixed=entry_dict["run_fixed"])
user_coro = stackless.coroutine()
user_coro.bind(skypy.start_script, user_script_namespace.__dict__[entry_dict["entry_point"]], entry_dict["entry_args"])
resume_state = skypy.ResumeState(persistent_state, user_coro)
def from_safe_dict(in_dict):
try:
in_dict["strdata"] = decode_datavalue_string(in_dict["strdata"])
except KeyError:
pass
return FileOrString(**in_dict)
def ref_from_safe_string(string, id):
if len(string) < 1024:
return SWDataValue(id, value=string)
else:
return ref_from_string(decode_datavalue_string(string), id)