def stream_consumer(chunk_size, in_ref, may_stream, use_direct_pipes, must_block):
bytes_read = 0
next_threshold = chunk_size
events = []
events.append(("STARTED", datetime.now()))
with skypy.deref_as_raw_file(in_ref, may_stream=may_stream, sole_consumer=use_direct_pipes, chunk_size=chunk_size, must_block=must_block) as in_file:
events.append(("START_READ", datetime.now()))
while True:
str = in_file.read(4096)
bytes_read += len(str)
if len(str) == 0:
break
if bytes_read >= next_threshold:
next_threshold += chunk_size
events.append(("READ_CHUNK", datetime.now()))
events.append(("FINISHED", datetime.now()))
with skypy.open_output(skypy.get_extra_output_indices()[0]) as log_out:
pickle.dump(events, log_out)
return "Read %d bytes" % bytes_read
def stream_producer(chunk_size, chunks_to_produce):
bytes_written = 0
with skypy.open_output(skypy.extra_outputs[0], may_pipe=True) as file_out:
while bytes_written < (chunk_size * chunks_to_produce):
file_out.write("Have some bytes!")
bytes_written += 16
return "Wrote %d bytes" % bytes_written
def stream_producer(chunk_size, chunks_to_produce):
bytes_written = 0
with skypy.open_output(skypy.extra_outputs[0], may_pipe=True) as file_out:
while bytes_written < (chunk_size * chunks_to_produce):
file_out.write("Have some bytes!")
bytes_written += 16
return "Wrote %d bytes" % bytes_written
def reader_function(refs):
print >> sys.stderr, "SkyPy example reader function:", len(refs), "inputs"
results = []
for ref in refs:
with skypy.deref_as_raw_file(ref) as in_file:
results.append(in_file.read())
with skypy.open_output(skypy.extra_outputs[0]) as file_out:
file_out.write("Complete read results: %s\n" % str(results))
return "Read %d results" % len(refs)
def reader_function(refs):
print >>sys.stderr, "SkyPy example reader function:", len(refs), "inputs"
results = []
for ref in refs:
with skypy.deref_as_raw_file(ref) as in_file:
results.append(in_file.read())
with skypy.open_output(skypy.extra_outputs[0]) as file_out:
file_out.write("Complete read results: %s\n" % str(results))
return "Read %d results" % len(refs)
def stream_producer(chunk_size, chunks_to_produce):
bytes_written = 0
assert len(skypy.get_extra_output_indices()) > 0
with skypy.open_output(1, may_stream=True, may_pipe=True) as file_out:
while bytes_written < (chunk_size * chunks_to_produce):
file_out.write("Have some bytes!")
bytes_written += 16
return "Wrote %d bytes" % bytes_written
def skypy_main():
print >> sys.stderr, "SkyPy example producer:", len(
skypy.extra_outputs), "outputs"
# Step 1: Test writing our external raw outputs.
for i, id in enumerate(skypy.extra_outputs):
with skypy.open_output(id) as file_out:
file_out.write("Skypy writing output %d" % i)
# Step 2: Test writing fresh outputs.
refs = []
for i in range(3):
name = skypy.get_fresh_output_name()
file_out = skypy.open_output(name)
with file_out:
file_out.write("Skypy writing anonymous output %d" % i)
refs.append(file_out.get_completed_ref())
# Step 3: Test reading those results back.
reader_result = skypy.spawn(reader_function, refs, n_extra_outputs=1)
# cooked_result, raw_result = read_result(reader_result)
cooked_result, raw_result = "Dummy", "text"
# Step 4: Test a stream producer/consumer pair.
producer = skypy.spawn(stream_producer, 262144, 100, n_extra_outputs=1)
consumer_out = skypy.spawn(stream_consumer, 262144,
producer.extra_outputs[0])
ret_outs = [producer.ret_output, consumer_out]
with skypy.RequiredRefs(ret_outs):
results = [skypy.deref(x) for x in ret_outs]
return "I wrote %d external outputs\nI created 3 myself\nThe reader's cooked result was '%s'\n The reader's raw result was '%s'\nFinally the streamers' reports are %s\n" % (
len(skypy.extra_outputs), cooked_result, raw_result, results)
def stream_producer(chunk_size, chunks_to_produce, may_stream, use_direct_pipes):
chunks_written = 0
write_string = "A" * 4096
events = []
events.append(("STARTED", datetime.now()))
with skypy.open_output(skypy.get_extra_output_indices()[0], may_stream=may_stream, may_pipe=use_direct_pipes) as file_out:
events.append(("START_WRITE", datetime.now()))
while chunks_written < chunks_to_produce:
bytes_written = 0
while bytes_written < chunk_size:
file_out.write(write_string)
bytes_written += 4096
chunks_written += 1
events.append(("WROTE_CHUNK", datetime.now()))
events.append(("FINISHED", datetime.now()))
with skypy.open_output(skypy.get_extra_output_indices()[1]) as log_out:
pickle.dump(events, log_out)
return "Wrote %d bytes" % (chunk_size * chunks_to_produce)
def skypy_main():
print >>sys.stderr, "SkyPy example producer:", len(skypy.extra_outputs), "outputs"
# Step 1: Test writing our external raw outputs.
for i, id in enumerate(skypy.extra_outputs):
with skypy.open_output(id) as file_out:
file_out.write("Skypy writing output %d" % i)
# Step 2: Test writing fresh outputs.
refs = []
for i in range(3):
name = skypy.get_fresh_output_name()
file_out = skypy.open_output(name)
with file_out:
file_out.write("Skypy writing anonymous output %d" % i)
refs.append(file_out.get_completed_ref())
# Step 3: Test reading those results back.
reader_result = skypy.spawn(reader_function, refs, n_extra_outputs=1)
# cooked_result, raw_result = read_result(reader_result)
cooked_result, raw_result = "Dummy", "text"
# Step 4: Test a stream producer/consumer pair.
producer = skypy.spawn(stream_producer, 262144, 100, n_extra_outputs=1)
consumer_out = skypy.spawn(stream_consumer, 262144, producer.extra_outputs[0])
ret_outs = [producer.ret_output, consumer_out]
with skypy.RequiredRefs(ret_outs):
results = [skypy.deref(x) for x in ret_outs]
return "I wrote %d external outputs\nI created 3 myself\nThe reader's cooked result was '%s'\n The reader's raw result was '%s'\nFinally the streamers' reports are %s\n" % (len(skypy.extra_outputs), cooked_result, raw_result, results)
def stream_link(chunk_size, input_ref, may_stream, producer_pipe, consumer_pipe, must_block):
bytes_written = 0
# Convoluted structure to avoid blocking on a ref whilst we've got an output in progress
with skypy.open_output(skypy.get_extra_output_indices()[0], may_stream=may_stream, may_pipe=producer_pipe) as out_file:
with skypy.deref_as_raw_file(input_ref, may_stream=may_stream, sole_consumer=consumer_pipe, chunk_size=chunk_size, must_block=must_block) as in_file:
while True:
buf = in_file.read(4096)
if len(buf) == 0:
break
out_file.write(buf)
bytes_written += len(buf)
return "Read/wrote %d bytes" % bytes_written
def skypy_main():
new_output_index = skypy.get_fresh_output_index()
out_fp = skypy.open_output(new_output_index)
with out_fp:
for i in range(100000):
out_fp.write("Whoozit")
out_ref = out_fp.get_completed_ref()
first_counter = skypy.spawn(counter, out_ref, extra_dependencies=[out_ref])
second_counter = skypy.spawn(counter, out_ref, extra_dependencies=[out_ref])
with skypy.RequiredRefs([first_counter, second_counter]):
first_result = skypy.deref(first_counter)
second_result = skypy.deref(second_counter)
return "First counter said", first_result, "and second one said", second_result
report = "User script exception %s\n%s" % (str(skypy.current_task.script_return_val), skypy.current_task.script_backtrace)
out_message = ("error", {"report": report})
else:
if skypy.current_task.halt_reason == skypy.HALT_REFERENCE_UNAVAILABLE:
out_dict = {"executor_name": "skypy",
"extra_dependencies": [SW2_FutureReference(x) for x in skypy.current_task.persistent_state.ref_dependencies.keys()],
"is_fixed": skypy.current_task.persistent_state.is_fixed
}
if not skypy.current_task.persistent_state.is_fixed:
coro_ref = skypy.save_state(resume_state, make_local_sweetheart=True)
soft_cache.put_cache([coro_ref], "coro", resume_state)
out_dict.update({"pyfile_ref": skypy.current_task.persistent_state.py_ref,
"coro_ref": coro_ref})
write_framed_json(("tail_spawn", out_dict), write_fp)
elif skypy.current_task.halt_reason == skypy.HALT_DONE:
out_fp = MaybeFile(open_callback=lambda: skypy.open_output(0))
with out_fp:
if skypy.current_task.persistent_state.export_json:
simplejson.dump(skypy.current_task.script_return_val, out_fp)
else:
pickle.dump(skypy.current_task.script_return_val, out_fp)
skypy.ref_from_maybe_file(out_fp, 0)
if skypy.current_task.persistent_state.is_fixed:
out_message = ("exit", {"keep_process": "must_keep"})
else:
out_message = ("exit", {"keep_process": "may_keep", "soft_cache_keys": soft_cache.get_cache_keys()})
write_framed_json(out_message, write_fp)
if skypy.current_task.halt_reason == skypy.HALT_RUNTIME_EXCEPTION:
sys.exit(0)
if not skypy.current_task.persistent_state.is_fixed:
skypy.current_task = None
