def stream_consumer(chunk_size, in_ref, may_stream, use_direct_pipes, must_block): bytes_read = 0 next_threshold = chunk_size events = [] events.append(("STARTED", datetime.now())) with skypy.deref_as_raw_file(in_ref, may_stream=may_stream, sole_consumer=use_direct_pipes, chunk_size=chunk_size, must_block=must_block) as in_file: events.append(("START_READ", datetime.now())) while True: str = in_file.read(4096) bytes_read += len(str) if len(str) == 0: break if bytes_read >= next_threshold: next_threshold += chunk_size events.append(("READ_CHUNK", datetime.now())) events.append(("FINISHED", datetime.now())) with skypy.open_output(skypy.get_extra_output_indices()[0]) as log_out: pickle.dump(events, log_out) return "Read %d bytes" % bytes_read
def stream_producer(chunk_size, chunks_to_produce): bytes_written = 0 with skypy.open_output(skypy.extra_outputs[0], may_pipe=True) as file_out: while bytes_written < (chunk_size * chunks_to_produce): file_out.write("Have some bytes!") bytes_written += 16 return "Wrote %d bytes" % bytes_written
def reader_function(refs): print >> sys.stderr, "SkyPy example reader function:", len(refs), "inputs" results = [] for ref in refs: with skypy.deref_as_raw_file(ref) as in_file: results.append(in_file.read()) with skypy.open_output(skypy.extra_outputs[0]) as file_out: file_out.write("Complete read results: %s\n" % str(results)) return "Read %d results" % len(refs)
def reader_function(refs): print >>sys.stderr, "SkyPy example reader function:", len(refs), "inputs" results = [] for ref in refs: with skypy.deref_as_raw_file(ref) as in_file: results.append(in_file.read()) with skypy.open_output(skypy.extra_outputs[0]) as file_out: file_out.write("Complete read results: %s\n" % str(results)) return "Read %d results" % len(refs)
def stream_producer(chunk_size, chunks_to_produce): bytes_written = 0 assert len(skypy.get_extra_output_indices()) > 0 with skypy.open_output(1, may_stream=True, may_pipe=True) as file_out: while bytes_written < (chunk_size * chunks_to_produce): file_out.write("Have some bytes!") bytes_written += 16 return "Wrote %d bytes" % bytes_written
def skypy_main(): print >> sys.stderr, "SkyPy example producer:", len( skypy.extra_outputs), "outputs" # Step 1: Test writing our external raw outputs. for i, id in enumerate(skypy.extra_outputs): with skypy.open_output(id) as file_out: file_out.write("Skypy writing output %d" % i) # Step 2: Test writing fresh outputs. refs = [] for i in range(3): name = skypy.get_fresh_output_name() file_out = skypy.open_output(name) with file_out: file_out.write("Skypy writing anonymous output %d" % i) refs.append(file_out.get_completed_ref()) # Step 3: Test reading those results back. reader_result = skypy.spawn(reader_function, refs, n_extra_outputs=1) # cooked_result, raw_result = read_result(reader_result) cooked_result, raw_result = "Dummy", "text" # Step 4: Test a stream producer/consumer pair. producer = skypy.spawn(stream_producer, 262144, 100, n_extra_outputs=1) consumer_out = skypy.spawn(stream_consumer, 262144, producer.extra_outputs[0]) ret_outs = [producer.ret_output, consumer_out] with skypy.RequiredRefs(ret_outs): results = [skypy.deref(x) for x in ret_outs] return "I wrote %d external outputs\nI created 3 myself\nThe reader's cooked result was '%s'\n The reader's raw result was '%s'\nFinally the streamers' reports are %s\n" % ( len(skypy.extra_outputs), cooked_result, raw_result, results)
def stream_producer(chunk_size, chunks_to_produce, may_stream, use_direct_pipes): chunks_written = 0 write_string = "A" * 4096 events = [] events.append(("STARTED", datetime.now())) with skypy.open_output(skypy.get_extra_output_indices()[0], may_stream=may_stream, may_pipe=use_direct_pipes) as file_out: events.append(("START_WRITE", datetime.now())) while chunks_written < chunks_to_produce: bytes_written = 0 while bytes_written < chunk_size: file_out.write(write_string) bytes_written += 4096 chunks_written += 1 events.append(("WROTE_CHUNK", datetime.now())) events.append(("FINISHED", datetime.now())) with skypy.open_output(skypy.get_extra_output_indices()[1]) as log_out: pickle.dump(events, log_out) return "Wrote %d bytes" % (chunk_size * chunks_to_produce)
def skypy_main(): print >>sys.stderr, "SkyPy example producer:", len(skypy.extra_outputs), "outputs" # Step 1: Test writing our external raw outputs. for i, id in enumerate(skypy.extra_outputs): with skypy.open_output(id) as file_out: file_out.write("Skypy writing output %d" % i) # Step 2: Test writing fresh outputs. refs = [] for i in range(3): name = skypy.get_fresh_output_name() file_out = skypy.open_output(name) with file_out: file_out.write("Skypy writing anonymous output %d" % i) refs.append(file_out.get_completed_ref()) # Step 3: Test reading those results back. reader_result = skypy.spawn(reader_function, refs, n_extra_outputs=1) # cooked_result, raw_result = read_result(reader_result) cooked_result, raw_result = "Dummy", "text" # Step 4: Test a stream producer/consumer pair. producer = skypy.spawn(stream_producer, 262144, 100, n_extra_outputs=1) consumer_out = skypy.spawn(stream_consumer, 262144, producer.extra_outputs[0]) ret_outs = [producer.ret_output, consumer_out] with skypy.RequiredRefs(ret_outs): results = [skypy.deref(x) for x in ret_outs] return "I wrote %d external outputs\nI created 3 myself\nThe reader's cooked result was '%s'\n The reader's raw result was '%s'\nFinally the streamers' reports are %s\n" % (len(skypy.extra_outputs), cooked_result, raw_result, results)
def stream_link(chunk_size, input_ref, may_stream, producer_pipe, consumer_pipe, must_block): bytes_written = 0 # Convoluted structure to avoid blocking on a ref whilst we've got an output in progress with skypy.open_output(skypy.get_extra_output_indices()[0], may_stream=may_stream, may_pipe=producer_pipe) as out_file: with skypy.deref_as_raw_file(input_ref, may_stream=may_stream, sole_consumer=consumer_pipe, chunk_size=chunk_size, must_block=must_block) as in_file: while True: buf = in_file.read(4096) if len(buf) == 0: break out_file.write(buf) bytes_written += len(buf) return "Read/wrote %d bytes" % bytes_written
def skypy_main(): new_output_index = skypy.get_fresh_output_index() out_fp = skypy.open_output(new_output_index) with out_fp: for i in range(100000): out_fp.write("Whoozit") out_ref = out_fp.get_completed_ref() first_counter = skypy.spawn(counter, out_ref, extra_dependencies=[out_ref]) second_counter = skypy.spawn(counter, out_ref, extra_dependencies=[out_ref]) with skypy.RequiredRefs([first_counter, second_counter]): first_result = skypy.deref(first_counter) second_result = skypy.deref(second_counter) return "First counter said", first_result, "and second one said", second_result
report = "User script exception %s\n%s" % (str(skypy.current_task.script_return_val), skypy.current_task.script_backtrace) out_message = ("error", {"report": report}) else: if skypy.current_task.halt_reason == skypy.HALT_REFERENCE_UNAVAILABLE: out_dict = {"executor_name": "skypy", "extra_dependencies": [SW2_FutureReference(x) for x in skypy.current_task.persistent_state.ref_dependencies.keys()], "is_fixed": skypy.current_task.persistent_state.is_fixed } if not skypy.current_task.persistent_state.is_fixed: coro_ref = skypy.save_state(resume_state, make_local_sweetheart=True) soft_cache.put_cache([coro_ref], "coro", resume_state) out_dict.update({"pyfile_ref": skypy.current_task.persistent_state.py_ref, "coro_ref": coro_ref}) write_framed_json(("tail_spawn", out_dict), write_fp) elif skypy.current_task.halt_reason == skypy.HALT_DONE: out_fp = MaybeFile(open_callback=lambda: skypy.open_output(0)) with out_fp: if skypy.current_task.persistent_state.export_json: simplejson.dump(skypy.current_task.script_return_val, out_fp) else: pickle.dump(skypy.current_task.script_return_val, out_fp) skypy.ref_from_maybe_file(out_fp, 0) if skypy.current_task.persistent_state.is_fixed: out_message = ("exit", {"keep_process": "must_keep"}) else: out_message = ("exit", {"keep_process": "may_keep", "soft_cache_keys": soft_cache.get_cache_keys()}) write_framed_json(out_message, write_fp) if skypy.current_task.halt_reason == skypy.HALT_RUNTIME_EXCEPTION: sys.exit(0) if not skypy.current_task.persistent_state.is_fixed: skypy.current_task = None