def read_result(reader_result):
with skypy.RequiredRefs(list(reader_result)):
cooked_result = skypy.deref(reader_result.ret_output)
with skypy.deref_as_raw_file(
reader_result.extra_outputs[0]) as in_file:
return (cooked_result, in_file.read())
def stream_consumer(chunk_size, in_ref, may_stream, use_direct_pipes, must_block):
bytes_read = 0
next_threshold = chunk_size
events = []
events.append(("STARTED", datetime.now()))
with skypy.deref_as_raw_file(in_ref, may_stream=may_stream, sole_consumer=use_direct_pipes, chunk_size=chunk_size, must_block=must_block) as in_file:
events.append(("START_READ", datetime.now()))
while True:
str = in_file.read(4096)
bytes_read += len(str)
if len(str) == 0:
break
if bytes_read >= next_threshold:
next_threshold += chunk_size
events.append(("READ_CHUNK", datetime.now()))
events.append(("FINISHED", datetime.now()))
with skypy.open_output(skypy.get_extra_output_indices()[0]) as log_out:
pickle.dump(events, log_out)
return "Read %d bytes" % bytes_read
def reader_function(refs):
print >> sys.stderr, "SkyPy example reader function:", len(refs), "inputs"
results = []
for ref in refs:
with skypy.deref_as_raw_file(ref) as in_file:
results.append(in_file.read())
with skypy.open_output(skypy.extra_outputs[0]) as file_out:
file_out.write("Complete read results: %s\n" % str(results))
return "Read %d results" % len(refs)
def counter(ref):
cached_result = soft_cache.try_get_cache([ref], "wc")
if cached_result is not None:
print "Counter: got cached result! It was", cached_result
else:
with skypy.deref_as_raw_file(ref, make_sweetheart=True) as in_fp:
cached_result = len(in_fp.read())
print "Counter: calculated result anew. It was", cached_result
soft_cache.put_cache([ref], "wc", cached_result)
return cached_result
def reader_function(refs):
print >>sys.stderr, "SkyPy example reader function:", len(refs), "inputs"
results = []
for ref in refs:
with skypy.deref_as_raw_file(ref) as in_file:
results.append(in_file.read())
with skypy.open_output(skypy.extra_outputs[0]) as file_out:
file_out.write("Complete read results: %s\n" % str(results))
return "Read %d results" % len(refs)
def stream_consumer(chunk_size, in_ref):
bytes_read = 0
with skypy.deref_as_raw_file(in_ref, may_stream=True, chunk_size=chunk_size) as in_file:
while True:
str = in_file.read(4096)
bytes_read += len(str)
if len(str) == 0:
break
return "Read %d bytes" % bytes_read
def stream_consumer(chunk_size, in_ref):
bytes_read = 0
with skypy.deref_as_raw_file(in_ref,
may_stream=True,
chunk_size=chunk_size) as in_file:
while True:
str = in_file.read(4096)
bytes_read += len(str)
if len(str) == 0:
break
return "Read %d bytes" % bytes_read
def stream_link(chunk_size, input_ref, may_stream, producer_pipe, consumer_pipe, must_block):
bytes_written = 0
# Convoluted structure to avoid blocking on a ref whilst we've got an output in progress
with skypy.open_output(skypy.get_extra_output_indices()[0], may_stream=may_stream, may_pipe=producer_pipe) as out_file:
with skypy.deref_as_raw_file(input_ref, may_stream=may_stream, sole_consumer=consumer_pipe, chunk_size=chunk_size, must_block=must_block) as in_file:
while True:
buf = in_file.read(4096)
if len(buf) == 0:
break
out_file.write(buf)
bytes_written += len(buf)
return "Read/wrote %d bytes" % bytes_written
import skypy
import sys
def skypy_main(run_seconds, async, direct):
if async.find("true") != -1:
may_stream = True
else:
may_stream = False
if direct.find("true") != -1:
try_direct = True
else:
try_direct = False
tests_jar = skypy.package_lookup("java_tests")
refs = skypy.spawn_exec("java", args={"inputs": [], "argv": [str(run_seconds)], "lib": [tests_jar], "class": "tests.JitteryProducer", "stream_output": True, "pipe_output": try_direct}, n_outputs=2)
got_bytes = 0
with skypy.deref_as_raw_file(refs[0], may_stream=may_stream, sole_consumer=try_direct, chunk_size=1048576) as file_in:
while True:
file_str = file_in.read(1048576)
if len(file_str) == 0:
break
print >>sys.stderr, "Read", len(file_str), "bytes"
got_bytes += len(file_str)
with skypy.deref_as_raw_file(refs[1]) as n_bytes:
byte_count = n_bytes.read()
return "Producer wrote %s, I got %d starting with %s" % (byte_count, got_bytes, file_str[:20])
def read_result(reader_result):
with skypy.RequiredRefs(list(reader_result)):
cooked_result = skypy.deref(reader_result.ret_output)
with skypy.deref_as_raw_file(reader_result.extra_outputs[0]) as in_file:
return (cooked_result, in_file.read())
from __future__ import with_statement
import skypy
import sys
def skypy_main(run_seconds, async):
if async.find("true") != -1:
may_stream = True
else:
may_stream = False
tests_jar = skypy.package_lookup("java_tests")
refs = skypy.spawn_exec("java", args={"inputs": [], "argv": [str(run_seconds)], "lib": [tests_jar], "class": "tests.JitteryProducer", "stream_output": True}, n_outputs=2)
got_bytes = 0
with skypy.deref_as_raw_file(refs[0], may_stream=may_stream, chunk_size=1048576) as file_in:
while True:
file_str = file_in.read(1048576)
if len(file_str) == 0:
break
print >>sys.stderr, "Read", len(file_str), "bytes"
got_bytes += len(file_str)
with skypy.deref_as_raw_file(refs[1]) as n_bytes:
byte_count = n_bytes.read()
return "Producer wrote %s, I got %d starting with %s" % (byte_count, got_bytes, file_str[:20])
may_stream = True
else:
may_stream = False
tests_jar = skypy.package_lookup("java_tests")
refs = skypy.spawn_exec("java",
args={
"inputs": [],
"argv": [str(run_seconds)],
"lib": [tests_jar],
"class": "tests.JitteryProducer",
"stream_output": True
},
n_outputs=2)
got_bytes = 0
with skypy.deref_as_raw_file(refs[0],
may_stream=may_stream,
chunk_size=1048576) as file_in:
while True:
file_str = file_in.read(1048576)
if len(file_str) == 0:
break
print >> sys.stderr, "Read", len(file_str), "bytes"
got_bytes += len(file_str)
with skypy.deref_as_raw_file(refs[1]) as n_bytes:
byte_count = n_bytes.read()
return "Producer wrote %s, I got %d starting with %s" % (
byte_count, got_bytes, file_str[:20])