def test_files_sink(sc):
# Write initial test files
path_template = '/tmp/files_source_test_{:d}'
num_elements = 4
input_paths = []
for i in range(num_elements):
path = path_template.format(i)
with open(path, 'wb') as f:
# Write data
f.write(struct.pack('=Q', i))
input_paths.append(path)
# Write output test files
path_template = '/tmp/files_sink_test_{:d}'
num_elements = 4
output_paths = []
for i in range(num_elements):
path = path_template.format(i)
output_paths.append(path)
data = sc.io.Input([FilesStream(paths=input_paths)])
pass_data = sc.ops.Pass(input=data)
output = FilesStream(paths=output_paths)
output_op = sc.io.Output(pass_data, [output])
sc.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
# Read output test files
for i, s in enumerate(output.load()):
d, = struct.unpack('=Q', s)
assert d == i
def test_files_source(sc):
# Write test files
path_template = '/tmp/files_source_test_{:d}'
num_elements = 4
paths = []
for i in range(num_elements):
path = path_template.format(i)
with open(path, 'wb') as f:
# Write data
f.write(struct.pack('=Q', i))
paths.append(path)
data = sc.io.Input([FilesStream(paths=paths)])
pass_data = sc.ops.Pass(input=data)
output = NamedStream(sc, 'test_files_source')
output_op = sc.io.Output(pass_data, [output])
sc.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
num_rows = 0
for buf in output.load():
(val, ) = struct.unpack('=Q', buf)
assert val == num_rows
num_rows += 1
assert num_elements == num_rows
def test_overlapping_slice(cl):
input = NamedVideoStream(cl, 'test1')
frame = cl.io.Input([input])
slice_frame = cl.streams.Slice(frame,
partitions=[
cl.partitioner.strided_ranges(
[(0, 15), (5, 25), (15, 35)], 1)
])
sample_frame = cl.streams.Range(slice_frame,
ranges=[
SliceList([
{
'start': 0,
'end': 10
},
{
'start': 5,
'end': 15
},
{
'start': 5,
'end': 15
},
])
])
unsliced_frame = cl.streams.Unslice(sample_frame)
output = NamedStream(cl, 'test_slicing')
output_op = cl.io.Output(unsliced_frame, [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
assert output.len() == 30
def test_bind_op_args(cl):
input = NamedVideoStream(cl, 'test1')
frame = cl.io.Input([input, input])
range_frame = cl.streams.Range(frame,
ranges=[{
'start': 0,
'end': 1
} for _ in range(2)])
test_out = cl.ops.TestPy(frame=range_frame,
kernel_arg=1,
x=[1, 10],
y=[5, 50])
outputs = [NamedStream(cl, 'test_hist_0'), NamedStream(cl, 'test_hist_1')]
output_op = cl.io.Output(test_out, outputs)
pairs = [(1, 5), (10, 50)]
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
for i, (x, y) in enumerate(pairs):
values = list(outputs[i].load())
p = values[0]
assert p['x'] == x
assert p['y'] == y
def test_sql(sql_sc):
(sc, storage, cur) = sql_sc
cur.execute('SELECT COUNT(*) FROM test')
n, = cur.fetchone()
row = sc.io.Input([
SQLInputStream(query=protobufs.SQLQuery(
fields='test.id as id, test.a, test.c, test.d, test.e, test.f',
table='test',
id='test.id',
group='test.id'),
filter='true',
storage=storage,
num_elements=n)
])
row2 = sc.ops.AddOne(row=row)
output_op = sc.io.Output(row2, [
SQLOutputStream(
table='test', storage=storage, job_name='foobar', insert=False)
])
sc.run(output_op, PerfParams.estimate())
cur.execute('SELECT b FROM test')
assert cur.fetchone()[0] == 11
cur.execute('SELECT name FROM jobs')
assert cur.fetchone()[0] == 'foobar'
def test_stencil(cl):
input = NamedVideoStream(cl, 'test1')
frame = cl.io.Input([input])
sample_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 1}])
flow = cl.ops.OpticalFlow(frame=sample_frame, stencil=[-1, 0])
output = NamedStream(cl, 'test_stencil')
output_op = cl.io.Output(flow, [output])
cl.run(output_op,
PerfParams.estimate(pipeline_instances_per_node=1),
cache_mode=CacheMode.Overwrite,
show_progress=False)
assert output.len() == 1
frame = cl.io.Input([input])
sample_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 1}])
flow = cl.ops.OpticalFlow(frame=sample_frame, stencil=[0, 1])
output = NamedStream(cl, 'test_stencil')
output_op = cl.io.Output(flow, [output])
cl.run(output_op,
PerfParams.estimate(pipeline_instances_per_node=1),
cache_mode=CacheMode.Overwrite,
show_progress=False)
frame = cl.io.Input([input])
sample_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 2}])
flow = cl.ops.OpticalFlow(frame=sample_frame, stencil=[0, 1])
output = NamedStream(cl, 'test_stencil')
output_op = cl.io.Output(flow, [output])
cl.run(output_op,
PerfParams.estimate(pipeline_instances_per_node=1),
cache_mode=CacheMode.Overwrite,
show_progress=False)
assert output.len() == 2
frame = cl.io.Input([input])
flow = cl.ops.OpticalFlow(frame=frame, stencil=[-1, 0])
sample_flow = cl.streams.Range(flow, ranges=[{'start': 0, 'end': 1}])
output = NamedStream(cl, 'test_stencil')
output_op = cl.io.Output(sample_flow, [output])
cl.run(output_op,
PerfParams.estimate(pipeline_instances_per_node=1),
cache_mode=CacheMode.Overwrite,
show_progress=False)
assert output.len() == 1
def test_captions(sc):
caption_path = download_transcript()
captions = sc.io.Input(
[CaptionStream(caption_path, window_size=10.0, max_time=3600)])
ignored = sc.ops.DecodeCap(cap=captions)
output = sc.io.Output(ignored, [NamedStream(sc, 'caption_test')])
sc.run(output,
PerfParams.estimate(pipeline_instances_per_node=1),
cache_mode=CacheMode.Overwrite)
def run_spacer_job(spacer, spacing):
frame = cl.io.Input([NamedVideoStream(cl, 'test1')])
hist = cl.ops.Histogram(frame=frame)
space_hist = spacer(input=hist, spacings=[spacing])
output = NamedStream(cl, 'test_space')
output_op = cl.io.Output(space_hist, [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
return output
def run(self, cl, device):
input = NamedVideoStream(cl, 'test1_inplace')
frame = cl.io.Input([input])
hist = cl.ops.Histogram(frame=frame, device=device)
output = NamedStream(cl, 'test_hist')
output_op = cl.io.Output(hist, [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
next(output.load())
def test_python_stencil_batch_kernel(cl):
input = NamedVideoStream(cl, 'test1')
frame = cl.io.Input([input])
range_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 30}])
test_out = cl.ops.TestPyStencilBatch(frame=range_frame, batch=50)
output = NamedStream(cl, 'test_hist')
output_op = cl.io.Output(test_out, [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
next(output.load())
def test_slice(cl):
input = NamedVideoStream(cl, 'test1')
frame = cl.io.Input([input])
slice_frame = cl.streams.Slice(frame, partitions=[cl.partitioner.all(50)])
unsliced_frame = cl.streams.Unslice(slice_frame)
output = NamedStream(cl, 'test_slicing')
output_op = cl.io.Output(unsliced_frame, [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
assert input.len() == output.len()
def test_py_variadic(cl):
input = NamedVideoStream(cl, 'test1')
frame = cl.io.Input([input])
range_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 30}])
out_frame = cl.ops.TestPyVariadic(range_frame, range_frame, range_frame)
output = NamedVideoStream(cl, 'test_variadic')
output_op = cl.io.Output(out_frame.lossless(), [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
next(output.load())
def run(sc, op, name):
vid = NamedVideoStream(sc, 'test1')
inp = sc.io.Input([vid])
#f = sc.streams.Gather(inp, [list(range(1000))])
tf = op(frame=inp, batch=100, device=DeviceType.CPU)
out = NamedStream(sc, 'qq')
outp = sc.io.Output(tf, [out])
s = now()
sc.run(outp, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, pipeline_instances_per_node=1)
sc.table('qq').profiler().write_trace('{}.trace'.format(name))
print('{:.1f}s'.format(now() - s))
def test_stream_args(cl):
frame = cl.io.Input([NamedVideoStream(cl, 'test1')])
resized_frame = cl.ops.Resize(frame=frame, width=[640], height=[480])
range_frame = cl.streams.Range(resized_frame, [(0, 10)])
output_stream = NamedVideoStream(cl, 'test_stream_args')
output_op = cl.io.Output(range_frame, [output_stream])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
list(output_stream.load())
def test_lossless(cl):
input = NamedVideoStream(cl, 'test1')
frame = cl.io.Input([input])
range_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 30}])
blurred_frame = cl.ops.Blur(frame=range_frame, kernel_size=3, sigma=0.1)
output = NamedVideoStream(cl, 'test_blur')
output_op = cl.io.Output(blurred_frame.lossless(), [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
next(output.load())
def run_sampler_job(sampler, sampler_args, expected_rows):
frame = cl.io.Input([NamedVideoStream(cl, 'test1')])
sample_frame = sampler(input=frame, **sampler_args)
output = NamedVideoStream(cl, 'test_sample')
output_op = cl.io.Output(sample_frame, [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
num_rows = len(list(output.load()))
assert num_rows == expected_rows
def run_op(sc, op):
input = NamedVideoStream(sc, 'test1')
frame = sc.io.Input([input])
gather_frame = sc.streams.Gather(frame, [[0]])
faces = op(frame=gather_frame)
output = NamedStream(sc, 'output')
output_op = sc.io.Output(faces, [output])
sc.run(output_op,
PerfParams.estimate(pipeline_instances_per_node=1),
cache_mode=CacheMode.Overwrite,
show_progress=False)
return list(output.load())
def test_unbounded_state(cl):
input = NamedVideoStream(cl, 'test1')
frame = cl.io.Input([input])
slice_frame = cl.streams.Slice(frame, partitions=[cl.partitioner.all(50)])
increment = cl.ops.TestIncrementUnbounded(ignore=slice_frame)
unsliced_increment = cl.streams.Unslice(increment)
output = NamedStream(cl, 'test_unbounded_state')
output_op = cl.io.Output(unsliced_increment, [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
assert output.len() == input.len()
def run_job(args_1, args_2):
frame = cl.io.Input([NamedVideoStream(cl, 'test1')])
sample_frame_1 = cl.streams.Range(input=frame, ranges=[args_1])
sample_frame_2 = cl.streams.Range(input=frame, ranges=[args_2])
output_op_1 = cl.io.Output(sample_frame_1,
[NamedVideoStream(cl, 'test_mp_1')])
output_op_2 = cl.io.Output(sample_frame_2,
[NamedVideoStream(cl, 'test_mp_2')])
cl.run([output_op_1, output_op_2],
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
def test_auto_ingest(cl):
(vid1_path, vid2_path) = download_videos()
input = NamedVideoStream(cl, 'test3', path=vid1_path)
frame = cl.io.Input([input])
hist = cl.ops.Histogram(frame=frame)
output = NamedStream(cl, 'test_hist')
output_op = cl.io.Output(hist, [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
run(['rm', '-rf', vid1_path, vid2_path])
def test_perf_params(cl):
frame = cl.io.Input([NamedVideoStream(cl, 'test1')])
hist = cl.ops.Histogram(frame=frame)
ghist = cl.streams.Gather(hist, [[0]])
output_op = cl.io.Output(ghist, [NamedStream(cl, '_ignore')])
cl.run(output_op,
PerfParams.manual(10, 10),
show_progress=False,
cache_mode=CacheMode.Overwrite)
cl.run(output_op,
PerfParams.estimate(),
show_progress=False,
cache_mode=CacheMode.Overwrite)
def test_fetch_resources(cl):
with tempfile.NamedTemporaryFile() as f:
f.write(b'0')
f.flush()
input = NamedVideoStream(cl, 'test1')
frame = cl.io.Input([input])
range_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 3}])
test_out = cl.ops.ResourceTest(frame=frame, path=f.name)
output = NamedStream(cl, 'test_hist')
output_op = cl.io.Output(test_out, [output])
cl.run(output_op,
PerfParams.estimate(pipeline_instances_per_node=2),
cache_mode=CacheMode.Overwrite,
show_progress=False)
def test_slice_args(cl):
frame = cl.io.Input([NamedVideoStream(cl, 'test1')])
slice_frame = cl.streams.Slice(
frame, [cl.partitioner.ranges([[0, 1], [1, 2], [2, 3]])])
test = cl.ops.TestSliceArgs(frame=slice_frame,
arg=[SliceList([i for i in range(3)])])
unsliced_frame = cl.streams.Unslice(test)
output = NamedStream(cl, 'test_slicing')
output_op = cl.io.Output(unsliced_frame, [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
num_rows = 0
list(output.load())
def test_save_mp4(cl):
input = NamedVideoStream(cl, 'test1')
frame = cl.io.Input([input])
range_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 30}])
blurred_frame = cl.ops.Blur(frame=range_frame, kernel_size=3, sigma=0.1)
output = NamedVideoStream(cl, 'test_save_mp4')
output_op = cl.io.Output(blurred_frame, [output])
cl.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
f = tempfile.NamedTemporaryFile(delete=False, suffix='.mp4')
f.close()
output.save_mp4(f.name)
run(['rm', '-rf', f.name])
def test_blur(sc):
input = NamedVideoStream(sc, 'test1')
frame = sc.io.Input([input])
range_frame = sc.streams.Range(frame, ranges=[{'start': 0, 'end': 30}])
blurred_frame = sc.ops.Blur(frame=range_frame, kernel_size=3, sigma=0.1)
output = NamedVideoStream(sc, 'test_blur')
output_op = sc.io.Output(blurred_frame, [output])
sc.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
frame_array = next(output.load())
assert frame_array.dtype == np.uint8
assert frame_array.shape[0] == 480
assert frame_array.shape[1] == 640
assert frame_array.shape[2] == 3
def test_no_workers(no_workers_cl):
cl = no_workers_cl
input = NamedVideoStream(cl, 'test1')
frame = cl.io.Input([input])
hist = cl.ops.Histogram(frame=frame)
output_op = cl.io.Output(hist, [NamedStream(cl, '_ignore')])
exc = False
try:
cl.run(output_op,
PerfParams.estimate(),
show_progress=False,
cache_mode=CacheMode.Overwrite)
except ScannerException:
exc = True
assert exc
def test_profiler(cl):
frame = cl.io.Input([NamedVideoStream(cl, 'test1')])
hist = cl.ops.Histogram(frame=frame)
ghist = cl.streams.Gather(hist, [[0]])
output_op = cl.io.Output(ghist, [NamedStream(cl, '_ignore')])
time_start = time.time()
job_id = cl.run(output_op,
PerfParams.estimate(),
show_progress=False,
cache_mode=CacheMode.Overwrite)
print('Time', time.time() - time_start)
profile = cl.get_profile(job_id)
f = tempfile.NamedTemporaryFile(delete=False, suffix='.trace')
f.close()
profile.write_trace(f.name)
profile.statistics()
run(['rm', '-f', f.name])
def run(self, sc, device):
input = NamedVideoStream(sc, 'test1')
frame = sc.io.Input([input])
flow = sc.ops.OpticalFlow(frame=frame, stencil=[-1, 0], device=device)
flow_range = sc.streams.Range(flow, ranges=[{'start': 0, 'end': 50}])
output = NamedStream(sc, 'test_flow')
output_op = sc.io.Output(flow_range, [output])
sc.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
assert output.len() == 50
flow_array = next(output.load())
assert flow_array.dtype == np.float32
assert flow_array.shape[0] == 480
assert flow_array.shape[1] == 640
assert flow_array.shape[2] == 2
def test_pose(sc):
vid = [NamedVideoStream(sc, 'test1')]
frame = sc.io.Input(vid)
frame_sample = sc.streams.Gather(frame, [list(range(0, 1000, 100))])
pose = sc.ops.OpenPose(
frame=frame_sample,
device=DeviceType.GPU,
pose_num_scales=6,
pose_scale_gap=0.16,
compute_hands=True,
hand_num_scales=6,
hand_scale_gap=0.16,
compute_face=True,
batch=5
)
output = NamedStream(sc, 'test1-pose')
output_op = sc.io.Output(pose, [output])
sc.run(output_op, PerfParams.estimate())
def test_python_source(sc):
# Write test files
py_data = [{'{:d}'.format(i): i} for i in range(4)]
data = sc.io.Input([PythonStream(py_data)])
pass_data = sc.ops.Pass(input=data)
output = NamedStream(sc, 'test_python_source')
output_op = sc.io.Output(pass_data, [output])
sc.run(output_op,
PerfParams.estimate(),
cache_mode=CacheMode.Overwrite,
show_progress=False)
num_rows = 0
for i, buf in enumerate(output.load()):
d = pickle.loads(buf)
assert d['{:d}'.format(i)] == i
num_rows += 1
assert num_rows == 4
