def test_files_sink(sc): # Write initial test files path_template = '/tmp/files_source_test_{:d}' num_elements = 4 input_paths = [] for i in range(num_elements): path = path_template.format(i) with open(path, 'wb') as f: # Write data f.write(struct.pack('=Q', i)) input_paths.append(path) # Write output test files path_template = '/tmp/files_sink_test_{:d}' num_elements = 4 output_paths = [] for i in range(num_elements): path = path_template.format(i) output_paths.append(path) data = sc.io.Input([FilesStream(paths=input_paths)]) pass_data = sc.ops.Pass(input=data) output = FilesStream(paths=output_paths) output_op = sc.io.Output(pass_data, [output]) sc.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) # Read output test files for i, s in enumerate(output.load()): d, = struct.unpack('=Q', s) assert d == i
def test_files_source(sc): # Write test files path_template = '/tmp/files_source_test_{:d}' num_elements = 4 paths = [] for i in range(num_elements): path = path_template.format(i) with open(path, 'wb') as f: # Write data f.write(struct.pack('=Q', i)) paths.append(path) data = sc.io.Input([FilesStream(paths=paths)]) pass_data = sc.ops.Pass(input=data) output = NamedStream(sc, 'test_files_source') output_op = sc.io.Output(pass_data, [output]) sc.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) num_rows = 0 for buf in output.load(): (val, ) = struct.unpack('=Q', buf) assert val == num_rows num_rows += 1 assert num_elements == num_rows
def test_overlapping_slice(cl): input = NamedVideoStream(cl, 'test1') frame = cl.io.Input([input]) slice_frame = cl.streams.Slice(frame, partitions=[ cl.partitioner.strided_ranges( [(0, 15), (5, 25), (15, 35)], 1) ]) sample_frame = cl.streams.Range(slice_frame, ranges=[ SliceList([ { 'start': 0, 'end': 10 }, { 'start': 5, 'end': 15 }, { 'start': 5, 'end': 15 }, ]) ]) unsliced_frame = cl.streams.Unslice(sample_frame) output = NamedStream(cl, 'test_slicing') output_op = cl.io.Output(unsliced_frame, [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) assert output.len() == 30
def test_bind_op_args(cl): input = NamedVideoStream(cl, 'test1') frame = cl.io.Input([input, input]) range_frame = cl.streams.Range(frame, ranges=[{ 'start': 0, 'end': 1 } for _ in range(2)]) test_out = cl.ops.TestPy(frame=range_frame, kernel_arg=1, x=[1, 10], y=[5, 50]) outputs = [NamedStream(cl, 'test_hist_0'), NamedStream(cl, 'test_hist_1')] output_op = cl.io.Output(test_out, outputs) pairs = [(1, 5), (10, 50)] cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) for i, (x, y) in enumerate(pairs): values = list(outputs[i].load()) p = values[0] assert p['x'] == x assert p['y'] == y
def test_sql(sql_sc): (sc, storage, cur) = sql_sc cur.execute('SELECT COUNT(*) FROM test') n, = cur.fetchone() row = sc.io.Input([ SQLInputStream(query=protobufs.SQLQuery( fields='test.id as id, test.a, test.c, test.d, test.e, test.f', table='test', id='test.id', group='test.id'), filter='true', storage=storage, num_elements=n) ]) row2 = sc.ops.AddOne(row=row) output_op = sc.io.Output(row2, [ SQLOutputStream( table='test', storage=storage, job_name='foobar', insert=False) ]) sc.run(output_op, PerfParams.estimate()) cur.execute('SELECT b FROM test') assert cur.fetchone()[0] == 11 cur.execute('SELECT name FROM jobs') assert cur.fetchone()[0] == 'foobar'
def test_stencil(cl): input = NamedVideoStream(cl, 'test1') frame = cl.io.Input([input]) sample_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 1}]) flow = cl.ops.OpticalFlow(frame=sample_frame, stencil=[-1, 0]) output = NamedStream(cl, 'test_stencil') output_op = cl.io.Output(flow, [output]) cl.run(output_op, PerfParams.estimate(pipeline_instances_per_node=1), cache_mode=CacheMode.Overwrite, show_progress=False) assert output.len() == 1 frame = cl.io.Input([input]) sample_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 1}]) flow = cl.ops.OpticalFlow(frame=sample_frame, stencil=[0, 1]) output = NamedStream(cl, 'test_stencil') output_op = cl.io.Output(flow, [output]) cl.run(output_op, PerfParams.estimate(pipeline_instances_per_node=1), cache_mode=CacheMode.Overwrite, show_progress=False) frame = cl.io.Input([input]) sample_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 2}]) flow = cl.ops.OpticalFlow(frame=sample_frame, stencil=[0, 1]) output = NamedStream(cl, 'test_stencil') output_op = cl.io.Output(flow, [output]) cl.run(output_op, PerfParams.estimate(pipeline_instances_per_node=1), cache_mode=CacheMode.Overwrite, show_progress=False) assert output.len() == 2 frame = cl.io.Input([input]) flow = cl.ops.OpticalFlow(frame=frame, stencil=[-1, 0]) sample_flow = cl.streams.Range(flow, ranges=[{'start': 0, 'end': 1}]) output = NamedStream(cl, 'test_stencil') output_op = cl.io.Output(sample_flow, [output]) cl.run(output_op, PerfParams.estimate(pipeline_instances_per_node=1), cache_mode=CacheMode.Overwrite, show_progress=False) assert output.len() == 1
def test_captions(sc): caption_path = download_transcript() captions = sc.io.Input( [CaptionStream(caption_path, window_size=10.0, max_time=3600)]) ignored = sc.ops.DecodeCap(cap=captions) output = sc.io.Output(ignored, [NamedStream(sc, 'caption_test')]) sc.run(output, PerfParams.estimate(pipeline_instances_per_node=1), cache_mode=CacheMode.Overwrite)
def run_spacer_job(spacer, spacing): frame = cl.io.Input([NamedVideoStream(cl, 'test1')]) hist = cl.ops.Histogram(frame=frame) space_hist = spacer(input=hist, spacings=[spacing]) output = NamedStream(cl, 'test_space') output_op = cl.io.Output(space_hist, [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) return output
def run(self, cl, device): input = NamedVideoStream(cl, 'test1_inplace') frame = cl.io.Input([input]) hist = cl.ops.Histogram(frame=frame, device=device) output = NamedStream(cl, 'test_hist') output_op = cl.io.Output(hist, [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) next(output.load())
def test_python_stencil_batch_kernel(cl): input = NamedVideoStream(cl, 'test1') frame = cl.io.Input([input]) range_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 30}]) test_out = cl.ops.TestPyStencilBatch(frame=range_frame, batch=50) output = NamedStream(cl, 'test_hist') output_op = cl.io.Output(test_out, [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) next(output.load())
def test_slice(cl): input = NamedVideoStream(cl, 'test1') frame = cl.io.Input([input]) slice_frame = cl.streams.Slice(frame, partitions=[cl.partitioner.all(50)]) unsliced_frame = cl.streams.Unslice(slice_frame) output = NamedStream(cl, 'test_slicing') output_op = cl.io.Output(unsliced_frame, [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) assert input.len() == output.len()
def test_py_variadic(cl): input = NamedVideoStream(cl, 'test1') frame = cl.io.Input([input]) range_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 30}]) out_frame = cl.ops.TestPyVariadic(range_frame, range_frame, range_frame) output = NamedVideoStream(cl, 'test_variadic') output_op = cl.io.Output(out_frame.lossless(), [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) next(output.load())
def run(sc, op, name): vid = NamedVideoStream(sc, 'test1') inp = sc.io.Input([vid]) #f = sc.streams.Gather(inp, [list(range(1000))]) tf = op(frame=inp, batch=100, device=DeviceType.CPU) out = NamedStream(sc, 'qq') outp = sc.io.Output(tf, [out]) s = now() sc.run(outp, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, pipeline_instances_per_node=1) sc.table('qq').profiler().write_trace('{}.trace'.format(name)) print('{:.1f}s'.format(now() - s))
def test_stream_args(cl): frame = cl.io.Input([NamedVideoStream(cl, 'test1')]) resized_frame = cl.ops.Resize(frame=frame, width=[640], height=[480]) range_frame = cl.streams.Range(resized_frame, [(0, 10)]) output_stream = NamedVideoStream(cl, 'test_stream_args') output_op = cl.io.Output(range_frame, [output_stream]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) list(output_stream.load())
def test_lossless(cl): input = NamedVideoStream(cl, 'test1') frame = cl.io.Input([input]) range_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 30}]) blurred_frame = cl.ops.Blur(frame=range_frame, kernel_size=3, sigma=0.1) output = NamedVideoStream(cl, 'test_blur') output_op = cl.io.Output(blurred_frame.lossless(), [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) next(output.load())
def run_sampler_job(sampler, sampler_args, expected_rows): frame = cl.io.Input([NamedVideoStream(cl, 'test1')]) sample_frame = sampler(input=frame, **sampler_args) output = NamedVideoStream(cl, 'test_sample') output_op = cl.io.Output(sample_frame, [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) num_rows = len(list(output.load())) assert num_rows == expected_rows
def run_op(sc, op): input = NamedVideoStream(sc, 'test1') frame = sc.io.Input([input]) gather_frame = sc.streams.Gather(frame, [[0]]) faces = op(frame=gather_frame) output = NamedStream(sc, 'output') output_op = sc.io.Output(faces, [output]) sc.run(output_op, PerfParams.estimate(pipeline_instances_per_node=1), cache_mode=CacheMode.Overwrite, show_progress=False) return list(output.load())
def test_unbounded_state(cl): input = NamedVideoStream(cl, 'test1') frame = cl.io.Input([input]) slice_frame = cl.streams.Slice(frame, partitions=[cl.partitioner.all(50)]) increment = cl.ops.TestIncrementUnbounded(ignore=slice_frame) unsliced_increment = cl.streams.Unslice(increment) output = NamedStream(cl, 'test_unbounded_state') output_op = cl.io.Output(unsliced_increment, [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) assert output.len() == input.len()
def run_job(args_1, args_2): frame = cl.io.Input([NamedVideoStream(cl, 'test1')]) sample_frame_1 = cl.streams.Range(input=frame, ranges=[args_1]) sample_frame_2 = cl.streams.Range(input=frame, ranges=[args_2]) output_op_1 = cl.io.Output(sample_frame_1, [NamedVideoStream(cl, 'test_mp_1')]) output_op_2 = cl.io.Output(sample_frame_2, [NamedVideoStream(cl, 'test_mp_2')]) cl.run([output_op_1, output_op_2], PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False)
def test_auto_ingest(cl): (vid1_path, vid2_path) = download_videos() input = NamedVideoStream(cl, 'test3', path=vid1_path) frame = cl.io.Input([input]) hist = cl.ops.Histogram(frame=frame) output = NamedStream(cl, 'test_hist') output_op = cl.io.Output(hist, [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) run(['rm', '-rf', vid1_path, vid2_path])
def test_perf_params(cl): frame = cl.io.Input([NamedVideoStream(cl, 'test1')]) hist = cl.ops.Histogram(frame=frame) ghist = cl.streams.Gather(hist, [[0]]) output_op = cl.io.Output(ghist, [NamedStream(cl, '_ignore')]) cl.run(output_op, PerfParams.manual(10, 10), show_progress=False, cache_mode=CacheMode.Overwrite) cl.run(output_op, PerfParams.estimate(), show_progress=False, cache_mode=CacheMode.Overwrite)
def test_fetch_resources(cl): with tempfile.NamedTemporaryFile() as f: f.write(b'0') f.flush() input = NamedVideoStream(cl, 'test1') frame = cl.io.Input([input]) range_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 3}]) test_out = cl.ops.ResourceTest(frame=frame, path=f.name) output = NamedStream(cl, 'test_hist') output_op = cl.io.Output(test_out, [output]) cl.run(output_op, PerfParams.estimate(pipeline_instances_per_node=2), cache_mode=CacheMode.Overwrite, show_progress=False)
def test_slice_args(cl): frame = cl.io.Input([NamedVideoStream(cl, 'test1')]) slice_frame = cl.streams.Slice( frame, [cl.partitioner.ranges([[0, 1], [1, 2], [2, 3]])]) test = cl.ops.TestSliceArgs(frame=slice_frame, arg=[SliceList([i for i in range(3)])]) unsliced_frame = cl.streams.Unslice(test) output = NamedStream(cl, 'test_slicing') output_op = cl.io.Output(unsliced_frame, [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) num_rows = 0 list(output.load())
def test_save_mp4(cl): input = NamedVideoStream(cl, 'test1') frame = cl.io.Input([input]) range_frame = cl.streams.Range(frame, ranges=[{'start': 0, 'end': 30}]) blurred_frame = cl.ops.Blur(frame=range_frame, kernel_size=3, sigma=0.1) output = NamedVideoStream(cl, 'test_save_mp4') output_op = cl.io.Output(blurred_frame, [output]) cl.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) f = tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') f.close() output.save_mp4(f.name) run(['rm', '-rf', f.name])
def test_blur(sc): input = NamedVideoStream(sc, 'test1') frame = sc.io.Input([input]) range_frame = sc.streams.Range(frame, ranges=[{'start': 0, 'end': 30}]) blurred_frame = sc.ops.Blur(frame=range_frame, kernel_size=3, sigma=0.1) output = NamedVideoStream(sc, 'test_blur') output_op = sc.io.Output(blurred_frame, [output]) sc.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) frame_array = next(output.load()) assert frame_array.dtype == np.uint8 assert frame_array.shape[0] == 480 assert frame_array.shape[1] == 640 assert frame_array.shape[2] == 3
def test_no_workers(no_workers_cl): cl = no_workers_cl input = NamedVideoStream(cl, 'test1') frame = cl.io.Input([input]) hist = cl.ops.Histogram(frame=frame) output_op = cl.io.Output(hist, [NamedStream(cl, '_ignore')]) exc = False try: cl.run(output_op, PerfParams.estimate(), show_progress=False, cache_mode=CacheMode.Overwrite) except ScannerException: exc = True assert exc
def test_profiler(cl): frame = cl.io.Input([NamedVideoStream(cl, 'test1')]) hist = cl.ops.Histogram(frame=frame) ghist = cl.streams.Gather(hist, [[0]]) output_op = cl.io.Output(ghist, [NamedStream(cl, '_ignore')]) time_start = time.time() job_id = cl.run(output_op, PerfParams.estimate(), show_progress=False, cache_mode=CacheMode.Overwrite) print('Time', time.time() - time_start) profile = cl.get_profile(job_id) f = tempfile.NamedTemporaryFile(delete=False, suffix='.trace') f.close() profile.write_trace(f.name) profile.statistics() run(['rm', '-f', f.name])
def run(self, sc, device): input = NamedVideoStream(sc, 'test1') frame = sc.io.Input([input]) flow = sc.ops.OpticalFlow(frame=frame, stencil=[-1, 0], device=device) flow_range = sc.streams.Range(flow, ranges=[{'start': 0, 'end': 50}]) output = NamedStream(sc, 'test_flow') output_op = sc.io.Output(flow_range, [output]) sc.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) assert output.len() == 50 flow_array = next(output.load()) assert flow_array.dtype == np.float32 assert flow_array.shape[0] == 480 assert flow_array.shape[1] == 640 assert flow_array.shape[2] == 2
def test_pose(sc): vid = [NamedVideoStream(sc, 'test1')] frame = sc.io.Input(vid) frame_sample = sc.streams.Gather(frame, [list(range(0, 1000, 100))]) pose = sc.ops.OpenPose( frame=frame_sample, device=DeviceType.GPU, pose_num_scales=6, pose_scale_gap=0.16, compute_hands=True, hand_num_scales=6, hand_scale_gap=0.16, compute_face=True, batch=5 ) output = NamedStream(sc, 'test1-pose') output_op = sc.io.Output(pose, [output]) sc.run(output_op, PerfParams.estimate())
def test_python_source(sc): # Write test files py_data = [{'{:d}'.format(i): i} for i in range(4)] data = sc.io.Input([PythonStream(py_data)]) pass_data = sc.ops.Pass(input=data) output = NamedStream(sc, 'test_python_source') output_op = sc.io.Output(pass_data, [output]) sc.run(output_op, PerfParams.estimate(), cache_mode=CacheMode.Overwrite, show_progress=False) num_rows = 0 for i, buf in enumerate(output.load()): d = pickle.loads(buf) assert d['{:d}'.format(i)] == i num_rows += 1 assert num_rows == 4