def main(): # Look at resize_op/resize_op.cpp to start this tutorial. db = Database() cwd = os.path.dirname(os.path.abspath(__file__)) if not os.path.isfile(os.path.join(cwd, 'resize_op/build/libresize_op.so')): print( 'You need to build the custom op first: \n' '$ pushd {}/resize_op; mkdir build && cd build; cmake ..; make; popd' .format(cwd)) exit() # To load a custom op into the Scanner runtime, we use db.load_op to open the # shared library we compiled. If the op takes arguments, it also optionally # takes a path to the generated python file for the arg protobuf. db.load_op(os.path.join(cwd, 'resize_op/build/libresize_op.so'), os.path.join(cwd, 'resize_op/build/resize_pb2.py')) frame = db.sources.FrameColumn() # Then we use our op just like in the other examples. resize = db.ops.MyResize(frame=frame, width=200, height=300) output_op = db.sinks.Column(columns={'resized_frame': resize}) job = Job(op_args={ frame: db.table('example').column('frame'), output_op: 'example_resized', }) db.run(output_op, [job], force=True)
def main(): db = Database() # What if, instead of a video, you had a list of image files that you # wanted to process? Scanner provides an extensible interface for reading and # writing data to locations other than the database. # For example, let's download a few images now and create a list of their paths: util.download_images() image_paths = [ 'sample-frame-1.jpg', 'sample-frame-2.jpg', 'sample-frame-3.jpg' ] # Scanner provides a built-in source to read files from the local filesystem: compressed_images = db.sources.Files() # Like with db.sources.FrameColumn, we will bind the inputs to this source when # we define a job later on. # Let's write a pipeline that reads our images, resizes them, and writes them # back out as files to the filesystem. # Since the input images are compressed, we decompress them with the # ImageDecoder frame = db.ops.ImageDecoder(img=compressed_images) resized = db.ops.Resize(frame=frame, width=640, height=360) # Rencode the image to jpg encoded_frame = db.ops.ImageEncoder(frame=resized, format='jpg') # Write the compressed images to files output = db.sinks.Files(input=encoded_frame) resized_paths = ['resized-1.jpg', 'resized-2.jpg', 'resized-3.jpg'] job = Job( op_args={ compressed_images: { 'paths': image_paths }, output: { 'paths': resized_paths } }) db.run(output=output, jobs=[job]) print('Finished! Wrote the following images: ' + ', '.join(resized_paths))
def main(): db = Database() # Frames on disk can either be stored uncompressed (raw bits) or compressed # (encoded using some form of image or video compression). When Scanner # reads frames from a table, it automatically decodes the data if necessary. # The Op DAG only sees the raw frames. For example, this table is stored # as compressed video. def make_blurred_frame(): frame = db.sources.FrameColumn() blurred_frame = db.ops.Blur(frame=frame, kernel_size=3, sigma=0.5) sampled_frame = db.streams.Range(blurred_frame, 0, 30) return frame, sampled_frame # By default, if an Op outputs a frame with 3 channels with type uint8, # those frames will be compressed using video encoding. No other frame # type is currently compressed. frame, blurred_frame = make_blurred_frame() output_op = db.sinks.Column(columns={'frame': blurred_frame}) job = Job( op_args={ frame: db.table('example').column('frame'), output_op: 'output_table_name', }) db.run(output_op, [job], force=True) frame, blurred_frame = make_blurred_frame() # The compression parameters can be controlled by annotating the column low_quality_frame = blurred_frame.compress_video(quality=35) output_op = db.sinks.Column(columns={'frame': low_quality_frame}) job = Job( op_args={ frame: db.table('example').column('frame'), output_op: 'low_quality_table', }) db.run(output_op, [job], force=True) # If no compression is desired, this can be specified by indicating that # the column should be lossless. frame, blurred_frame = make_blurred_frame() # The compression parameters can be controlled by annotating the column lossless_frame = blurred_frame.lossless() output_op = db.sinks.Column(columns={'frame': lossless_frame}) job = Job(op_args={ frame: db.table('example').column('frame'), output_op: 'pristine_frame', }) db.run(output_op, [job], force=True) # Any column which is saved as compressed video can be exported as an mp4 # file by calling save_mp4 on the column. This will output a file called # 'low_quality_video.mp4' in the current directory. db.table('low_quality_table').column('frame').save_mp4('low_quality_video')
def main(): db = Database() frame = db.sources.FrameColumn() histogram = db.ops.Histogram(frame=frame) output_op = db.sinks.Column(columns={'hist': histogram}) job = Job( op_args={ frame: db.table('example').column('frame'), output_op: 'example_hist_profile' }) [output_table] = db.run(output_op, [job], force=True) # The profiler contains information about how long different parts of your # computation take to run. We use Google Chrome's trace format, which you # can view by going to chrome://tracing in Chrome and clicking "load" in # the top left. output_table.profiler().write_trace('hist.trace')
def main(): db = Database() frame = db.sources.FrameColumn() # You can tell Scanner which frames of the video (or which rows of a video # table) you want to sample. Here, we indicate that we want to stride # the frame column by 4 (select every 4th frame) strided_frame = db.streams.Stride(frame, 4) # We process the sampled frame same as before. hist = db.ops.Histogram(frame=strided_frame) output_op = db.sinks.Column(columns={'hist': hist}) # For each job, you can specify how sampling should be performed for # a specific stream. In the same way we used the op_args argument to bind # a table to an input column, we can bind sampling arguments to strided_frame # to override the default striding of 4 we specified above job = Job( op_args={ frame: db.table('example').column('frame'), # The "strided" sampling mode will run over every 8th frame, # i.e. frames [0, 8, 16, ...] strided_frame: 8, output_op: 'example_hist_strided' }) output_tables = db.run(output_op, [job], force=True) # Loop over the column's rows. Each row is a tuple of the frame number and # value for that row. video_hists = output_tables[0].column('hist').load(readers.histograms) num_rows = 0 for frame_hists in video_hists: assert len(frame_hists) == 3 assert frame_hists[0].shape[0] == 16 num_rows += 1 assert num_rows == round(db.table('example').num_rows() / 8) # Here's some examples of other sampling modes. # Range takes a specific subset of a video. Here, it runs over all frames # from 0 to 100 db.streams.Range(frame, 0, 100) # Gather takes an arbitrary list of frames from a video. db.streams.Gather(frame, [10, 17, 32])
def main(): # Now we can use these new Ops in Scanner: db = Database() # Download an example video example_video_path = util.download_video() # Ingest it into the database [input_table], _ = db.ingest_videos([('example', example_video_path)], force=True) frame = db.sources.FrameColumn() resized_frame_fn = db.ops.resize_fn(frame=frame, width=640, height=480) resized_frame_class = db.ops.ResizeClass(frame=frame, width=320, height=240) output = db.sinks.FrameColumn(columns={ 'frame1': resized_frame_fn, 'frame2': resized_frame_class }) job = Job(op_args={ frame: input_table.column('frame'), output: 'example_python_op' }) [table] = db.run(output=output, jobs=[job], force=True) table.column('frame1').save_mp4('01_resized_fn') table.column('frame2').save_mp4('01_resized_class') print('Finished! Two videos were saved to the current directory: ' '01_resized_fn.mp4, 01_resized_class.mp4')
end=args.num_reg_images) R, T, K, width, height, scan_width, bitmap, depth_min, depth_max, id = db.ops.PreparePatchMatch( sparse_reconstruction_path=args.input_path, batch=args.num_reg_images, image_id=image_id_sampled) output = db.sinks.Column( columns={ 'R': R, 'T': T, 'K': K, 'width': width, 'height': height, 'scan_width': scan_width, 'bitmap': bitmap, 'depth_min': depth_min, 'depth_max': depth_max, 'image_id': id, }) job = Job(op_args={ image_id: db.table('frames').column('image_id'), output: args.output_table, }) output_tables = db.run(output, [job], force=True, work_packet_size=args.num_reg_images, io_packet_size=args.num_reg_images) print(db.summarize())
# we define a job later on. # Let's write a pipeline that reads our images, resizes them, and writes them # back out as files to the filesystem. # Since the input images are compressed, we decompress them with the # ImageDecoder frame = db.ops.ImageDecoder(img=compressed_images) resized = db.ops.Resize(frame=frame, width=640, height=360) # Rencode the image to jpg encoded_frame = db.ops.ImageEncoder(frame=resized, format='jpg') # Write the compressed images to files output = db.sinks.Files(input=encoded_frame) resized_paths = ['resized-1.jpg', 'resized-2.jpg', 'resized-3.jpg'] job = Job(op_args={ compressed_images: {'paths': image_paths}, output: {'paths': resized_paths} }) db.run(output=output, jobs=[job]) print('Finished! Wrote the following images: ' + ', '.join(resized_paths)) # If you want to learn how write your own custom Source or Sink in C++, check # out tutorials 09_defining_cpp_sources.py and 10_defining_cpp_sinks.py
encoded_image: { 'paths': image_files, **params }, encoded_mask: { 'paths': mask_files, **params }, output_op: 'example_resized55' }) start = time.time() [out_table ] = db.run(output_op, [job], force=True, work_packet_size=opt.work_packet_size, io_packet_size=opt.io_packet_size, pipeline_instances_per_node=opt.pipeline_instances_per_node) end = time.time() print('scanner distance transform: {0:.4f} for {1} frames'.format( end - start, len(image_files))) if opt.save: # out_table.profiler().write_trace(join(dataset, 'calib.trace')) # print('Trace saved in {0}'.format(join(dataset, 'calib.trace'))) results = out_table.column('frame').load() dist_transf_list = [res[:, :, 0] for res in results] A, R, T = cam_data['A'], cam_data['R'], cam_data['T'] h, w = 1080, 1920
job = Job( op_args={ encoded_image: { 'paths': image_files, **params }, encoded_mask: { 'paths': mask_files, **params }, output_op: 'example_resized5', }) start = time.time() [out_table] = db.run(output_op, [job], force=True, work_packet_size=16, io_packet_size=16) end = time.time() print('Total time for depth estimation in scanner: {0:.3f} sec'.format( end - start)) results = out_table.column('frame').load() # path_to_save = join(dataset, 'players', 'prediction_scanner') # if not os.path.exists(path_to_save): # os.mkdir(path_to_save) # # for i, res in enumerate(results): # pred_scanner = np.argmax(res, axis=0) # np.save(join(path_to_save, '{0:05d}.npy'.format(i)), res)
table_input = db.ops.Input() caffe_input = db.ops.FacenetInput(inputs=[(table_input, ["frame", "frame_info"])], args=facenet_args, device=DeviceType.GPU) caffe = db.ops.Facenet(inputs=[(caffe_input, ["caffe_frame"]), (table_input, ["frame_info"])], args=facenet_args, device=DeviceType.GPU) caffe_output = db.ops.FacenetOutput(inputs=[(caffe, ["caffe_output"]), (table_input, ["frame_info"])], args=facenet_args) input_collection = db.ingest_video_collection('test', ['test.mp4']) output_collection = db.run(input_collection, caffe_output, 'test_faces') def parse_bboxes(db, buf): (num_bboxes, ) = struct.unpack("=Q", buf[:8]) buf = buf[8:] bboxes = [] for i in range(num_bboxes): (bbox_size, ) = struct.unpack("=i", buf[:4]) buf = buf[4:] box = db.protobufs.BoundingBox() box.ParseFromString(buf[:bbox_size]) buf = buf[bbox_size:] bbox = [ box.x1, box.y1, box.x2, box.y2, box.score, box.track_id, box.track_score
from scannerpy import Database, Job, DeviceType ################################################################################ # This tutorial shows how to look at profiling information for your job. # ################################################################################ db = Database() frame = db.sources.FrameColumn() histogram = db.ops.Histogram(frame=frame) output_op = db.sinks.Column(columns={'hist': histogram}) job = Job(op_args={ frame: db.table('example').column('frame'), output_op: 'example_hist_profile' }) [output_table] = db.run(output_op, [job], force=True) # The profiler contains information about how long different parts of your # computation take to run. We use Google Chrome's trace format, which you # can view by going to chrome://tracing in Chrome and clicking "load" in # the top left. output_table.profiler().write_trace('hist.trace') # Each row corresponds to a different part of the system, e.g. the thread # loading bytes from disk or the thread running your kernels. If you have # multiple pipelines or multiple nodes, you will see many of these evaluate # threads.
db = Database() video_path = movie_path if not db.has_table(movie_name): print('Ingesting video into Scanner ...') db.ingest_videos([(movie_name, video_path)], force=True) input_table = db.table(movie_name) sampler = db.streams.Range sampler_args = {'start': 120, 'end': 480} [poses_table] = pipelines.detect_poses(db, [input_table.column('frame')], sampler, sampler_args, '{:s}_poses'.format(movie_name)) print('Drawing on frames...') frame = db.sources.FrameColumn() sampled_frame = sampler(frame) poses = db.sources.Column() drawn_frame = db.ops.PoseDraw(frame=sampled_frame, poses=poses) output = db.sinks.Column(columns={'frame': drawn_frame}) job = Job( op_args={ frame: input_table.column('frame'), sampled_frame: sampler_args, poses: poses_table.column('poses'), output: movie_name + '_drawn_poses', }) [drawn_poses_table] = db.run(output=output, jobs=[job], force=True) print('Writing output video...') drawn_poses_table.column('frame').save_mp4('{:s}_poses'.format(movie_name))
# Ingest the videos into the database db.ingest_videos(videos_to_process) # Define the same Computation Graph frame = db.sources.FrameColumn() # Read from the database sampled_frame = db.streams.Stride(frame, 3) # Select every third frame resized = db.ops.Resize(frame=sampled_frame, width=640, height=480) # Resize input frame output_frame = db.sinks.Column(columns={'frame': resized}) # Save resized frame # Create multiple jobs now, one for each video we want to process jobs = [] for table_name, _ in videos_to_process: job = Job( op_args={ frame: db.table(table_name).column( 'frame'), # Column to read input frames from output_frame: '{:s}-resized'.format( table_name) # Name of output table }) jobs.append(job) # Execute the computation graph and return a handle to the newly produced tables output_tables = db.run(output=output_frame, jobs=jobs, force=True) # Save the resized video as an mp4 file for output_table in output_tables: output_table.column('frame').save_mp4(output_table.name())
[input_table], _ = db.ingest_videos([('example', example_video_path)], force=True) frame = db.sources.FrameColumn() resized_frame_fn = db.ops.resize_fn(frame=frame, width=640, height=480) resized_frame_class = db.ops.ResizeClass(frame=frame, width=320, height=240) output = db.sinks.FrameColumn(columns={ 'frame1': resized_frame_fn, 'frame2': resized_frame_class }) job = Job(op_args={ frame: input_table.column('frame'), output_op: 'example_python_op' }) [table] = db.run(output=output, jobs=[job], force=True) table.column('frame1').save_mp4('01_resized_fn') table.column('frame2').save_mp4('01_resized_class') print('Finished! Two videos were saved to the current directory: ' '01_resized_fn.mp4, 01_resized_class.mp4') # If you are trying to integrate with a C++ library or you want a more efficient # implementation for your Ops, you can also define Ops in C++. See the # 08_defining_cpp_ops.py tutorial.
args=caffe_args, device=DeviceType.GPU) bboxes, feature = db.ops.FasterRCNNOutput(cls_prob=cls_prob, rois=rois, fc7=fc7, args=caffe_args, device=DeviceType.CPU) output = db.ops.Output(columns=[bboxes, feature]) job = Job(op_args={ frame: input_table.column('frame'), output: input_table.name() + '_detections' }) [output] = db.run(output=output, jobs=[job], pipeline_instances_per_node=1, work_packet_size=10, io_packet_size=40, force=True) output = db.table(input_table.name() + '_detections') output.profiler().write_trace('detect_test.trace') print('Extracting frames...') def parse_features(buf, db): if len(buf) == 1: return np.zeros((1), dtype=np.dtype(np.float32)) else: out = np.frombuffer(buf, dtype=np.dtype(np.int32))
def main(): db = Database() example_video_path = util.download_video() [input_table], _ = db.ingest_videos([('example', example_video_path)], force=True) videos = [] # Many ops simply involve applying some processing to their inputs and then # returning their outputs. But there are also many operations in video # processing that require the ability to see adjacent frames (such as for # computing optical flow), need to keep state over time (such as for tracking # objects), or need to process multiple elements for efficiency reasons (such as # batching for DNNs). # Scanner ops therefore have several optional attributes that enable them to # support these forms of operations: # 1. Device Type: # Ops can specify that they require CPUs or GPUs by declaring their device # type. By default, the device_type is DeviceType.CPU. @scannerpy.register_python_op(device_type=DeviceType.CPU) def device_resize(config, frame: FrameType) -> FrameType: return cv2.resize(frame, (config.args['width'], config.args['height'])) frame = db.sources.FrameColumn() resized_frame = db.ops.device_resize(frame=frame, width=640, height=480) output = db.sinks.FrameColumn(columns={'frame': resized_frame}) job = Job(op_args={ frame: input_table.column('frame'), output: 'example_resize' }) [table] = db.run(output=output, jobs=[job], force=True) table.column('frame').save_mp4('02_device_resize') videos.append('02_device_resize.mp4') # 2. Batch: # The Op can receive multiple elements at once to enable SIMD or # vector-style processing. @scannerpy.register_python_op(batch=10) def batch_resize(config, frame: Sequence[FrameType]) -> Sequence[FrameType]: output_frames = [] for fr in frame: output_frames.append( cv2.resize(fr, (config.args['width'], config.args['height']))) return output_frames # Here we specify that the resize op should receive a batch of 10 # input elements at once. Logically, each element is still processed # independently but multiple elements are provided to enable efficient # batch processing. If there are not enough elements left in a stream, # the Op may receive less than a batch worth of elements. frame = db.sources.FrameColumn() resized_frame = db.ops.batch_resize(frame=frame, width=640, height=480, batch=10) output = db.sinks.FrameColumn(columns={'frame': resized_frame}) job = Job(op_args={ frame: input_table.column('frame'), output: 'example_batch_resize' }) [table] = db.run(output=output, jobs=[job], force=True) table.column('frame').save_mp4('02_batch_resize') videos.append('02_batch_resize.mp4') # 3. Stencil: # The Op requires a window of input elements (for example, the # previous and next element) at the same time to produce an # output. # Here, we use the stencil attribute to write an optical flow op which # computes flow between the current and next frame. @scannerpy.register_python_op(stencil=[0, 1]) def optical_flow(config, frame: Sequence[FrameType]) -> FrameType: gray1 = cv2.cvtColor(frame[0], cv2.COLOR_BGR2GRAY) gray2 = cv2.cvtColor(frame[1], cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(gray1, gray2, None, 0.5, 3, 15, 3, 5, 1.2, 0) return flow # This op visualizes the flow field by converting it into an rgb image @scannerpy.register_python_op() def visualize_flow(config, flow: FrameType) -> FrameType: hsv = np.zeros(shape=(flow.shape[0], flow.shape[1], 3), dtype=np.uint8) hsv[..., 1] = 255 mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) hsv[..., 0] = ang * 180 / np.pi / 2 hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) return rgb frame = db.sources.FrameColumn() # This next line is using a feature we'll discuss in the next tutorial, but you # can think of it as selecting a subset of elements from the stream (here, # frames 0 to 30) range_frame = db.streams.Range(frame, 0, 30) flow = db.ops.optical_flow(frame=range_frame, stencil=[0, 1]) flow_viz = db.ops.visualize_flow(flow=flow) output = db.sinks.FrameColumn(columns={'flow_viz': flow_viz}) job = Job(op_args={ frame: input_table.column('frame'), output: 'example_flow' }) [table] = db.run(output=output, jobs=[job], force=True) table.column('flow_viz').save_mp4('02_flow') videos.append('02_flow.mp4') # 4. Bounded State: # For each output, the Op requires at least W sequential # "warmup" elements before it can produce a valid output. # For example, if the output of this Op is sampled # sparsely, this guarantees that the Op can "warmup" # its state on a stream of W elements before producing the # requested output. import subprocess @scannerpy.register_python_op(bounded_state=60) class BackgroundSubtraction(scannerpy.Kernel): def __init__(self, config): self.config = config self.alpha = config.args['alpha'] self.thresh = config.args['threshold'] # Reset is called when the kernel switches to a new part of the stream # and so shouldn't maintain it's previous state def reset(self): self.average_image = None def execute(self, frame: FrameType) -> FrameType: if self.average_image is None: self.average_image = frame mask = np.abs(frame - self.average_image) < 255 * self.thresh mask = np.any(mask, axis=2) masked_image = np.copy(frame) wmask = np.where(mask) masked_image[wmask[0], wmask[1], :] = 0 self.average_image = (self.average_image * (1.0 - self.alpha) + frame * self.alpha) return masked_image # Here we wrote an op that performs background subtraction by keeping a # running average image over the past frames. We set `bounded_state=60` # to indicate that this kernel needs at least 60 frames before the output # should be considered reasonable. # First, we download a static camera video from youtube # subprocess.check_call( # 'youtube-dl -f 137 \'https://youtu.be/cVHqFqNz7eM\' -o test.mp4', # shell=True) # [static_table], _ = db.ingest_videos([('static_video', 'test.mp4')], # force=True) static_table = input_table # Then we perform background subtraction and indicate we need 60 prior # frames to produce correct output frame = db.sources.FrameColumn() masked_frame = db.ops.BackgroundSubtraction(frame=frame, alpha=0.05, threshold=0.05, bounded_state=60) # Here, we say that we only want the outputs for this range of frames sampled_frame = db.streams.Range(masked_frame, 0, 120) output = db.sinks.Column(columns={'frame': sampled_frame}) job = Job(op_args={ frame: static_table.column('frame'), output: 'masked_video', }) [table] = db.run(output=output, jobs=[job], force=True) table.column('frame').save_mp4('02_masked') videos.append('02_masked.mp4') # 5. Unbounded State: # This Op will always process all preceding elements of # its input streams before producing a requested output. # This means that sampling operations after this Op # can not change how many inputs it receives. In the next # tutorial, we will show how this can be relaxed for # sub-streams of the input. @scannerpy.register_python_op(unbounded_state=True) class Example(scannerpy.Kernel): def __init__(self, config): pass def reset(self): pass def execute(self, frame: FrameType) -> bytes: pass print('Finished! The following videos were written: {:s}'.format( ', '.join(videos)))
encoded_edges2 = db.ops.Pass(input=encoded_edges) output_op = db.sinks.Column( columns={ 'image': encoded_image2, 'poseimg': encoded_poseimg2, 'edges': encoded_edges2 }) job = Job( op_args={ encoded_image: { 'paths': image_files, **params }, encoded_poseimg: { 'paths': poseimg_files, **params }, encoded_edges: { 'paths': edge_files, **params }, output_op: opt.table_name, }) [_out_table] = db.run(output_op, [job], force=True, tasks_in_queue_per_pu=1) print(db.summarize())
jq '.spec.ports[0].nodePort' -r ''', shell=True).strip().decode('utf-8') master = '{}:{}'.format(ip, port) print('Connecting to Scanner database...') db = Database(master=master, start_cluster=False, config_path='./config.toml') print('Running Scanner job...') example_video_path = 'sample.mp4' [input_table], _ = db.ingest_videos([('example', example_video_path)], force=True, inplace=True) print(db.summarize()) frame = db.sources.FrameColumn() r_frame = db.ops.Resize(frame=frame, width=320, height=240) output_op = db.sinks.Column(columns={'frame': r_frame}) job = Job(op_args={ frame: db.table('example').column('frame'), output_op: 'example_frame' }) output_tables = db.run(output=output_op, jobs=[job], force=True) output_tables[0].column('frame').save_mp4('resized_video') print('Complete!')
job = Job( op_args={ encoded_image: { 'paths': image_files, **params }, encoded_mask: { 'paths': mask_files, **params }, output_op: 'example_resized5', }) start = time.time() [out_table] = db.run(output_op, [job], force=True, work_packet_size=opt.work_packet_size, io_packet_size=opt.io_packet_size) end = time.time() print('Total time for depth estimation in scanner: {0:.3f} sec'.format( end - start)) # results = out_table.column('frame').load() # # path_to_save = join(dataset, 'players', 'prediction_scanner') # if not os.path.exists(path_to_save): # os.mkdir(path_to_save) # # for i, res in enumerate(results): # pred_scanner = np.argmax(res, axis=0) # np.save(join(path_to_save, '{0:05d}.npy'.format(i)), res)
def main(): # Initialize a connection to the Scanner database. Loads configuration from the # ~/.scanner.toml configuration file. db = Database() # Create a Scanner table from our video in the format (table name, # video path). If any videos fail to ingest, they'll show up in the failed # list. If force is true, it will overwrite existing tables of the same # name. example_video_path = util.download_video() _, failed = db.ingest_videos([('example', example_video_path), ('example2', example_video_path), ('thisshouldfail', 'thisshouldfail.mp4')], force=True) print(db.summarize()) print('Failures:', failed) # Scanner processes videos by forming a graph of operations that operate # on input frames from a table and produce outputs to a new table. # FrameColumn declares that we want to read from a table column that # represents a video frame. frame = db.sources.FrameColumn() # These frames are input into a Histogram op that computes a color histogram # for each frame. hist = db.ops.Histogram(frame=frame) # Finally, any columns provided to Output will be saved to the output # table at the end of the computation. Here, 'hist' is the name of the # column for the output table. output_op = db.sinks.Column(columns={'hist': hist}) # A job defines a table you want to create. In op_args, we bind the # FrameColumn from above to the table we want to read from and name # the output table 'example_hist' by binding a string to output_op. job = Job(op_args={ frame: db.table('example').column('frame'), output_op: 'example_hist' }) job2 = Job(op_args={ frame: db.table('example2').column('frame'), output_op: 'example_hist2' }) # This executes the job and produces the output table. You'll see a progress # bar while Scanner is computing the outputs. output_tables = db.run(output=output_op, jobs=[job, job2], force=True) # Load the histograms from a column of the output table. The # readers.histograms function converts the raw bytes output by Scanner # into a numpy array for each channel. video_hists = output_tables[0].column('hist').load(readers.histograms) # Loop over the column's values, a set of 3 histograms (for each color channel) per element. num_rows = 0 for frame_hists in video_hists: assert len(frame_hists) == 3 assert frame_hists[0].shape[0] == 16 num_rows += 1 assert num_rows == db.table('example').num_rows()
keypoints = db.sources.Column() descriptors = db.sources.Column() camera = db.sources.Column() pair_image_ids, two_view_geometries = db.ops.SequentialMatchingCPU( image_ids=image_ids, keypoints=keypoints, descriptors=descriptors, stencil=matching_stencil) output = db.sinks.Column(columns={ 'pair_image_ids': pair_image_ids, 'two_view_geometries': two_view_geometries }) job = Job( op_args={ image_ids: db.table(args.input_table).column('image_id'), keypoints: db.table(args.input_table).column('keypoints'), descriptors: db.table(args.input_table).column('descriptors'), camera: db.table(args.input_table).column('camera'), output: args.output_table }) output_tables = db.run( output, [job], force=True, io_packet_size=args.packet_size, work_packet_size=args.packet_size) print(db.summarize())
encoded_image = db.sources.Files(**params) output_op = db.sinks.FrameColumn(columns={'image': encoded_image}) job = Job( op_args={ encoded_image: { 'paths': image_files, **params }, output_op: 'edge_detect', }) [_out_table] = db.run(output_op, [job], force=True, tasks_in_queue_per_pu=1) print(db.summarize()) encoded_image = db.sources.FrameColumn() frame = db.ops.ImageDecoder(img=encoded_image) my_edge_detection_class = db.ops.EdgeDetection(frame=frame, model_path='model.yml.gz') output_op = db.sinks.FrameColumn( columns={'frame': my_edge_detection_class}) job = Job( op_args={ encoded_image: db.table('edge_detect').column('image'), output_op: 'edge_output',
def main(movie_path): total_start = time.time() print('Detecting shots in movie {}'.format(movie_path)) movie_name = os.path.basename(movie_path) # Use GPU kernels if we have a GPU db = Database() print('Loading movie into Scanner database...') s = time.time() if db.has_gpu(): device = DeviceType.GPU else: device = DeviceType.CPU ############ ############ ############ ############ # 0. Ingest the video into the database ############ ############ ############ ############ [movie_table], _ = db.ingest_videos([(movie_name, movie_path)], force=True) print('Time: {:.1f}s'.format(time.time() - s)) print('Number of frames in movie: {:d}'.format(movie_table.num_rows())) s = time.time() ############ ############ ############ ############ # 1. Run Histogram over the entire video in Scanner ############ ############ ############ ############ print('Computing a color histogram for each frame...') frame = db.sources.FrameColumn() histogram = db.ops.Histogram(frame=frame, device=device) output = db.sinks.Column(columns={'histogram': histogram}) job = Job(op_args={ frame: movie_table.column('frame'), output: movie_name + '_hist' }) [hists_table] = db.run(output=output, jobs=[job], force=True) print('\nTime: {:.1f}s, {:.1f} fps'.format( time.time() - s, movie_table.num_rows() / (time.time() - s))) s = time.time() ############ ############ ############ ############ # 2. Load histograms and compute shot boundaries # in python ############ ############ ############ ############ print('Computing shot boundaries...') # Read histograms from disk hists = [ h for h in hists_table.column('histogram').load(parsers.histograms) ] boundaries = compute_shot_boundaries(hists) print('Found {:d} shots.'.format(len(boundaries))) print('Time: {:.1f}s'.format(time.time() - s)) s = time.time() ############ ############ ############ ############ # 3. Create montage in Scanner ############ ############ ############ ############ print('Creating shot montage...') row_length = min(16, len(boundaries)) rows_per_item = 1 target_width = 256 # Compute partial row montages that we will stack together # at the end frame = db.sources.FrameColumn() gather_frame = frame.sample() sliced_frame = gather_frame.slice() montage = db.ops.Montage(frame=sliced_frame, num_frames=row_length * rows_per_item, target_width=target_width, frames_per_row=row_length, device=device) sampled_montage = montage.sample() output = db.sinks.Column( columns={'montage': sampled_montage.unslice().lossless()}) item_size = row_length * rows_per_item starts_remainder = len(boundaries) % item_size evenly_divisible = (starts_remainder == 0) if not evenly_divisible: boundaries = boundaries[0:len(boundaries) - starts_remainder] job = Job( op_args={ frame: movie_table.column('frame'), gather_frame: db.sampler.gather(boundaries), sliced_frame: db.partitioner.all(item_size), sampled_montage: [ db.sampler.gather([item_size - 1]) for _ in range(len(boundaries) / item_size) ], output: 'montage_image' }) [montage_table] = db.run(output=output, jobs=[job], force=True) # Stack all partial montages together montage_img = np.zeros((1, target_width * row_length, 3), dtype=np.uint8) for img in montage_table.column('montage').load(): img = np.flip(img, 2) montage_img = np.vstack((montage_img, img)) print('') print('Time: {:.1f}s'.format(time.time() - s)) ############ ############ ############ ############ # 4. Write montage to disk ############ ############ ############ ############ cv2.imwrite('shots.jpg', montage_img) print('Successfully generated shots.jpg') print('Total time: {:.2f} s'.format(time.time() - total_start))
frame = db.sources.FrameColumn() blurred_frame = db.ops.Blur(frame=frame, kernel_size=3, sigma=0.5) sampled_frame = db.streams.Range(blurred_frame, 0, 30) return frame, sampled_frame # By default, if an Op outputs a frame with 3 channels with type uint8, # those frames will be compressed using video encoding. No other frame # type is currently compressed. frame, blurred_frame = make_blurred_frame() output_op = db.sinks.Column(columns={'frame': blurred_frame}) job = Job(op_args={ frame: db.table('example').column('frame'), output_op: 'output_table_name', }) db.run(output_op, [job], force=True) frame, blurred_frame = make_blurred_frame() # The compression parameters can be controlled by annotating the column low_quality_frame = blurred_frame.compress_video(quality=35) output_op = db.sinks.Column(columns={'frame': low_quality_frame}) job = Job(op_args={ frame: db.table('example').column('frame'), output_op: 'low_quality_table', }) db.run(output_op, [job], force=True) # If no compression is desired, this can be specified by indicating that # the column should be lossless. frame, blurred_frame = make_blurred_frame() # The compression parameters can be controlled by annotating the column
**params }, calibrate_video_class: { 'w': 1920, 'h': 1080, 'A': calibs[i]['A'], 'R': calibs[i]['R'], 'T': calibs[i]['T'] }, output_op: 'example_resized_{0}'.format(i), }) jobs.append(job) start = time.time() tables = db.run(output_op, jobs, force=True) end = time.time() for i in range(len(video_names)): savename = join(path_to_data, video_names[i], 'calib_scanner2') tables[i].column('frame').save_mp4(savename) print('Successfully generated {0}_faces.mp4 in {1:.3f} secs'.format( savename, end - start)) # # ====================================================================================================================== # # Images # # ====================================================================================================================== # # total_files = -1 # image_files = glob.glob(join(dataset_list[0], 'images', '*.jpg')) # image_files.sort()
h=128, sigma1=1.0, sigma2=0.01, model_path=model_path) output_op = db.sinks.FrameColumn(columns={'frame': my_segment_imageset_class}) job = Job( op_args={ encoded_image: { 'paths': image_files, **params }, encoded_mask: { 'paths': mask_files, **params }, output_op: 'example_resized', }) start = time.time() [out_table] = db.run(output_op, [job], force=True, work_packet_size=8, io_packet_size=32, tasks_in_queue_per_pu=4) end = time.time() print('Total time for instance segmentation in scanner: {0:.3f} sec'.format( end - start)) # out_table.column('frame').save_mp4(join(dataset, 'players', 'instance_segm.mp4'))
cluster_ids, cameras, images, points3d = db.ops.IncrementalMappingCPU( image_id=image_ids, pair_image_ids=pair_image_ids, two_view_geometries=two_view_geometries, keypoints=keypoints, camera=cameras, batch=batch_size, stencil=matching_stencil) ################################################################################ # remove empty rows from sparse reconstruction ################################################################################ def remove_empty_rows(*input_cols): return [db.streams.Stride(col, batch_size) for col in input_cols] cluster_id, cameras, images, points3d = remove_empty_rows( cluster_ids, cameras, images, points3d) ################################################################################ # merge submodels ################################################################################ cluster_id, cameras, images, points3d = db.ops.MergeMappingCPU( cluster_id=cluster_id, cameras=cameras, images=images, points3d=points3d, batch=num_submodels) output = db.sinks.Column( columns={'cluster_id': cluster_id, 'cameras': cameras, 'images': images, 'points3d': points3d}) job = Job(op_args={ files: {'paths': image_paths}, output: args.output_table}) output_tables = db.run(output, [job], force=True) print(db.summarize())
if len(sys.argv) <= 1: print('Usage: main.py <video_file>') exit(1) video_path = sys.argv[1] print('Performing optical flow on {}...'.format(video_path)) video_name = os.path.splitext(os.path.basename(video_path))[0] db = Database() if not db.has_table(video_name): db.ingest_videos([(video_name, video_path)]) input_table = db.table(video_name) frame = db.sources.FrameColumn() flow = db.ops.OpticalFlow( frame = frame, device=DeviceType.CPU) sampled_flow = db.streams.Range(flow, 0, 60) output = db.sinks.Column(columns={'flow': sampled_flow}) job = Job(op_args={ frame: input_table.column('frame'), output: input_table.name() + '_flow' }) [output_table] = db.run(output=output, jobs=[job], pipeline_instances_per_node=1, force=True) vid_flows = [flow[0] for flow in output_table.column('flow').load(rows=[0])] np.save('flows.npy', vid_flows)
scales = 3 else: print('Using CPUs') device = DeviceType.CPU pipeline_instances = 1 scales = 1 frame = db.sources.FrameColumn() poses_out = db.ops.OpenPose(frame=frame, pose_num_scales=scales, pose_scale_gap=0.33, device=device) drawn_frame = db.ops.PoseDraw(frame=frame, frame_poses=poses_out) sampled_frames = sampler(drawn_frame) output = db.sinks.Column(columns={'frame': sampled_frames}) job = Job( op_args={ frame: input_table.column('frame'), sampled_frames: sampler_args, output: movie_name + '_drawn_poses', }) [drawn_poses_table] = db.run(output=output, jobs=[job], work_packet_size=8, io_packet_size=64, pipeline_instances_per_node=pipeline_instances, force=True) print('Writing output video...') drawn_poses_table.column('frame').save_mp4('{:s}_poses'.format(movie_name))