def main():
# Look at resize_op/resize_op.cpp to start this tutorial.
db = Database()
cwd = os.path.dirname(os.path.abspath(__file__))
if not os.path.isfile(os.path.join(cwd,
'resize_op/build/libresize_op.so')):
print(
'You need to build the custom op first: \n'
'$ pushd {}/resize_op; mkdir build && cd build; cmake ..; make; popd'
.format(cwd))
exit()
# To load a custom op into the Scanner runtime, we use db.load_op to open the
# shared library we compiled. If the op takes arguments, it also optionally
# takes a path to the generated python file for the arg protobuf.
db.load_op(os.path.join(cwd, 'resize_op/build/libresize_op.so'),
os.path.join(cwd, 'resize_op/build/resize_pb2.py'))
frame = db.sources.FrameColumn()
# Then we use our op just like in the other examples.
resize = db.ops.MyResize(frame=frame, width=200, height=300)
output_op = db.sinks.Column(columns={'resized_frame': resize})
job = Job(op_args={
frame: db.table('example').column('frame'),
output_op: 'example_resized',
})
db.run(output_op, [job], force=True)
def main():
db = Database()
# What if, instead of a video, you had a list of image files that you
# wanted to process? Scanner provides an extensible interface for reading and
# writing data to locations other than the database.
# For example, let's download a few images now and create a list of their paths:
util.download_images()
image_paths = [
'sample-frame-1.jpg', 'sample-frame-2.jpg', 'sample-frame-3.jpg'
]
# Scanner provides a built-in source to read files from the local filesystem:
compressed_images = db.sources.Files()
# Like with db.sources.FrameColumn, we will bind the inputs to this source when
# we define a job later on.
# Let's write a pipeline that reads our images, resizes them, and writes them
# back out as files to the filesystem.
# Since the input images are compressed, we decompress them with the
# ImageDecoder
frame = db.ops.ImageDecoder(img=compressed_images)
resized = db.ops.Resize(frame=frame, width=640, height=360)
# Rencode the image to jpg
encoded_frame = db.ops.ImageEncoder(frame=resized, format='jpg')
# Write the compressed images to files
output = db.sinks.Files(input=encoded_frame)
resized_paths = ['resized-1.jpg', 'resized-2.jpg', 'resized-3.jpg']
job = Job(
op_args={
compressed_images: {
'paths': image_paths
},
output: {
'paths': resized_paths
}
})
db.run(output=output, jobs=[job])
print('Finished! Wrote the following images: ' + ', '.join(resized_paths))
def main():
db = Database()
# Frames on disk can either be stored uncompressed (raw bits) or compressed
# (encoded using some form of image or video compression). When Scanner
# reads frames from a table, it automatically decodes the data if necessary.
# The Op DAG only sees the raw frames. For example, this table is stored
# as compressed video.
def make_blurred_frame():
frame = db.sources.FrameColumn()
blurred_frame = db.ops.Blur(frame=frame, kernel_size=3, sigma=0.5)
sampled_frame = db.streams.Range(blurred_frame, 0, 30)
return frame, sampled_frame
# By default, if an Op outputs a frame with 3 channels with type uint8,
# those frames will be compressed using video encoding. No other frame
# type is currently compressed.
frame, blurred_frame = make_blurred_frame()
output_op = db.sinks.Column(columns={'frame': blurred_frame})
job = Job(
op_args={
frame: db.table('example').column('frame'),
output_op: 'output_table_name',
})
db.run(output_op, [job], force=True)
frame, blurred_frame = make_blurred_frame()
# The compression parameters can be controlled by annotating the column
low_quality_frame = blurred_frame.compress_video(quality=35)
output_op = db.sinks.Column(columns={'frame': low_quality_frame})
job = Job(
op_args={
frame: db.table('example').column('frame'),
output_op: 'low_quality_table',
})
db.run(output_op, [job], force=True)
# If no compression is desired, this can be specified by indicating that
# the column should be lossless.
frame, blurred_frame = make_blurred_frame()
# The compression parameters can be controlled by annotating the column
lossless_frame = blurred_frame.lossless()
output_op = db.sinks.Column(columns={'frame': lossless_frame})
job = Job(op_args={
frame: db.table('example').column('frame'),
output_op: 'pristine_frame',
})
db.run(output_op, [job], force=True)
# Any column which is saved as compressed video can be exported as an mp4
# file by calling save_mp4 on the column. This will output a file called
# 'low_quality_video.mp4' in the current directory.
db.table('low_quality_table').column('frame').save_mp4('low_quality_video')
def main():
db = Database()
frame = db.sources.FrameColumn()
histogram = db.ops.Histogram(frame=frame)
output_op = db.sinks.Column(columns={'hist': histogram})
job = Job(
op_args={
frame: db.table('example').column('frame'),
output_op: 'example_hist_profile'
})
[output_table] = db.run(output_op, [job], force=True)
# The profiler contains information about how long different parts of your
# computation take to run. We use Google Chrome's trace format, which you
# can view by going to chrome://tracing in Chrome and clicking "load" in
# the top left.
output_table.profiler().write_trace('hist.trace')
def main():
db = Database()
frame = db.sources.FrameColumn()
# You can tell Scanner which frames of the video (or which rows of a video
# table) you want to sample. Here, we indicate that we want to stride
# the frame column by 4 (select every 4th frame)
strided_frame = db.streams.Stride(frame, 4)
# We process the sampled frame same as before.
hist = db.ops.Histogram(frame=strided_frame)
output_op = db.sinks.Column(columns={'hist': hist})
# For each job, you can specify how sampling should be performed for
# a specific stream. In the same way we used the op_args argument to bind
# a table to an input column, we can bind sampling arguments to strided_frame
# to override the default striding of 4 we specified above
job = Job(
op_args={
frame: db.table('example').column('frame'),
# The "strided" sampling mode will run over every 8th frame,
# i.e. frames [0, 8, 16, ...]
strided_frame: 8,
output_op: 'example_hist_strided'
})
output_tables = db.run(output_op, [job], force=True)
# Loop over the column's rows. Each row is a tuple of the frame number and
# value for that row.
video_hists = output_tables[0].column('hist').load(readers.histograms)
num_rows = 0
for frame_hists in video_hists:
assert len(frame_hists) == 3
assert frame_hists[0].shape[0] == 16
num_rows += 1
assert num_rows == round(db.table('example').num_rows() / 8)
# Here's some examples of other sampling modes.
# Range takes a specific subset of a video. Here, it runs over all frames
# from 0 to 100
db.streams.Range(frame, 0, 100)
# Gather takes an arbitrary list of frames from a video.
db.streams.Gather(frame, [10, 17, 32])
def main():
# Now we can use these new Ops in Scanner:
db = Database()
# Download an example video
example_video_path = util.download_video()
# Ingest it into the database
[input_table], _ = db.ingest_videos([('example', example_video_path)],
force=True)
frame = db.sources.FrameColumn()
resized_frame_fn = db.ops.resize_fn(frame=frame, width=640, height=480)
resized_frame_class = db.ops.ResizeClass(frame=frame,
width=320,
height=240)
output = db.sinks.FrameColumn(columns={
'frame1': resized_frame_fn,
'frame2': resized_frame_class
})
job = Job(op_args={
frame: input_table.column('frame'),
output: 'example_python_op'
})
[table] = db.run(output=output, jobs=[job], force=True)
table.column('frame1').save_mp4('01_resized_fn')
table.column('frame2').save_mp4('01_resized_class')
print('Finished! Two videos were saved to the current directory: '
'01_resized_fn.mp4, 01_resized_class.mp4')
end=args.num_reg_images)
R, T, K, width, height, scan_width, bitmap, depth_min, depth_max, id = db.ops.PreparePatchMatch(
sparse_reconstruction_path=args.input_path,
batch=args.num_reg_images,
image_id=image_id_sampled)
output = db.sinks.Column(
columns={
'R': R,
'T': T,
'K': K,
'width': width,
'height': height,
'scan_width': scan_width,
'bitmap': bitmap,
'depth_min': depth_min,
'depth_max': depth_max,
'image_id': id,
})
job = Job(op_args={
image_id: db.table('frames').column('image_id'),
output: args.output_table,
})
output_tables = db.run(output, [job],
force=True,
work_packet_size=args.num_reg_images,
io_packet_size=args.num_reg_images)
print(db.summarize())
# we define a job later on.
# Let's write a pipeline that reads our images, resizes them, and writes them
# back out as files to the filesystem.
# Since the input images are compressed, we decompress them with the
# ImageDecoder
frame = db.ops.ImageDecoder(img=compressed_images)
resized = db.ops.Resize(frame=frame, width=640, height=360)
# Rencode the image to jpg
encoded_frame = db.ops.ImageEncoder(frame=resized, format='jpg')
# Write the compressed images to files
output = db.sinks.Files(input=encoded_frame)
resized_paths = ['resized-1.jpg', 'resized-2.jpg', 'resized-3.jpg']
job = Job(op_args={
compressed_images: {'paths': image_paths},
output: {'paths': resized_paths}
})
db.run(output=output, jobs=[job])
print('Finished! Wrote the following images: ' + ', '.join(resized_paths))
# If you want to learn how write your own custom Source or Sink in C++, check
# out tutorials 09_defining_cpp_sources.py and 10_defining_cpp_sinks.py
encoded_image: {
'paths': image_files,
**params
},
encoded_mask: {
'paths': mask_files,
**params
},
output_op: 'example_resized55'
})
start = time.time()
[out_table
] = db.run(output_op, [job],
force=True,
work_packet_size=opt.work_packet_size,
io_packet_size=opt.io_packet_size,
pipeline_instances_per_node=opt.pipeline_instances_per_node)
end = time.time()
print('scanner distance transform: {0:.4f} for {1} frames'.format(
end - start, len(image_files)))
if opt.save:
# out_table.profiler().write_trace(join(dataset, 'calib.trace'))
# print('Trace saved in {0}'.format(join(dataset, 'calib.trace')))
results = out_table.column('frame').load()
dist_transf_list = [res[:, :, 0] for res in results]
A, R, T = cam_data['A'], cam_data['R'], cam_data['T']
h, w = 1080, 1920
job = Job(
op_args={
encoded_image: {
'paths': image_files,
**params
},
encoded_mask: {
'paths': mask_files,
**params
},
output_op: 'example_resized5',
})
start = time.time()
[out_table] = db.run(output_op, [job],
force=True,
work_packet_size=16,
io_packet_size=16)
end = time.time()
print('Total time for depth estimation in scanner: {0:.3f} sec'.format(
end - start))
results = out_table.column('frame').load()
# path_to_save = join(dataset, 'players', 'prediction_scanner')
# if not os.path.exists(path_to_save):
# os.mkdir(path_to_save)
#
# for i, res in enumerate(results):
# pred_scanner = np.argmax(res, axis=0)
# np.save(join(path_to_save, '{0:05d}.npy'.format(i)), res)
table_input = db.ops.Input()
caffe_input = db.ops.FacenetInput(inputs=[(table_input,
["frame", "frame_info"])],
args=facenet_args,
device=DeviceType.GPU)
caffe = db.ops.Facenet(inputs=[(caffe_input, ["caffe_frame"]),
(table_input, ["frame_info"])],
args=facenet_args,
device=DeviceType.GPU)
caffe_output = db.ops.FacenetOutput(inputs=[(caffe, ["caffe_output"]),
(table_input, ["frame_info"])],
args=facenet_args)
input_collection = db.ingest_video_collection('test', ['test.mp4'])
output_collection = db.run(input_collection, caffe_output, 'test_faces')
def parse_bboxes(db, buf):
(num_bboxes, ) = struct.unpack("=Q", buf[:8])
buf = buf[8:]
bboxes = []
for i in range(num_bboxes):
(bbox_size, ) = struct.unpack("=i", buf[:4])
buf = buf[4:]
box = db.protobufs.BoundingBox()
box.ParseFromString(buf[:bbox_size])
buf = buf[bbox_size:]
bbox = [
box.x1, box.y1, box.x2, box.y2, box.score, box.track_id,
box.track_score
from scannerpy import Database, Job, DeviceType
################################################################################
# This tutorial shows how to look at profiling information for your job. #
################################################################################
db = Database()
frame = db.sources.FrameColumn()
histogram = db.ops.Histogram(frame=frame)
output_op = db.sinks.Column(columns={'hist': histogram})
job = Job(op_args={
frame: db.table('example').column('frame'),
output_op: 'example_hist_profile'
})
[output_table] = db.run(output_op, [job], force=True)
# The profiler contains information about how long different parts of your
# computation take to run. We use Google Chrome's trace format, which you
# can view by going to chrome://tracing in Chrome and clicking "load" in
# the top left.
output_table.profiler().write_trace('hist.trace')
# Each row corresponds to a different part of the system, e.g. the thread
# loading bytes from disk or the thread running your kernels. If you have
# multiple pipelines or multiple nodes, you will see many of these evaluate
# threads.
db = Database()
video_path = movie_path
if not db.has_table(movie_name):
print('Ingesting video into Scanner ...')
db.ingest_videos([(movie_name, video_path)], force=True)
input_table = db.table(movie_name)
sampler = db.streams.Range
sampler_args = {'start': 120, 'end': 480}
[poses_table] = pipelines.detect_poses(db, [input_table.column('frame')],
sampler, sampler_args,
'{:s}_poses'.format(movie_name))
print('Drawing on frames...')
frame = db.sources.FrameColumn()
sampled_frame = sampler(frame)
poses = db.sources.Column()
drawn_frame = db.ops.PoseDraw(frame=sampled_frame, poses=poses)
output = db.sinks.Column(columns={'frame': drawn_frame})
job = Job(
op_args={
frame: input_table.column('frame'),
sampled_frame: sampler_args,
poses: poses_table.column('poses'),
output: movie_name + '_drawn_poses',
})
[drawn_poses_table] = db.run(output=output, jobs=[job], force=True)
print('Writing output video...')
drawn_poses_table.column('frame').save_mp4('{:s}_poses'.format(movie_name))
# Ingest the videos into the database
db.ingest_videos(videos_to_process)
# Define the same Computation Graph
frame = db.sources.FrameColumn() # Read from the database
sampled_frame = db.streams.Stride(frame, 3) # Select every third frame
resized = db.ops.Resize(frame=sampled_frame, width=640,
height=480) # Resize input frame
output_frame = db.sinks.Column(columns={'frame':
resized}) # Save resized frame
# Create multiple jobs now, one for each video we want to process
jobs = []
for table_name, _ in videos_to_process:
job = Job(
op_args={
frame: db.table(table_name).column(
'frame'), # Column to read input frames from
output_frame: '{:s}-resized'.format(
table_name) # Name of output table
})
jobs.append(job)
# Execute the computation graph and return a handle to the newly produced tables
output_tables = db.run(output=output_frame, jobs=jobs, force=True)
# Save the resized video as an mp4 file
for output_table in output_tables:
output_table.column('frame').save_mp4(output_table.name())
[input_table], _ = db.ingest_videos([('example', example_video_path)],
force=True)
frame = db.sources.FrameColumn()
resized_frame_fn = db.ops.resize_fn(frame=frame, width=640, height=480)
resized_frame_class = db.ops.ResizeClass(frame=frame, width=320, height=240)
output = db.sinks.FrameColumn(columns={
'frame1': resized_frame_fn,
'frame2': resized_frame_class
})
job = Job(op_args={
frame: input_table.column('frame'),
output_op: 'example_python_op'
})
[table] = db.run(output=output, jobs=[job], force=True)
table.column('frame1').save_mp4('01_resized_fn')
table.column('frame2').save_mp4('01_resized_class')
print('Finished! Two videos were saved to the current directory: '
'01_resized_fn.mp4, 01_resized_class.mp4')
# If you are trying to integrate with a C++ library or you want a more efficient
# implementation for your Ops, you can also define Ops in C++. See the
# 08_defining_cpp_ops.py tutorial.
args=caffe_args,
device=DeviceType.GPU)
bboxes, feature = db.ops.FasterRCNNOutput(cls_prob=cls_prob,
rois=rois,
fc7=fc7,
args=caffe_args,
device=DeviceType.CPU)
output = db.ops.Output(columns=[bboxes, feature])
job = Job(op_args={
frame: input_table.column('frame'),
output: input_table.name() + '_detections'
})
[output] = db.run(output=output,
jobs=[job],
pipeline_instances_per_node=1,
work_packet_size=10,
io_packet_size=40,
force=True)
output = db.table(input_table.name() + '_detections')
output.profiler().write_trace('detect_test.trace')
print('Extracting frames...')
def parse_features(buf, db):
if len(buf) == 1:
return np.zeros((1), dtype=np.dtype(np.float32))
else:
out = np.frombuffer(buf, dtype=np.dtype(np.int32))
def main():
db = Database()
example_video_path = util.download_video()
[input_table], _ = db.ingest_videos([('example', example_video_path)],
force=True)
videos = []
# Many ops simply involve applying some processing to their inputs and then
# returning their outputs. But there are also many operations in video
# processing that require the ability to see adjacent frames (such as for
# computing optical flow), need to keep state over time (such as for tracking
# objects), or need to process multiple elements for efficiency reasons (such as
# batching for DNNs).
# Scanner ops therefore have several optional attributes that enable them to
# support these forms of operations:
# 1. Device Type:
# Ops can specify that they require CPUs or GPUs by declaring their device
# type. By default, the device_type is DeviceType.CPU.
@scannerpy.register_python_op(device_type=DeviceType.CPU)
def device_resize(config, frame: FrameType) -> FrameType:
return cv2.resize(frame, (config.args['width'], config.args['height']))
frame = db.sources.FrameColumn()
resized_frame = db.ops.device_resize(frame=frame, width=640, height=480)
output = db.sinks.FrameColumn(columns={'frame': resized_frame})
job = Job(op_args={
frame: input_table.column('frame'),
output: 'example_resize'
})
[table] = db.run(output=output, jobs=[job], force=True)
table.column('frame').save_mp4('02_device_resize')
videos.append('02_device_resize.mp4')
# 2. Batch:
# The Op can receive multiple elements at once to enable SIMD or
# vector-style processing.
@scannerpy.register_python_op(batch=10)
def batch_resize(config,
frame: Sequence[FrameType]) -> Sequence[FrameType]:
output_frames = []
for fr in frame:
output_frames.append(
cv2.resize(fr, (config.args['width'], config.args['height'])))
return output_frames
# Here we specify that the resize op should receive a batch of 10
# input elements at once. Logically, each element is still processed
# independently but multiple elements are provided to enable efficient
# batch processing. If there are not enough elements left in a stream,
# the Op may receive less than a batch worth of elements.
frame = db.sources.FrameColumn()
resized_frame = db.ops.batch_resize(frame=frame,
width=640,
height=480,
batch=10)
output = db.sinks.FrameColumn(columns={'frame': resized_frame})
job = Job(op_args={
frame: input_table.column('frame'),
output: 'example_batch_resize'
})
[table] = db.run(output=output, jobs=[job], force=True)
table.column('frame').save_mp4('02_batch_resize')
videos.append('02_batch_resize.mp4')
# 3. Stencil:
# The Op requires a window of input elements (for example, the
# previous and next element) at the same time to produce an
# output.
# Here, we use the stencil attribute to write an optical flow op which
# computes flow between the current and next frame.
@scannerpy.register_python_op(stencil=[0, 1])
def optical_flow(config, frame: Sequence[FrameType]) -> FrameType:
gray1 = cv2.cvtColor(frame[0], cv2.COLOR_BGR2GRAY)
gray2 = cv2.cvtColor(frame[1], cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(gray1, gray2, None, 0.5, 3, 15, 3,
5, 1.2, 0)
return flow
# This op visualizes the flow field by converting it into an rgb image
@scannerpy.register_python_op()
def visualize_flow(config, flow: FrameType) -> FrameType:
hsv = np.zeros(shape=(flow.shape[0], flow.shape[1], 3), dtype=np.uint8)
hsv[..., 1] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
return rgb
frame = db.sources.FrameColumn()
# This next line is using a feature we'll discuss in the next tutorial, but you
# can think of it as selecting a subset of elements from the stream (here,
# frames 0 to 30)
range_frame = db.streams.Range(frame, 0, 30)
flow = db.ops.optical_flow(frame=range_frame, stencil=[0, 1])
flow_viz = db.ops.visualize_flow(flow=flow)
output = db.sinks.FrameColumn(columns={'flow_viz': flow_viz})
job = Job(op_args={
frame: input_table.column('frame'),
output: 'example_flow'
})
[table] = db.run(output=output, jobs=[job], force=True)
table.column('flow_viz').save_mp4('02_flow')
videos.append('02_flow.mp4')
# 4. Bounded State:
# For each output, the Op requires at least W sequential
# "warmup" elements before it can produce a valid output.
# For example, if the output of this Op is sampled
# sparsely, this guarantees that the Op can "warmup"
# its state on a stream of W elements before producing the
# requested output.
import subprocess
@scannerpy.register_python_op(bounded_state=60)
class BackgroundSubtraction(scannerpy.Kernel):
def __init__(self, config):
self.config = config
self.alpha = config.args['alpha']
self.thresh = config.args['threshold']
# Reset is called when the kernel switches to a new part of the stream
# and so shouldn't maintain it's previous state
def reset(self):
self.average_image = None
def execute(self, frame: FrameType) -> FrameType:
if self.average_image is None:
self.average_image = frame
mask = np.abs(frame - self.average_image) < 255 * self.thresh
mask = np.any(mask, axis=2)
masked_image = np.copy(frame)
wmask = np.where(mask)
masked_image[wmask[0], wmask[1], :] = 0
self.average_image = (self.average_image * (1.0 - self.alpha) +
frame * self.alpha)
return masked_image
# Here we wrote an op that performs background subtraction by keeping a
# running average image over the past frames. We set `bounded_state=60`
# to indicate that this kernel needs at least 60 frames before the output
# should be considered reasonable.
# First, we download a static camera video from youtube
# subprocess.check_call(
# 'youtube-dl -f 137 \'https://youtu.be/cVHqFqNz7eM\' -o test.mp4',
# shell=True)
# [static_table], _ = db.ingest_videos([('static_video', 'test.mp4')],
# force=True)
static_table = input_table
# Then we perform background subtraction and indicate we need 60 prior
# frames to produce correct output
frame = db.sources.FrameColumn()
masked_frame = db.ops.BackgroundSubtraction(frame=frame,
alpha=0.05,
threshold=0.05,
bounded_state=60)
# Here, we say that we only want the outputs for this range of frames
sampled_frame = db.streams.Range(masked_frame, 0, 120)
output = db.sinks.Column(columns={'frame': sampled_frame})
job = Job(op_args={
frame: static_table.column('frame'),
output: 'masked_video',
})
[table] = db.run(output=output, jobs=[job], force=True)
table.column('frame').save_mp4('02_masked')
videos.append('02_masked.mp4')
# 5. Unbounded State:
# This Op will always process all preceding elements of
# its input streams before producing a requested output.
# This means that sampling operations after this Op
# can not change how many inputs it receives. In the next
# tutorial, we will show how this can be relaxed for
# sub-streams of the input.
@scannerpy.register_python_op(unbounded_state=True)
class Example(scannerpy.Kernel):
def __init__(self, config):
pass
def reset(self):
pass
def execute(self, frame: FrameType) -> bytes:
pass
print('Finished! The following videos were written: {:s}'.format(
', '.join(videos)))
encoded_edges2 = db.ops.Pass(input=encoded_edges)
output_op = db.sinks.Column(
columns={
'image': encoded_image2,
'poseimg': encoded_poseimg2,
'edges': encoded_edges2
})
job = Job(
op_args={
encoded_image: {
'paths': image_files,
**params
},
encoded_poseimg: {
'paths': poseimg_files,
**params
},
encoded_edges: {
'paths': edge_files,
**params
},
output_op: opt.table_name,
})
[_out_table] = db.run(output_op, [job],
force=True,
tasks_in_queue_per_pu=1)
print(db.summarize())
jq '.spec.ports[0].nodePort' -r
''',
shell=True).strip().decode('utf-8')
master = '{}:{}'.format(ip, port)
print('Connecting to Scanner database...')
db = Database(master=master, start_cluster=False, config_path='./config.toml')
print('Running Scanner job...')
example_video_path = 'sample.mp4'
[input_table], _ = db.ingest_videos([('example', example_video_path)],
force=True,
inplace=True)
print(db.summarize())
frame = db.sources.FrameColumn()
r_frame = db.ops.Resize(frame=frame, width=320, height=240)
output_op = db.sinks.Column(columns={'frame': r_frame})
job = Job(op_args={
frame: db.table('example').column('frame'),
output_op: 'example_frame'
})
output_tables = db.run(output=output_op, jobs=[job], force=True)
output_tables[0].column('frame').save_mp4('resized_video')
print('Complete!')
job = Job(
op_args={
encoded_image: {
'paths': image_files,
**params
},
encoded_mask: {
'paths': mask_files,
**params
},
output_op: 'example_resized5',
})
start = time.time()
[out_table] = db.run(output_op, [job],
force=True,
work_packet_size=opt.work_packet_size,
io_packet_size=opt.io_packet_size)
end = time.time()
print('Total time for depth estimation in scanner: {0:.3f} sec'.format(
end - start))
# results = out_table.column('frame').load()
#
# path_to_save = join(dataset, 'players', 'prediction_scanner')
# if not os.path.exists(path_to_save):
# os.mkdir(path_to_save)
#
# for i, res in enumerate(results):
# pred_scanner = np.argmax(res, axis=0)
# np.save(join(path_to_save, '{0:05d}.npy'.format(i)), res)
def main():
# Initialize a connection to the Scanner database. Loads configuration from the
# ~/.scanner.toml configuration file.
db = Database()
# Create a Scanner table from our video in the format (table name,
# video path). If any videos fail to ingest, they'll show up in the failed
# list. If force is true, it will overwrite existing tables of the same
# name.
example_video_path = util.download_video()
_, failed = db.ingest_videos([('example', example_video_path),
('example2', example_video_path),
('thisshouldfail', 'thisshouldfail.mp4')],
force=True)
print(db.summarize())
print('Failures:', failed)
# Scanner processes videos by forming a graph of operations that operate
# on input frames from a table and produce outputs to a new table.
# FrameColumn declares that we want to read from a table column that
# represents a video frame.
frame = db.sources.FrameColumn()
# These frames are input into a Histogram op that computes a color histogram
# for each frame.
hist = db.ops.Histogram(frame=frame)
# Finally, any columns provided to Output will be saved to the output
# table at the end of the computation. Here, 'hist' is the name of the
# column for the output table.
output_op = db.sinks.Column(columns={'hist': hist})
# A job defines a table you want to create. In op_args, we bind the
# FrameColumn from above to the table we want to read from and name
# the output table 'example_hist' by binding a string to output_op.
job = Job(op_args={
frame: db.table('example').column('frame'),
output_op: 'example_hist'
})
job2 = Job(op_args={
frame: db.table('example2').column('frame'),
output_op: 'example_hist2'
})
# This executes the job and produces the output table. You'll see a progress
# bar while Scanner is computing the outputs.
output_tables = db.run(output=output_op, jobs=[job, job2], force=True)
# Load the histograms from a column of the output table. The
# readers.histograms function converts the raw bytes output by Scanner
# into a numpy array for each channel.
video_hists = output_tables[0].column('hist').load(readers.histograms)
# Loop over the column's values, a set of 3 histograms (for each color channel) per element.
num_rows = 0
for frame_hists in video_hists:
assert len(frame_hists) == 3
assert frame_hists[0].shape[0] == 16
num_rows += 1
assert num_rows == db.table('example').num_rows()
keypoints = db.sources.Column()
descriptors = db.sources.Column()
camera = db.sources.Column()
pair_image_ids, two_view_geometries = db.ops.SequentialMatchingCPU(
image_ids=image_ids,
keypoints=keypoints,
descriptors=descriptors,
stencil=matching_stencil)
output = db.sinks.Column(columns={
'pair_image_ids': pair_image_ids,
'two_view_geometries': two_view_geometries
})
job = Job(
op_args={
image_ids: db.table(args.input_table).column('image_id'),
keypoints: db.table(args.input_table).column('keypoints'),
descriptors: db.table(args.input_table).column('descriptors'),
camera: db.table(args.input_table).column('camera'),
output: args.output_table
})
output_tables = db.run(
output, [job],
force=True,
io_packet_size=args.packet_size,
work_packet_size=args.packet_size)
print(db.summarize())
encoded_image = db.sources.Files(**params)
output_op = db.sinks.FrameColumn(columns={'image': encoded_image})
job = Job(
op_args={
encoded_image: {
'paths': image_files,
**params
},
output_op: 'edge_detect',
})
[_out_table] = db.run(output_op, [job],
force=True,
tasks_in_queue_per_pu=1)
print(db.summarize())
encoded_image = db.sources.FrameColumn()
frame = db.ops.ImageDecoder(img=encoded_image)
my_edge_detection_class = db.ops.EdgeDetection(frame=frame,
model_path='model.yml.gz')
output_op = db.sinks.FrameColumn(
columns={'frame': my_edge_detection_class})
job = Job(
op_args={
encoded_image: db.table('edge_detect').column('image'),
output_op: 'edge_output',
def main(movie_path):
total_start = time.time()
print('Detecting shots in movie {}'.format(movie_path))
movie_name = os.path.basename(movie_path)
# Use GPU kernels if we have a GPU
db = Database()
print('Loading movie into Scanner database...')
s = time.time()
if db.has_gpu():
device = DeviceType.GPU
else:
device = DeviceType.CPU
############ ############ ############ ############
# 0. Ingest the video into the database
############ ############ ############ ############
[movie_table], _ = db.ingest_videos([(movie_name, movie_path)], force=True)
print('Time: {:.1f}s'.format(time.time() - s))
print('Number of frames in movie: {:d}'.format(movie_table.num_rows()))
s = time.time()
############ ############ ############ ############
# 1. Run Histogram over the entire video in Scanner
############ ############ ############ ############
print('Computing a color histogram for each frame...')
frame = db.sources.FrameColumn()
histogram = db.ops.Histogram(frame=frame, device=device)
output = db.sinks.Column(columns={'histogram': histogram})
job = Job(op_args={
frame: movie_table.column('frame'),
output: movie_name + '_hist'
})
[hists_table] = db.run(output=output, jobs=[job], force=True)
print('\nTime: {:.1f}s, {:.1f} fps'.format(
time.time() - s,
movie_table.num_rows() / (time.time() - s)))
s = time.time()
############ ############ ############ ############
# 2. Load histograms and compute shot boundaries
# in python
############ ############ ############ ############
print('Computing shot boundaries...')
# Read histograms from disk
hists = [
h for h in hists_table.column('histogram').load(parsers.histograms)
]
boundaries = compute_shot_boundaries(hists)
print('Found {:d} shots.'.format(len(boundaries)))
print('Time: {:.1f}s'.format(time.time() - s))
s = time.time()
############ ############ ############ ############
# 3. Create montage in Scanner
############ ############ ############ ############
print('Creating shot montage...')
row_length = min(16, len(boundaries))
rows_per_item = 1
target_width = 256
# Compute partial row montages that we will stack together
# at the end
frame = db.sources.FrameColumn()
gather_frame = frame.sample()
sliced_frame = gather_frame.slice()
montage = db.ops.Montage(frame=sliced_frame,
num_frames=row_length * rows_per_item,
target_width=target_width,
frames_per_row=row_length,
device=device)
sampled_montage = montage.sample()
output = db.sinks.Column(
columns={'montage': sampled_montage.unslice().lossless()})
item_size = row_length * rows_per_item
starts_remainder = len(boundaries) % item_size
evenly_divisible = (starts_remainder == 0)
if not evenly_divisible:
boundaries = boundaries[0:len(boundaries) - starts_remainder]
job = Job(
op_args={
frame:
movie_table.column('frame'),
gather_frame:
db.sampler.gather(boundaries),
sliced_frame:
db.partitioner.all(item_size),
sampled_montage: [
db.sampler.gather([item_size - 1])
for _ in range(len(boundaries) / item_size)
],
output:
'montage_image'
})
[montage_table] = db.run(output=output, jobs=[job], force=True)
# Stack all partial montages together
montage_img = np.zeros((1, target_width * row_length, 3), dtype=np.uint8)
for img in montage_table.column('montage').load():
img = np.flip(img, 2)
montage_img = np.vstack((montage_img, img))
print('')
print('Time: {:.1f}s'.format(time.time() - s))
############ ############ ############ ############
# 4. Write montage to disk
############ ############ ############ ############
cv2.imwrite('shots.jpg', montage_img)
print('Successfully generated shots.jpg')
print('Total time: {:.2f} s'.format(time.time() - total_start))
frame = db.sources.FrameColumn()
blurred_frame = db.ops.Blur(frame=frame, kernel_size=3, sigma=0.5)
sampled_frame = db.streams.Range(blurred_frame, 0, 30)
return frame, sampled_frame
# By default, if an Op outputs a frame with 3 channels with type uint8,
# those frames will be compressed using video encoding. No other frame
# type is currently compressed.
frame, blurred_frame = make_blurred_frame()
output_op = db.sinks.Column(columns={'frame': blurred_frame})
job = Job(op_args={
frame: db.table('example').column('frame'),
output_op: 'output_table_name',
})
db.run(output_op, [job], force=True)
frame, blurred_frame = make_blurred_frame()
# The compression parameters can be controlled by annotating the column
low_quality_frame = blurred_frame.compress_video(quality=35)
output_op = db.sinks.Column(columns={'frame': low_quality_frame})
job = Job(op_args={
frame: db.table('example').column('frame'),
output_op: 'low_quality_table',
})
db.run(output_op, [job], force=True)
# If no compression is desired, this can be specified by indicating that
# the column should be lossless.
frame, blurred_frame = make_blurred_frame()
# The compression parameters can be controlled by annotating the column
**params
},
calibrate_video_class: {
'w': 1920,
'h': 1080,
'A': calibs[i]['A'],
'R': calibs[i]['R'],
'T': calibs[i]['T']
},
output_op: 'example_resized_{0}'.format(i),
})
jobs.append(job)
start = time.time()
tables = db.run(output_op, jobs, force=True)
end = time.time()
for i in range(len(video_names)):
savename = join(path_to_data, video_names[i], 'calib_scanner2')
tables[i].column('frame').save_mp4(savename)
print('Successfully generated {0}_faces.mp4 in {1:.3f} secs'.format(
savename, end - start))
# # ======================================================================================================================
# # Images
# # ======================================================================================================================
#
# total_files = -1
# image_files = glob.glob(join(dataset_list[0], 'images', '*.jpg'))
# image_files.sort()
h=128,
sigma1=1.0,
sigma2=0.01,
model_path=model_path)
output_op = db.sinks.FrameColumn(columns={'frame': my_segment_imageset_class})
job = Job(
op_args={
encoded_image: {
'paths': image_files,
**params
},
encoded_mask: {
'paths': mask_files,
**params
},
output_op: 'example_resized',
})
start = time.time()
[out_table] = db.run(output_op, [job],
force=True,
work_packet_size=8,
io_packet_size=32,
tasks_in_queue_per_pu=4)
end = time.time()
print('Total time for instance segmentation in scanner: {0:.3f} sec'.format(
end - start))
# out_table.column('frame').save_mp4(join(dataset, 'players', 'instance_segm.mp4'))
cluster_ids, cameras, images, points3d = db.ops.IncrementalMappingCPU(
image_id=image_ids, pair_image_ids=pair_image_ids, two_view_geometries=two_view_geometries, keypoints=keypoints, camera=cameras, batch=batch_size, stencil=matching_stencil)
################################################################################
# remove empty rows from sparse reconstruction
################################################################################
def remove_empty_rows(*input_cols):
return [db.streams.Stride(col, batch_size) for col in input_cols]
cluster_id, cameras, images, points3d = remove_empty_rows(
cluster_ids, cameras, images, points3d)
################################################################################
# merge submodels
################################################################################
cluster_id, cameras, images, points3d = db.ops.MergeMappingCPU(
cluster_id=cluster_id, cameras=cameras, images=images, points3d=points3d, batch=num_submodels)
output = db.sinks.Column(
columns={'cluster_id': cluster_id, 'cameras': cameras, 'images': images, 'points3d': points3d})
job = Job(op_args={
files: {'paths': image_paths},
output: args.output_table})
output_tables = db.run(output, [job], force=True)
print(db.summarize())
if len(sys.argv) <= 1:
print('Usage: main.py <video_file>')
exit(1)
video_path = sys.argv[1]
print('Performing optical flow on {}...'.format(video_path))
video_name = os.path.splitext(os.path.basename(video_path))[0]
db = Database()
if not db.has_table(video_name):
db.ingest_videos([(video_name, video_path)])
input_table = db.table(video_name)
frame = db.sources.FrameColumn()
flow = db.ops.OpticalFlow(
frame = frame,
device=DeviceType.CPU)
sampled_flow = db.streams.Range(flow, 0, 60)
output = db.sinks.Column(columns={'flow': sampled_flow})
job = Job(op_args={
frame: input_table.column('frame'),
output: input_table.name() + '_flow'
})
[output_table] = db.run(output=output, jobs=[job],
pipeline_instances_per_node=1, force=True)
vid_flows = [flow[0] for flow in output_table.column('flow').load(rows=[0])]
np.save('flows.npy', vid_flows)
scales = 3
else:
print('Using CPUs')
device = DeviceType.CPU
pipeline_instances = 1
scales = 1
frame = db.sources.FrameColumn()
poses_out = db.ops.OpenPose(frame=frame,
pose_num_scales=scales,
pose_scale_gap=0.33,
device=device)
drawn_frame = db.ops.PoseDraw(frame=frame, frame_poses=poses_out)
sampled_frames = sampler(drawn_frame)
output = db.sinks.Column(columns={'frame': sampled_frames})
job = Job(
op_args={
frame: input_table.column('frame'),
sampled_frames: sampler_args,
output: movie_name + '_drawn_poses',
})
[drawn_poses_table] = db.run(output=output,
jobs=[job],
work_packet_size=8,
io_packet_size=64,
pipeline_instances_per_node=pipeline_instances,
force=True)
print('Writing output video...')
drawn_poses_table.column('frame').save_mp4('{:s}_poses'.format(movie_name))
