from scannerpy import Database, Job
################################################################################
# This tutorial shows how to select different frames of a video to process. #
################################################################################
with Database() as db:
# We can access previously created tables with db.table(name).
input_table = db.table('example')
# You can tell Scanner which frames of the video (or which rows of a video
# table) you want to sample. Here, the "strided" sampling mode will run over
# every 8th frame, i.e. frames [0, 8, 16, ...]
frame, frame_info = input_table.as_op().strided(8)
# We pass the job to the database same as before, and can process the output
# same as before.
histogram = db.ops.Histogram(frame = frame, frame_info = frame_info)
job = Job(columns = [histogram], name = 'example_hist_strided')
output_table = db.run(job, force=True)
# Here's some examples of other sampling modes.
# Range takes a specific subset of a video. Here, it runs over all frames from
# 0 to 100
input_table.as_op().range(0, 100)
# Gather takes an arbitrary list of frames from a video.
input_table.as_op().gather([10, 17, 32])
strided_frame = db.streams.Stride(frame, stride)
model_name = 'ssd_mobilenet_v1_coco_2017_11_17'
model_url = MODEL_TEMPLATE_URL.format(model_name)
# Call the newly created object detect op
objdet_frame = db.ops.ObjDetect(
frame=strided_frame,
dnn_url=model_url,
device=DeviceType.GPU if db.has_gpu() else DeviceType.CPU,
batch=2)
output_op = db.sinks.Column(columns={'bundled_data': objdet_frame})
job = Job(
op_args={
frame: db.table('example').column('frame'),
output_op: 'example_obj_detect',
})
[out_table] = db.run(output=output_op,
jobs=[job],
force=True,
pipeline_instances_per_node=1)
out_table.profiler().write_trace('obj.trace')
print('Extracting data from Scanner output...')
# bundled_data_list is a list of bundled_data
# bundled data format: [box position(x1 y1 x2 y2), box class, box score]
bundled_data_list = [
def start_mxnet_pipeline(test_video_path='videos/example.mp4',
out_dir='./',
batch=BATCH_SIZE,
load_to_disk=False):
global timelist
if util.have_gpu():
device = DeviceType.GPU
print('with GPU device!')
else:
device = DeviceType.CPU
print('only has CPU device!')
script_dir = os.path.dirname(os.path.abspath(__file__))
num_rows = 0
# Start Scanner DB, use its load worker to generate .proto and .bin files
with Database() as db:
# register the fake kernel
db.register_op('Fake', [('frame', ColumnType.Video)], ['class'])
kernel_path = script_dir + '/fake_op.py'
db.register_python_kernel('Fake', device, kernel_path, batch=10)
# Choose Fake kernel can be faster, or you can choose PyMxnet
# db.register_op('PyMxnet', [('frame', ColumnType.Video)], ['class'])
# kernel_path = script_dir + '/pymxnet_op.py'
# db.register_python_kernel('PyMxnet', DeviceType.CPU, kernel_path, batch=10)
start = now()
[input_table
], failed = db.ingest_videos([('end2end_raw', test_video_path)],
force=True)
stop = now()
delta = stop - start
print('Time to ingest videos: {:.4f}s, fps: {:.4f}'.format(
delta,
input_table.num_rows() / delta))
# timelist += '"ingest-video" : %f,' % (delta)
timelist["ingest-video"] = delta
num_rows = input_table.num_rows()
print('Number of frames in movie: {:d}'.format(num_rows))
if len(failed) > 0:
print('Failures:', failed)
# Start to analyze the movie
start = now()
frame = db.ops.FrameInput()
# Then we use our op just like in the other examples.
# Choose Fake kernel can be faster, or you can choose PyMxnet
classes = db.ops.Fake(frame=frame, batch=batch)
# classes = db.ops.PyMxnet(frame = frame, batch = batch)
output_op = db.ops.Output(columns=[classes])
job = Job(op_args={
frame: input_table.column('frame'),
output_op: 'end2end_out'
})
bulk_job = BulkJob(output=output_op, jobs=[job])
[output_table] = db.run(bulk_job,
force=True,
profiling=False,
pipeline_instances_per_node=1,
load_to_disk=load_to_disk,
work_packet_size=WORK_PACKET_SIZE)
stop = now()
delta = stop - start
print(
'Batch: {:d} End-to-end Python Kernel time: {:.4f}s, {:.1f} fps\n'.
format(batch, delta,
input_table.num_rows() / delta))
# timelist += '"scanner-execution" : %f,' % (delta)
timelist["scanner-execution"] = delta
# output_table.profiler().write_trace(
# out_dir + 'end2end_{:d}.trace'.format(batch))
# If not load_to_disk, then it does not go to the next part
if load_to_disk == False:
video_classes = output_table.load(['class'], parsers.classes)
# Loop over the column's rows.
# Each row is a tuple of the frame number and value for that row.
num_rows = 0
for (frame_index, frame_classes) in video_classes:
assert len(frame_classes) == 1
assert frame_classes[0].shape[0] == 1
# print(frame_classes[0])
num_rows += 1
assert num_rows == db.table('end2end_raw').num_rows()
print(db.summarize())
exit()
# Then start the Lambda part
# extract video name
videoPrefix = test_video_path.split(".")[-2].split("/")[-1]
print('video name is: {:s}'.format(videoPrefix))
# uploadPrefix = UPLOAD_PREFIX + '/' + videoPrefix
uploadPrefix = UPLOAD_PREFIX + '/{}_{}'.format(videoPrefix,
WORK_PACKET_SIZE)
if load_to_disk == True:
# Upload all .proto files
start = now()
fileCount, totalSize = upload_output_to_s3(UPLOAD_BUCKET, uploadPrefix,
PROTO_EXT)
# Upload all .bin files
fileCount, totalSize = upload_output_to_s3(UPLOAD_BUCKET, uploadPrefix,
BIN_EXT)
stop = now()
delta = stop - start
print('Upload to S3 time: {:.4f} s'.format(delta))
# timelist += '"upload-s3" : %f,' % (delta)
timelist["upload-s3"] = delta
# Call Lambdas to decode, provide Bucket Name, File Prefix, Start Frame
# Then decoder Lambdas will write to S3, which will trigger MXNet Lambdas
start = now()
lambdaTotalCount = len(xrange(0, num_rows, WORK_PACKET_SIZE))
bar = progressbar.ProgressBar(maxval=lambdaTotalCount, \
widgets=[progressbar.Bar('=', 'Lambdas [', ']'), ' ',
progressbar.Percentage()])
bar.start()
lambdaCount = 0
for startFrame in xrange(0, num_rows, WORK_PACKET_SIZE):
# print("Invoke lambda for start frame {:d}".format(startFrame))
result = invoke_decoder_lambda(UPLOAD_BUCKET, uploadPrefix,
startFrame, batch)
if not result:
print(
'Fail to invoke for frame {:d}, retry.'.format(startFrame))
res = invoke_decoder_lambda(UPLOAD_BUCKET, uploadPrefix,
startFrame, batch)
if not res:
print('Frame {:d} still failed, exit'.format(startFrame))
exit()
lambdaCount += 1
bar.update(lambdaCount)
bar.finish()
stop = now()
delta = stop - start
assert (lambdaCount == lambdaTotalCount)
print('Triggered #{} Lambdas, time {:.4f} s'.format(
lambdaCount, delta))
# timelist += '"invoke-lambda" : %f,' % (delta)
timelist["invoke-lambda"] = delta
# Wait until all output files appear
fileCount = wait_until_all_finished(0, num_rows, batch, videoPrefix)
# assert(fileCount == len(xrange(0, num_rows, batch)))
totalCount = len(xrange(0, num_rows, batch))
print('Collected {:d} out of {:d} files, error rate: {:.4f}'.format(
fileCount, totalCount,
(totalCount - fileCount) * 1.0 / totalCount))
def start_fuse_pipeline(videoPath, args):
global timelist
if util.have_gpu():
device = DeviceType.GPU
print('has GPU device!')
else:
device = DeviceType.CPU
print('only has CPUs!')
scriptDir = './'
numFrames = 0
batch = args.batch
videoPrefix = videoPath.split(".")[-2].split("/")[-1]
print('Video name is: {:s}'.format(videoPrefix))
uploadPrefix = os.path.join(args.uploadPrefix,
'{}_{}'.format(videoPrefix, batch))
args.uploadPrefix = uploadPrefix
###################################################
# 0. Start Scanner DB
# Use its load worker to generate .proto and .bin files
###################################################
with Database() as db:
# Register the fake kernel
db.register_op('Fake', [('frame', ColumnType.Video)], ['class'])
kernelPath = os.path.join(scriptDir, 'fake_op.py')
db.register_python_kernel('Fake', device, kernelPath, batch=10)
####################
# Ingest the video
####################
start = now()
[input_table
], failed = db.ingest_videos([('end2end_fused_raw', videoPath)],
force=True)
stop = now()
delta = stop - start
print('Time to ingest videos: {:.4f}s, fps: {:.4f}'.format(
delta,
input_table.num_rows() / delta))
timelist["ingest-video"] = delta
numFrames = input_table.num_rows()
print('Number of frames in movie: {:d}'.format(numFrames))
if len(failed) > 0:
print('Failures:', failed)
######################
# Prepare decode data
######################
start = now()
frame = db.ops.FrameInput()
classes = db.ops.Fake(frame=frame, batch=batch)
output_op = db.ops.Output(columns=[classes])
job = Job(op_args={
frame: input_table.column('frame'),
output_op: 'end2end_fused_out'
})
bulk_job = BulkJob(output=output_op, jobs=[job])
[output_table] = db.run(bulk_job,
force=True,
profiling=False,
pipeline_instances_per_node=1,
load_to_disk=True,
work_packet_size=batch)
stop = now()
delta = stop - start
print('Batch: {:d} Decode preparation time: {:.4f}s, {:.1f} fps\n'.
format(batch, delta, numFrames / delta))
timelist["pre-decode"] = delta
###################################################
# 1. Start the Lambda part
# Use Lambdas to decode and evaluate kernels
###################################################
######################
# 1.0 Upload all .proto files
######################
uploadBucket = args.uploadBucket
start = now()
fileCount, totalSize = upload_output_to_s3(uploadBucket, uploadPrefix,
PROTO_EXT)
######################
# 1.1 Upload all .bin files
######################
fileCount, totalSize = upload_output_to_s3(uploadBucket, uploadPrefix,
BIN_EXT)
stop = now()
delta = stop - start
print('Upload args to S3 time: {:.4f} s'.format(delta))
timelist["upload-s3"] = delta
################################################
# 1.2 Call Lambdas to decode + evaluate,
# provide Bucket Name, File Prefix, numFrames, batch
################################################
start = now()
lambdaCount = invoke_lambdas(numFrames, args)
stop = now()
delta = stop - start
print('Triggered #{} Lambdas, time {:.4f} s'.format(lambdaCount, delta))
timelist["invoke-lambda"] = delta
################################################
# 1.3 Wait until all output files appear
################################################
fileCount = wait_until_all_finished(0, numFrames, videoPrefix, args)
totalCount = len(xrange(0, numFrames, batch))
print('Collected {:d} out of {:d} files, error rate: {:.4f}'.format(
fileCount, totalCount, (totalCount - fileCount) * 1.0 / totalCount))
def test_stencil(db):
frame = db.sources.FrameColumn()
sample_frame = db.streams.Range(frame, 0, 1)
flow = db.ops.OpticalFlow(frame=sample_frame, stencil=[-1, 0])
output_op = db.sinks.Column(columns={'flow': flow})
job = Job(op_args={
frame: db.table('test1').column('frame'),
output_op: 'test_sencil',
})
tables = db.run(
output_op, [job],
force=True,
show_progress=False,
pipeline_instances_per_node=1)
num_rows = 0
for _ in tables[0].column('flow').load():
num_rows += 1
assert num_rows == 1
frame = db.sources.FrameColumn()
sample_frame = db.streams.Range(frame, 0, 1)
flow = db.ops.OpticalFlow(frame=sample_frame, stencil=[0, 1])
output_op = db.sinks.Column(columns={'flow': flow})
job = Job(op_args={
frame: db.table('test1').column('frame'),
output_op: 'test_sencil',
})
tables = db.run(
output_op, [job],
force=True,
show_progress=False,
pipeline_instances_per_node=1)
frame = db.sources.FrameColumn()
sample_frame = db.streams.Range(frame, 0, 2)
flow = db.ops.OpticalFlow(frame=sample_frame, stencil=[0, 1])
output_op = db.sinks.Column(columns={'flow': flow})
job = Job(op_args={
frame: db.table('test1').column('frame'),
output_op: 'test_sencil',
})
tables = db.run(
output_op, [job],
force=True,
show_progress=False,
pipeline_instances_per_node=1)
num_rows = 0
for _ in tables[0].column('flow').load():
num_rows += 1
assert num_rows == 2
frame = db.sources.FrameColumn()
flow = db.ops.OpticalFlow(frame=frame, stencil=[-1, 0])
sample_flow = db.streams.Range(flow, 0, 1)
output_op = db.sinks.Column(columns={'flow': sample_flow})
job = Job(op_args={
frame: db.table('test1').column('frame'),
output_op: 'test_sencil',
})
tables = db.run(
output_op, [job],
force=True,
show_progress=False,
pipeline_instances_per_node=1)
num_rows = 0
for _ in tables[0].column('flow').load():
num_rows += 1
assert num_rows == 1
encoded_image = db.sources.Files()
frame = db.ops.ImageDecoder(img=encoded_image)
encoded_mask = db.sources.Files()
mask_frame = db.ops.ImageDecoder(img=encoded_mask)
my_depth_estimation_class = db.ops.MyDepthEstimationClass(
image=frame, mask=mask_frame, img_size=256, model_path=model_path)
output_op = db.sinks.FrameColumn(columns={'frame': my_depth_estimation_class})
job = Job(
op_args={
encoded_image: {
'paths': image_files
},
encoded_mask: {
'paths': mask_files
},
output_op: 'example_resized',
})
[out_table] = db.run(output_op, [job], force=True)
results = out_table.column('frame').load()
path_to_save = join(dataset, 'players', 'prediction_scanner')
if not os.path.exists(path_to_save):
os.mkdir(path_to_save)
for i, res in enumerate(results):
pred_scanner = np.argmax(res, axis=0)
with Database() as db:
# Scanner can take a directed acyclic graph (DAG) of operators and pass data
# between them. Each graph has starts with data from an input table.
frame = db.ops.FrameInput()
blurred_frame = db.ops.Blur(frame=frame, kernel_size=3, sigma=0.5)
# Multiple operators can be hooked up in a computation by using the outputs
# of one as the inputs of another.
histogram = db.ops.Histogram(frame=blurred_frame)
output_op = db.ops.Output(columns=[histogram])
job = Job(op_args={
frame: db.table('example').column('frame'),
output_op: 'output_table',
})
bulk_job = BulkJob(output=output_op, jobs=[job])
db.run(bulk_job, force=True)
# Ops can have several attributes that affect which stream elements they
# will receive or how they will receive them. These attributes include:
#
# - Batch: The Op can receive multiple elements at once to enable SIMD
# or vector-style processing.
#
# - Stencil: The Op requires a window of input elements (for example, the
# previous and next element) at the same time to produce an
# output.
#
# represents a video frame.
frame = db.ops.FrameInput()
# These frames are input into a Histogram op that computes a color histogram
# for each frame.
hist = db.ops.Histogram(frame=frame)
# Finally, any columns provided to Output will be saved to the output
# table at the end of the computation.
output_op = db.ops.Output(columns=[hist])
# A job defines a table you want to create. In op_args, we bind the frame
# input column from above to the table we want to read from and name
# the output table 'example_hist' by binding a string to output_op.
job1 = Job(op_args={
frame: db.table('test_raw1').column('frame'),
output_op: 'test_hist1'
})
job2 = Job(op_args={
frame: db.table('test_raw2').column('frame'),
output_op: 'test_hist2'
})
# Multiple tables can be created using the same execution graph using
# a bulk job. Here we specify the execution graph (or DAG) by providing
# the output_op and also specify the jobs we wish to compute.
bulk_job = BulkJob(output=output_op, jobs=[job1, job2])
# This executes the job and produces the output table. You'll see a progress
# bar while Scanner is computing the outputs.
start = now()
# 1. Device Type:
# Ops can specify that they require CPUs or GPUs by declaring their device
# type. By default, the device_type is DeviceType.CPU.
@scannerpy.register_python_op(device_type=DeviceType.CPU)
def device_resize(config, frame: FrameType) -> FrameType:
return cv2.resize(frame, (config.args['width'], config.args['height']))
frame = db.sources.FrameColumn()
resized_frame = db.ops.device_resize(frame=frame, width=640, height=480)
output = db.sinks.FrameColumn(columns={'frame': resized_frame})
job = Job(op_args={
frame: input_table.column('frame'),
output: 'example_resize'
})
#[table] = db.run(output=output, jobs=[job], force=True)
#table.column('frame').save_mp4('02_device_resize')
videos.append('02_device_resize.mp4')
# 2. Batch:
# The Op can receive multiple elements at once to enable SIMD or
# vector-style processing.
@scannerpy.register_python_op(batch=10)
def batch_resize(config, frame: Sequence[FrameType]) -> Sequence[FrameType]:
input_tables, failed = db.ingest_videos([(movie_name, movie_path), ('mask', mask_path)], force=True)
print(db.summarize())
print('Failures:', failed)
cam_data = np.load('/home/krematas/Mountpoints/grail/data/barcelona/calib/00114.npy').item()
# db.register_op('Calibrate', [('frame', ColumnType.Video), ('mask', ColumnType.Video)], [('resized', ColumnType.Video)])
# Custom Python kernels for ops reside in a separate file, here calibrate_kernel.py.
# cwd = '/home/krematas/code/scanner/examples/apps/soccer'
# db.register_python_kernel('Calibrate', DeviceType.CPU, cwd + '/calibrate_kernel.py')
frame = db.sources.FrameColumn()
mask = db.sources.FrameColumn()
# Then we use our op just like in the other examples.
calibrate_video_class = db.ops.CalibrationClass(frame=frame, mask=mask, w=3840, h=2160, A=cam_data['A'], R=cam_data['R'], T=cam_data['T'])
output_op = db.sinks.FrameColumn(columns={'frame': calibrate_video_class})
job = Job(op_args={
frame: input_tables[0].column('frame'),
mask: input_tables[1].column('frame'),
output_op: 'example_resized',
})
[out_table] = db.run(output_op, [job], force=True)
out_table.column('frame').save_mp4(movie_name + '_faces')
print('Successfully generated {:s}_faces.mp4'.format(movie_name))
if db.has_gpu():
print('Using GPUs')
device = DeviceType.GPU
pipeline_instances = -1
else:
print('Using CPUs')
device = DeviceType.CPU
pipeline_instances = 1
frame = db.sources.FrameColumn()
poses_out = db.ops.OpenPose(
frame=frame,
pose_num_scales=3,
pose_scale_gap=0.33,
device=device)
drawn_frame = db.ops.PoseDraw(frame=frame, frame_poses=poses_out)
sampled_frames = sampler(drawn_frame)
output = db.sinks.Column(columns={'frame': sampled_frames})
job = Job(
op_args={
frame: input_table.column('frame'),
sampled_frames: sampler_args,
output: movie_name + '_drawn_poses',
})
[drawn_poses_table] = db.run(output=output, jobs=[job], work_packet_size=8, io_packet_size=64,
pipeline_instances_per_node=pipelineinstances,
force=True)
print('Writing output video...')
drawn_poses_table.column('frame').save_mp4('{:s}_poses'.format(movie_name))
def test_mxnet_lambda(server="0.0.0.0", path="/hello", batch = 1, num = 3,
fm_num = 1, out_dir = './'):
if num > 4:
test_video_path = util.download_video2('http://web/stanford.edu/~jamesh93/video/wild720p.mkv')
else:
test_video_path = util.download_video1(num, fm_num)
print('#{:d} video, #{:d} format, outdir: {}'.format(num, fm_num, out_dir))
print('Lambda server: {}, Lambda path: {}'.format(server, path))
if util.have_gpu():
device = DeviceType.GPU
else:
device = DeviceType.CPU
with Database() as db:
if not os.path.isfile('lambda_op/build/libmxnetlambda_op.so'):
print('You need to build the custom op first: \n'
'$ cd lambda_op; mkdir build && cd build; cmake ..; make')
exit()
# To load a custom op into the Scanner runtime, we use db.load_op to open the
# shared library we compiled. If the op takes arguments, it also optionally
# takes a path to the generated python file for the arg protobuf.
db.load_op('lambda_op/build/libmxnetlambda_op.so', 'lambda_op/build/mxnetlambda_pb2.py')
start = now()
[input_table], failed = db.ingest_videos([
('test_mxnet_raw', test_video_path)], force=True)
stop = now()
delta = stop - start
print('Time to ingest videos: {:.4f}s, fps: {:.4f}'.format(
delta, input_table.num_rows() / delta))
num_rows = input_table.num_rows()
print('Number of frames in movie: {:d}'.format(num_rows))
if len(failed) > 0:
print('Failures:', failed)
# Start to analyze the movie
start = now()
frame = db.ops.FrameInput()
# Then we use our op just like in the other examples.
classes = db.ops.MxnetLambda(
frame = frame,
batch = batch,
device = device,
server = server, path = path)
output_op = db.ops.Output(columns=[classes])
job = Job(
op_args={
frame: db.table('test_mxnet_raw').column('frame'),
output_op: 'test_mxnet_lambda'
}
)
bulk_job = BulkJob(output=output_op, jobs=[job])
[output_table] = db.run(bulk_job, force=True, profiling=True)
stop = now()
delta = stop - start
print('Batch: {:d} MXNet Lambda time: {:.4f}s, {:.1f} fps\n'.format(
batch, delta, input_table.num_rows() / delta))
output_table.profiler().write_trace(
out_dir + 'test_mxnet_{:d}_{:d}_{:d}.trace'.format(num, fm_num, batch))
video_classes = output_table.load(['class'], parsers.classes)
# Loop over the column's rows. Each row is a tuple of the frame number and
# value for that row.
num_rows = 0
for (frame_index, frame_classes) in video_classes:
assert len(frame_classes) == 1
assert frame_classes[0].shape[0] == 1
print(frame_classes[0])
num_rows += 1
assert num_rows == db.table('test_mxnet_raw').num_rows()
print(db.summarize())
# we define a job later on.
# Let's write a pipeline that reads our images, resizes them, and writes them
# back out as files to the filesystem.
# Since the input images are compressed, we decompress them with the
# ImageDecoder
frame = db.ops.ImageDecoder(img=compressed_images)
resized = db.ops.Resize(frame=frame, width=640, height=360)
# Rencode the image to jpg
encoded_frame = db.ops.ImageEncoder(frame=resized, format='jpg')
# Write the compressed images to files
output = db.sinks.Files(input=encoded_frame)
resized_paths = ['resized-1.jpg', 'resized-2.jpg', 'resized-3.jpg']
job = Job(op_args={
compressed_images: {'paths': image_paths},
output: {'paths': resized_paths}
})
db.run(output=output, jobs=[job])
print('Finished! Wrote the following images: ' + ', '.join(resized_paths))
# If you want to learn how write your own custom Source or Sink in C++, check
# out tutorials 09_defining_cpp_sources.py and 10_defining_cpp_sinks.py
cluster_id, cameras, images, points3d = sample(cluster_id, cameras, images,
points3d)
output = db.sinks.Column(
columns={
'cluster_id': cluster_id,
'cameras': cameras,
'images': images,
'points3d': points3d
})
job = Job(
op_args={
image_ids:
db.table(args.extraction_table).column('image_id'),
pair_image_ids:
db.table(args.matching_table).column('pair_image_ids'),
two_view_geometries:
db.table(args.matching_table).column('two_view_geometries'),
keypoints:
db.table(args.extraction_table).column('keypoints'),
camera:
db.table(args.extraction_table).column('camera'),
output:
args.output_table
})
output_tables = db.run(output, [job], force=True)
print(db.summarize())
# the frame column (we will say how to sample when specifying a job).
sliced_frame = frame.slice()
# We process the sampled frame same as before.
hist = db.ops.Histogram(frame=sliced_frame)
unsliced_hist = hist.unslice()
output_op = db.ops.Output(columns=[unsliced_hist])
# For each job, you can specify how sampling should be performed for
# a specific column. In the same way we used the op_args argument to bind
# a table to an input column, we bind a sampling directive to strided_frame.
job = Job(
op_args={
frame: db.table('example').column('frame'),
# The "strided" sampling mode will run over # every 8th frame,
# i.e. frames [0, 8, 16, ...]
sliced_frame: db.partitioner.all(500),
output_op: 'example_hist_sliced'
})
bulk_job = BulkJob(output=output_op, jobs=[job])
output_tables = db.run(bulk_job, force=True, pipeline_instances_per_node=2)
# Loop over the column's rows. Each row is a tuple of the frame number and
# value for that row.
video_hists = output_tables[0].load(['histogram'], parsers.histograms)
num_rows = 0
for (frame_index, frame_hists) in video_hists:
assert len(frame_hists) == 3
assert frame_hists[0].shape[0] == 16
num_rows += 1
print(num_rows)
def job(self, db, ty):
frame, frame_info = db.table('test').as_op().all()
histogram = db.ops.Histogram(frame=frame,
frame_info=frame_info,
device=ty)
return Job(columns=[histogram], name='test_hist')
frame = db.ops.FrameInput()
caffe_frame = db.ops.CaffeInput(frame=frame,
args=caffe_args,
device=DeviceType.CPU)
cls_prob, rois, fc7 = db.ops.FasterRCNN(caffe_input=caffe_frame,
args=caffe_args,
device=DeviceType.GPU)
bboxes, feature = db.ops.FasterRCNNOutput(cls_prob=cls_prob,
rois=rois,
fc7=fc7,
args=caffe_args,
device=DeviceType.CPU)
output = db.ops.Output(columns=[bboxes, feature])
job = Job(op_args={
frame: input_table.column('frame'),
output: input_table.name() + '_detections'
})
[output] = db.run(output=output,
jobs=[job],
pipeline_instances_per_node=1,
work_packet_size=10,
io_packet_size=40,
force=True)
output = db.table(input_table.name() + '_detections')
output.profiler().write_trace('detect_test.trace')
print('Extracting frames...')
sliced_frame = db.streams.Slice(frame, partitioner=scene_partitions)
# Then we perform background subtraction and indicate we need 60 prior
# frames to produce correct output
masked_frame = db.ops.BackgroundSubtraction(frame=sliced_frame,
alpha=0.02,
threshold=0.05,
bounded_state=60)
# Since the background subtraction operation is done, we can unslice the
# sequence to join it back into a single contiguous stream. You must unslice
# sequences before feeding them back into sinks
unsliced_frame = db.streams.Unslice(masked_frame)
output = db.sinks.Column(columns={'frame': unsliced_frame})
job = Job(op_args={
frame: static_table.column('frame'),
output: '04_masked_video',
})
[table] = db.run(output=output, jobs=[job], force=True)
table.column('frame').save_mp4('04_masked')
videos = []
videos.append('04_masked.mp4')
# If you look at around 10 seconds in the output video, you should see a
# sharp transition in the output at the same time as the scene changes. This
# transition is the first cut we introduced using the slicing operator at frame
# 5350. If you look at 23 seconds, you should see smearing as the video changes
# scenes without a cut. This smearing is because we didn't introduce a cut at
# the right place in the second sequence (5400, 5830).
def handle(self, *args, **options):
face_labeler = Labeler.objects.get(name=options['labeler'])
feature_labeler, _ = Labeler.objects.get_or_create(name='facenet')
with open(options['path']) as f:
paths = [s.strip() for s in f.readlines()]
with Database() as db:
db.register_op('EmbedFaces', [('frame', ColumnType.Video), 'bboxes'], ['embeddings'])
db.register_python_kernel('EmbedFaces', DeviceType.CPU, cwd + '/embed_kernel.py')
frame = db.ops.FrameInput()
frame_strided = frame.sample()
bboxes = db.ops.Input()
embeddings = db.ops.EmbedFaces(frame=frame_strided, bboxes=bboxes)
output = db.ops.Output(columns=[embeddings])
jobs = []
face_insts = []
for path in paths:
video = Video.objects.get(path=path)
faces = Face.objects.filter(person__frame__video=video, labeler=face_labeler) \
.select_related('person__frame') \
.order_by('person__frame__video__id', 'person__frame__number')
faces = [f for f in faces if f.bbox_x2 - f.bbox_x1 >= .04]
frame_numbers = []
rows = []
cur_frame = None
insts = []
for f in faces:
if f.person.frame.id != cur_frame:
cur_frame = f.person.frame.id
rows.append([])
frame_numbers.append(f.person.frame.number)
rows[-1].append(db.protobufs.BoundingBox(
x1=f.bbox_x1, x2=f.bbox_x2, y1=f.bbox_y1, y2=f.bbox_y2))
insts.append(f.id)
face_insts.append(insts)
bbox_table = db.new_table(
path+'_bboxes', ['bboxes'], [[r] for r in rows], fn=writers.bboxes, force=True)
bbox_table = db.table(path+'_bboxes')
jobs.append(Job(op_args={
frame: db.table(path).column('frame'),
frame_strided: db.sampler.gather(frame_numbers),
bboxes: bbox_table.column('bboxes'),
output: path+'_embeddings'
}))
bulk_job = BulkJob(output=output, jobs=jobs)
output_tables = db.run(bulk_job, force=True, pipeline_instances_per_node=1)
output_tables = [db.table(path+'_embeddings') for path in paths]
features = []
for t, path, insts in zip(output_tables, paths, face_insts):
inst_idx = 0
embs = t.column('embeddings').load()
for _, emb in embs:
for i in range(0, len(emb), 512):
e = np.frombuffer(emb[i:i+512], dtype=np.float32)
features.append(FaceFeatures(
features=json.dumps(e.tolist()),
face_id=insts[inst_idx],
labeler=feature_labeler))
inst_idx += 1
FaceFeatures.objects.bulk_create(features)
# The Op DAG only sees the raw frames. For example, this table is stored
# as compressed video.
def make_blurred_frame():
frame = db.ops.FrameInput()
blurred_frame = db.ops.Blur(frame=frame, kernel_size=3, sigma=0.5)
return frame, blurred_frame
# By default, if an Op outputs a frame with 3 channels with type uint8,
# those frames will be compressed using video encoding. No other frame
# type is currently compressed.
frame, blurred_frame = make_blurred_frame()
output_op = db.ops.Output(columns=[blurred_frame])
job = Job(
op_args={
frame: db.table('example').column('frame'),
output_op: 'output_table_name',
})
bulk_job = BulkJob(output=output_op, jobs=[job])
db.run(bulk_job, force=True)
frame, blurred_frame = make_blurred_frame()
# The compression parameters can be controlled by annotating the column
low_quality_frame = blurred_frame.compress_video(quality=35)
output_op = db.ops.Output(columns=[low_quality_frame])
job = Job(
op_args={
frame: db.table('example').column('frame'),
output_op: 'low_quality_table',
})
bulk_job = BulkJob(output=output_op, jobs=[job])
db.load_op(os.path.join(cwd, 'op_cpp/build/libextraction_op.so'),
os.path.join(cwd, 'op_cpp/build/siftExtraction_pb2.py'))
image_ids = db.sources.Column()
frames = db.sources.FrameColumn()
# run SIFT extractions
keypoints, descriptors, cameras = db.ops.SiftExtraction(image_ids=image_ids,
frames=frames)
output = db.sinks.Column(
columns={
'image_id': image_ids,
'keypoints': keypoints,
'descriptors': descriptors,
'camera': cameras
})
job = Job(
op_args={
image_ids: db.table(args.input_table).column('image_id'),
frames: db.table(args.input_table).column('frame'),
output: args.output_table
})
output_tables = db.run(output, [job],
force=True,
io_packet_size=args.packet_size,
work_packet_size=args.packet_size)
print(db.summarize())
facenet_input = db.ops.FacenetInput(
frame = frame, frame_info = frame_info,
args = facenet_args,
device = DeviceType.GPU)
facenet = db.ops.Facenet(
facenet_input = facenet_input,
frame_info = frame_info,
args = facenet_args,
device = DeviceType.GPU)
facenet_output = db.ops.FacenetOutput(
facenet_output = facenet,
frame_info = frame_info,
args = facenet_args)
job = Job(columns = [facenet_output], name = 'example_faces_{}'.format(scale))
output = db.run(job, force=True, work_item_size=5)
outputs.append(output)
all_bboxes = [
[box for (_, box) in out.load(['bboxes'], parsers.bboxes)]
for out in outputs]
nms_bboxes = []
frames = len(all_bboxes[0])
runs = len(all_bboxes)
for fi in range(frames):
frame_bboxes = []
for r in range(runs):
frame_bboxes += (all_bboxes[r][fi])
def test_fault_tolerance(fault_db):
force_kill_spawn_port = 5012
normal_spawn_port = 5013
def worker_killer_task(config, master_address):
from scannerpy import ProtobufGenerator, Config, start_worker
import time
import grpc
import subprocess
import signal
import os
c = Config(None)
import scanner.metadata_pb2 as metadata_types
import scanner.engine.rpc_pb2 as rpc_types
import scanner.types_pb2 as misc_types
protobufs = ProtobufGenerator(config)
# Spawn a worker that we will force kill
script_dir = os.path.dirname(os.path.realpath(__file__))
with open(os.devnull, 'w') as fp:
p = subprocess.Popen(
[
'python3 ' + script_dir +
'/spawn_worker.py {:d}'.format(force_kill_spawn_port)
],
shell=True,
stdout=fp,
stderr=fp,
preexec_fn=os.setsid)
# Wait a bit for the worker to do its thing
time.sleep(10)
# Force kill worker process to trigger fault tolerance
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
p.kill()
p.communicate()
# Wait for fault tolerance to kick in
time.sleep(15)
# Spawn the worker again
subprocess.call(
[
'python3 ' + script_dir +
'/spawn_worker.py {:d}'.format(normal_spawn_port)
],
shell=True)
master_addr = fault_db._master_address
killer_process = Process(
target=worker_killer_task, args=(fault_db.config, master_addr))
killer_process.daemon = True
killer_process.start()
frame = fault_db.sources.FrameColumn()
range_frame = fault_db.streams.Range(frame, 0, 20)
sleep_frame = fault_db.ops.SleepFrame(ignore=range_frame)
output_op = fault_db.sinks.Column(columns={'dummy': sleep_frame})
job = Job(op_args={
frame: fault_db.table('test1').column('frame'),
output_op: 'test_fault',
})
table = fault_db.run(
output_op, [job],
pipeline_instances_per_node=1,
force=True,
show_progress=False)
table = table[0]
assert len([_ for _ in table.column('dummy').load()]) == 20
# Shutdown the spawned worker
channel = grpc.insecure_channel(
'localhost:' + str(normal_spawn_port),
options=[('grpc.max_message_length', 24499183 * 2)])
worker = fault_db.protobufs.WorkerStub(channel)
try:
worker.Shutdown(fault_db.protobufs.Empty())
except grpc.RpcError as e:
status = e.code()
if status == grpc.StatusCode.UNAVAILABLE:
print('could not shutdown worker!')
exit(1)
else:
raise ScannerException(
'Worker errored with status: {}'.format(status))
killer_process.join()
db = Database()
print('Ingesting video into Scanner ...')
[input_table], _ = db.ingest_videos([(movie_name, movie_path)], force=True)
sampler = db.streams.All
sampler_args = {}
print('Detecting faces...')
[bboxes_table] = pipelines.detect_faces(db, [input_table.column('frame')],
sampler, sampler_args,
movie_name + '_bboxes')
print('Drawing faces onto video...')
frame = db.sources.FrameColumn()
sampled_frame = sampler(frame)
bboxes = db.sources.Column()
out_frame = db.ops.DrawBox(frame=sampled_frame, bboxes=bboxes)
output = db.sinks.Column(columns={'frame': out_frame})
job = Job(
op_args={
frame: input_table.column('frame'),
sampled_frame: sampler_args,
bboxes: bboxes_table.column('bboxes'),
output: movie_name + '_bboxes_overlay',
})
[out_table] = db.run(output=output, jobs=[job], force=True)
out_table.column('frame').save_mp4(movie_name + '_faces')
print('Successfully generated {:s}_faces.mp4'.format(movie_name))
cameras_src = cameras = db.sources.Column()
images_src = images = db.sources.Column()
points3d_src = points3d = db.sources.Column()
cluster_id, cameras, images, points3d = db.ops.MergeMappingCPU(
cluster_id=cluster_id,
cameras=cameras,
images=images,
points3d=points3d,
num_models=num_submodels)
output = db.sinks.Column(
columns={
'cluster_id': cluster_id,
'cameras': cameras,
'images': images,
'points3d': points3d
})
job = Job(
op_args={
cluster_id_src: db.table(args.input_table).column('cluster_id'),
cameras_src: db.table(args.input_table).column('cameras'),
images_src: db.table(args.input_table).column('images'),
points3d_src: db.table(args.input_table).column('points3d'),
output: args.output_table
})
output_tables = db.run(output, [job], force=True)
print(db.summarize())
inst_frame = db.ops.ImageDecoder(img=encoded_inst_files)
@scannerpy.register_python_op(device_type=DeviceType.CPU)
def device_combine_masks(config, cnn_mask: FrameType, inst_mask: FrameType) -> bytes:
out = cnn_mask*inst_mask
return pickle.dumps(out)
combined_mask = db.ops.device_combine_masks(cnn_mask=cnn_frame, inst_mask=inst_frame)
output_op = db.sinks.FrameColumn(columns={'frame': combined_mask})
job = Job(
op_args={
encoded_cnn_files: {'paths': cnn_files, **params},
encoded_inst_files: {'paths': inst_files, **params},
output_op: 'masks',
})
start = time.time()
[out_table] = db.run(output_op, [job], force=True, work_packet_size=opt.work_packet_size,
io_packet_size=opt.io_packet_size, pipeline_instances_per_node=opt.pipeline_instances_per_node,
tasks_in_queue_per_pu=opt.tasks_in_queue_per_pu)
end = time.time()
print('Total time for instance segm in scanner: {0:.3f} sec for {1} images'.format(end - start, len(cnn_files)))
if opt.save:
def mkdir(path_to_dir):
if not os.path.exists(path_to_dir):
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/..')
import util
################################################################################
# This tutorial shows how to organize your videos into Collections. #
################################################################################
with Database() as db:
# Instead of ingesting each video into a table individually, we can group video
# tables into a single entity called a collection. Here, we create a collection
# called "example_collection" from the video in the previous example.
# Collections do not incur any runtime overhead, but are simply an abstraction
# for more easily managing your videos.
example_video_path = util.download_video()
input_collection, _ = db.ingest_video_collection('example_collection',
[example_video_path],
force=True)
print(db.summarize())
# You can use a collection the same way you use a table when defining a
# computation. This will run your computation over every table in the
# collection using the sampling mode you specify.
frame = input_collection.as_op().range(0, 100)
histogram = db.ops.Histogram(frame=frame)
job = Job(columns=[histogram], name='example_hist_collection')
output_collection = db.run(job, force=True)
# You can retrieve table objects off the collection.
output_table = output_collection.tables(0)
# table) you want to sample. Here, we indicate that we want to stride
# the frame column by 4 (select every 4th frame)
strided_frame = db.streams.Stride(frame, 4)
# We process the sampled frame same as before.
hist = db.ops.Histogram(frame=strided_frame)
output_op = db.sinks.Column(columns={'hist': hist})
# For each job, you can specify how sampling should be performed for
# a specific stream. In the same way we used the op_args argument to bind
# a table to an input column, we can bind sampling arguments to strided_frame
# to override the default striding of 4 we specified above
job = Job(
op_args={
frame: db.table('example').column('frame'),
# The "strided" sampling mode will run over every 8th frame,
# i.e. frames [0, 8, 16, ...]
strided_frame: 8,
output_op: 'example_hist_strided'
})
output_tables = db.run(output_op, [job], force=True)
# Loop over the column's rows. Each row is a tuple of the frame number and
# value for that row.
video_hists = output_tables[0].column('hist').load(readers.histograms)
num_rows = 0
for frame_hists in video_hists:
assert len(frame_hists) == 3
assert frame_hists[0].shape[0] == 16
num_rows += 1
assert num_rows == round(db.table('example').num_rows() / 8)
def main(movie_path):
total_start = time.time()
print('Detecting shots in movie {}'.format(movie_path))
movie_name = os.path.basename(movie_path)
# Use GPU kernels if we have a GPU
db = Database()
print('Loading movie into Scanner database...')
s = time.time()
if db.has_gpu():
device = DeviceType.GPU
else:
device = DeviceType.CPU
############ ############ ############ ############
# 0. Ingest the video into the database
############ ############ ############ ############
[movie_table], _ = db.ingest_videos([(movie_name, movie_path)], force=True)
print('Time: {:.1f}s'.format(time.time() - s))
print('Number of frames in movie: {:d}'.format(movie_table.num_rows()))
s = time.time()
############ ############ ############ ############
# 1. Run Histogram over the entire video in Scanner
############ ############ ############ ############
print('Computing a color histogram for each frame...')
frame = db.sources.FrameColumn()
histogram = db.ops.Histogram(frame=frame, device=device)
output = db.sinks.Column(columns={'histogram': histogram})
job = Job(op_args={
frame: movie_table.column('frame'),
output: movie_name + '_hist'
})
[hists_table] = db.run(output=output, jobs=[job], force=True)
print('\nTime: {:.1f}s, {:.1f} fps'.format(
time.time() - s,
movie_table.num_rows() / (time.time() - s)))
s = time.time()
############ ############ ############ ############
# 2. Load histograms and compute shot boundaries
# in python
############ ############ ############ ############
print('Computing shot boundaries...')
# Read histograms from disk
hists = [
h for h in hists_table.column('histogram').load(readers.histograms)
]
boundaries = compute_shot_boundaries(hists)
print('Found {:d} shots.'.format(len(boundaries)))
print('Time: {:.1f}s'.format(time.time() - s))
s = time.time()
############ ############ ############ ############
# 3. Create montage in Scanner
############ ############ ############ ############
print('Creating shot montage...')
row_length = min(16, len(boundaries))
rows_per_item = 1
target_width = 256
# Compute partial row montages that we will stack together
# at the end
item_size = row_length * rows_per_item
starts_remainder = len(boundaries) % item_size
evenly_divisible = (starts_remainder == 0)
if not evenly_divisible:
boundaries = boundaries[0:len(boundaries) - starts_remainder]
frame = db.sources.FrameColumn()
gather_frame = db.streams.Gather(frame, boundaries)
sliced_frame = db.streams.Slice(gather_frame,
partitioner=db.partitioner.all(item_size))
montage = db.ops.Montage(frame=sliced_frame,
num_frames=row_length * rows_per_item,
target_width=target_width,
frames_per_row=row_length,
device=device)
sampled_montage = db.streams.Gather(montage)
output = db.sinks.Column(
columns={'montage': db.streams.Unslice(sampled_montage).lossless()})
job = Job(
op_args={
frame:
movie_table.column('frame'),
sampled_montage: [[item_size - 1]
for _ in range(len(boundaries) // item_size)],
output:
'montage_image'
})
[montage_table] = db.run(output=output, jobs=[job], force=True)
# Stack all partial montages together
montage_img = np.zeros((1, target_width * row_length, 3), dtype=np.uint8)
for img in montage_table.column('montage').load():
img = np.flip(img, 2)
montage_img = np.vstack((montage_img, img))
print('')
print('Time: {:.1f}s'.format(time.time() - s))
############ ############ ############ ############
# 4. Write montage to disk
############ ############ ############ ############
cv2.imwrite('shots.jpg', montage_img)
print('Successfully generated shots.jpg')
print('Total time: {:.2f} s'.format(time.time() - total_start))
frame, frame_info = input_table.as_op().all(item_size=50)
cpm2_input = db.ops.CPM2Input(frame=frame,
frame_info=frame_info,
args=cpm2_args,
device=DeviceType.GPU)
cpm2_resized_map, cpm2_joints = db.ops.CPM2(cpm2_input=cpm2_input,
frame_info=frame_info,
args=cpm2_args,
device=DeviceType.GPU)
poses = db.ops.CPM2Output(cpm2_resized_map=cpm2_resized_map,
cpm2_joints=cpm2_joints,
frame_info=frame_info,
args=cpm2_args)
job = Job(columns=[poses], name='example_poses')
output = db.run(job, True)
print('Extracting frames...')
video_poses = [
pose for (_, pose) in output.columns('poses').load(parsers.poses)
]
video_frames = [f[0] for _, f in db.table('example').load(['frame'])]
print('Writing output video...')
frame_shape = video_frames[0].shape
output = cv2.VideoWriter('example_poses.mkv',
cv2.VideoWriter_fourcc(*'X264'), 24.0,
(frame_shape[1], frame_shape[0]))
for (frame, frame_poses) in zip(video_frames, video_poses):
