def load(self, fn=None, rows=None):
"""
Loads the results of a Scanner computation into Python.
Kwargs:
fn: Optional function to apply to the binary blobs as they are read
in.
Returns:
Generator that yields either a numpy array for frame columns or
a binary blob for non-frame columns (optionally processed by the
`fn`).
"""
self._load_meta()
# If the column is a video, then dump the requested frames to disk as
# PNGs and return the decoded PNGs
if (self._descriptor.type == self._db.protobufs.Video
and self._video_descriptor.codec_type
== self._db.protobufs.VideoDescriptor.H264):
png_table_name = self._db._png_dump_prefix.format(
self._table.name())
if self._db.has_table(png_table_name):
png_table = self._db.table(png_table_name)
if rows is None and \
png_table.num_rows() == self._table.num_rows() and \
png_table._descriptor.timestamp > \
self._table._descriptor.timestamp:
return png_table.load(['img'], parsers.image)
pair = [(self._table.name(), png_table_name)]
op_args = {}
frame = self._db.ops.FrameInput()
op_args[frame] = self
enc_input = frame
if rows is not None:
sampled_frame = frame.sample()
op_args[sampled_frame] = self._db.sampler.gather(rows)
enc_input = sampled_frame
img = self._db.ops.ImageEncoder(frame=enc_input)
output_op = self._db.ops.Output(columns=[img])
op_args[output_op] = png_table_name
job = Job(op_args=op_args)
bulk_job = BulkJob(output=output_op, jobs=[job])
[out_tbl] = self._db.run(bulk_job, force=True, show_progress=False)
return out_tbl.load(['img'], parsers.image)
elif self._descriptor.type == self._db.protobufs.Video:
frame_type = self._video_descriptor.frame_type
if frame_type == self._db.protobufs.U8:
dtype = np.uint8
elif frame_type == self._db.protobufs.F32:
dtype = np.float32
elif frame_type == self._db.protobufs.F64:
dtype = np.float64
parser_fn = parsers.raw_frame_gen(self._video_descriptor.height,
self._video_descriptor.width,
self._video_descriptor.channels,
dtype)
return self._load(fn=parser_fn, rows=rows)
else:
return self._load(fn, rows=rows)
def load(self, fn=None, rows=None, workers=16):
"""
Loads the results of a Scanner computation into Python.
Kwargs:
fn: Optional function to apply to the binary blobs as they are read
in.
Returns:
Generator that yields either a numpy array for frame columns or
a binary blob for non-frame columns (optionally processed by the
`fn`).
"""
self._load_meta()
# If the column is a video, then dump the requested frames to disk as
# PNGs and return the decoded PNGs
if (self._descriptor.type == self._db.protobufs.Video
and self._video_descriptor.codec_type
== self._db.protobufs.VideoDescriptor.H264):
png_table_name = self._db._png_dump_prefix.format(
self._table.name(), self._name)
pair = [(self._table.name(), png_table_name)]
op_args = {}
frame = self._db.sources.FrameColumn()
op_args[frame] = self
enc_input = frame
if rows is not None:
sampled_frame = self._db.streams.Gather(frame, rows=rows)
enc_input = sampled_frame
img = self._db.ops.ImageEncoder(frame=enc_input)
output_op = self._db.sinks.Column(columns={'img': img})
op_args[output_op] = png_table_name
job = Job(op_args=op_args)
[out_tbl] = self._db.run(output_op, [job],
force=True,
show_progress=False)
return out_tbl.column('img').load(readers.image)
elif self._descriptor.type == self._db.protobufs.Video:
frame_type = self._video_descriptor.frame_type
if frame_type == self._db.protobufs.U8:
dtype = np.uint8
elif frame_type == self._db.protobufs.F32:
dtype = np.float32
elif frame_type == self._db.protobufs.F64:
dtype = np.float64
parser_fn = readers.raw_frame_gen(self._video_descriptor.height,
self._video_descriptor.width,
self._video_descriptor.channels,
dtype)
return self._load(fn=parser_fn, rows=rows, workers=workers)
else:
return self._load(fn, rows=rows, workers=workers)