def test_generator_processor_threads(tmpdir, bees_video, filelists_path,
pipeline_config):
repo = Repository(str(tmpdir))
pipelines = [
Pipeline([Image, Timestamp], [PipelineResult], **pipeline_config)
for _ in range(3)
]
gen_processor = GeneratorProcessor(pipelines,
lambda: BBBinaryRepoSink(repo, camId=0))
gen = video_generator(bees_video,
ts_format="2015",
path_filelists=filelists_path)
gen_processor(gen)
fnames = list(repo.iter_fnames())
assert len(fnames) == 1
num_frames = 0
for fname in repo.iter_fnames():
with open(fname, "rb") as f:
fc = FrameContainer.read(f)
num_frames += len(list(fc.frames))
assert num_frames == 3
def test_generator_processor_video(tmpdir, bees_video, filelists_path,
pipeline_config):
repo = Repository(str(tmpdir))
pipeline = Pipeline([Image, Timestamp], [PipelineResult],
**pipeline_config)
gen_processor = GeneratorProcessor(pipeline,
lambda: BBBinaryRepoSink(repo, camId=0))
gen = video_generator(bees_video,
ts_format="2015",
path_filelists=filelists_path)
gen_processor(gen)
fnames = list(repo.iter_fnames())
assert len(fnames) == 1
last_ts = 0
num_frames = 0
for fname in repo.iter_fnames():
print("{}: {}".format(fname, os.path.getsize(fname)))
with open(fname, "rb") as f:
fc = FrameContainer.read(f)
num_frames += len(list(fc.frames))
assert fc.dataSources[0].filename == os.path.basename(bees_video)
assert last_ts < fc.fromTimestamp
last_ts = fc.fromTimestamp
assert num_frames == 3
def test_video_generator_2015(bees_video, filelists_path):
gen = video_generator(bees_video, ts_format="2015", path_filelists=filelists_path)
results = list(gen)
assert len(results) == 3
prev_ts = 0.0
for _, _, ts in results:
assert ts > prev_ts
prev_ts = ts
def test_video_generator_2016(bees_video_2016):
gen = video_generator(bees_video_2016, ts_format="2016", path_filelists=None)
results = list(gen)
assert len(results) == 4
prev_ts = 0.0
for _, _, ts in results:
assert ts > prev_ts
prev_ts = ts
def test_video_generator_2015(bees_video, filelists_path):
gen = video_generator(bees_video,
ts_format='2015',
path_filelists=filelists_path)
results = list(gen)
assert (len(results) == 3)
prev_ts = 0.
for _, _, ts in results:
assert (ts > prev_ts)
prev_ts = ts
def test_video_generator_2016(bees_video_2016):
gen = video_generator(bees_video_2016,
ts_format='2016',
path_filelists=None)
results = list(gen)
assert (len(results) == 4)
prev_ts = 0.
for _, _, ts in results:
assert (ts > prev_ts)
prev_ts = ts
def process_video(args):
config = get_auto_config()
logger.info('Initializing {} pipeline(s)'.format(args.num_threads))
plines = [Pipeline([Image, Timestamp], [PipelineResult], **config)
for _ in range(args.num_threads)]
logger.info('Loading bb_binary repository {}'.format(args.repo_output_path))
repo = Repository(args.repo_output_path)
camId, _, _ = parse_video_fname(args.video_path)
logger.info('Parsed camId = {}'.format(camId))
gen_processor = GeneratorProcessor(plines, lambda: BBBinaryRepoSink(repo, camId=camId))
logger.info('Processing video frames from {}'.format(args.video_path))
gen_processor(video_generator(args.video_path, args.timestamp_format, args.text_root_path))
def test_generator_processor_threads(tmpdir, bees_video, filelists_path, pipeline_config):
repo = Repository(str(tmpdir))
pipelines = [Pipeline([Image, Timestamp], [PipelineResult], **pipeline_config) for
_ in range(3)]
gen_processor = GeneratorProcessor(
pipelines, lambda: BBBinaryRepoSink(repo, camId=0))
gen = video_generator(bees_video, ts_format='2015', path_filelists=filelists_path)
gen_processor(gen)
fnames = list(repo.iter_fnames())
assert len(fnames) == 1
num_frames = 0
for fname in repo.iter_fnames():
with open(fname, 'rb') as f:
fc = FrameContainer.read(f)
num_frames += len(list(fc.frames))
assert(num_frames == 3)
def process_video(video_path, repo_output_path, ts_format, text_root_path,
rank):
info = lambda msg: logger.info('Process {}: {}'.format(rank, msg))
import theano
from pipeline import Pipeline
from pipeline.cmdline import logger
from pipeline.pipeline import GeneratorProcessor, get_auto_config
from pipeline.io import BBBinaryRepoSink, video_generator
from pipeline.objects import PipelineResult, Image, Timestamp
from bb_binary import Repository, parse_video_fname
repo_output_path = os.path.join(repo_output_path,
'process_{}'.format(rank))
info('Theano compile dir: {}'.format(theano.config.base_compiledir))
info('Output dir: {}'.format(repo_output_path))
config = get_auto_config()
info('Initializing pipeline')
pipeline = Pipeline([Image, Timestamp], [PipelineResult], **config)
info('Loading bb_binary repository {}'.format(repo_output_path))
repo = Repository(repo_output_path)
camId, _, _ = parse_video_fname(video_path)
info('Parsed camId = {}'.format(camId))
gen_processor = GeneratorProcessor(
pipeline, lambda: BBBinaryRepoSink(repo, camId=camId))
log_callback = lambda frame_idx: info('Processing frame {} from {}'.format(
frame_idx, video_path))
ffmpeg_stderr_fd = open('process_{}_ffmpeg_stderr.log'.format(rank), 'w')
info('Processing video frames from {}'.format(video_path))
gen_processor(
video_generator(video_path, ts_format, text_root_path, log_callback,
ffmpeg_stderr_fd))
def process_video(args):
config = get_auto_config()
logger.info('Initializing {} pipeline(s)'.format(args.num_threads))
plines = [
Pipeline([Image, Timestamp], [PipelineResult], **config)
for _ in range(args.num_threads)
]
logger.info('Loading bb_binary repository {}'.format(
args.repo_output_path))
repo = Repository(args.repo_output_path)
camId, _, _ = parse_video_fname(args.video_path)
logger.info('Parsed camId = {}'.format(camId))
gen_processor = GeneratorProcessor(
plines, lambda: BBBinaryRepoSink(repo, camId=camId))
logger.info('Processing video frames from {}'.format(args.video_path))
gen_processor(
video_generator(args.video_path, args.timestamp_format,
args.text_root_path))
def process_video(video_path, repo_output_path, ts_format, text_root_path,
rank):
info = lambda msg: logger.info(f"Process {rank}: {msg}")
import theano
from pipeline import Pipeline
from pipeline.cmdline import logger
from pipeline.pipeline import GeneratorProcessor, get_auto_config
from pipeline.io import BBBinaryRepoSink, video_generator
from pipeline.objects import PipelineResult, Image, Timestamp
from bb_binary import Repository, parse_video_fname
repo_output_path = os.path.join(repo_output_path, f"process_{rank}")
info(f"Theano compile dir: {theano.config.base_compiledir}")
info(f"Output dir: {repo_output_path}")
config = get_auto_config()
info("Initializing pipeline")
pipeline = Pipeline([Image, Timestamp], [PipelineResult], **config)
info(f"Loading bb_binary repository {repo_output_path}")
repo = Repository(repo_output_path)
camId, _, _ = parse_video_fname(video_path)
info(f"Parsed camId = {camId}")
gen_processor = GeneratorProcessor(
pipeline, lambda: BBBinaryRepoSink(repo, camId=camId))
log_callback = lambda frame_idx: info(
f"Processing frame {frame_idx} from {video_path}")
ffmpeg_stderr_fd = open(f"process_{rank}_ffmpeg_stderr.log", "w")
info(f"Processing video frames from {video_path}")
gen_processor(
video_generator(video_path, ts_format, text_root_path, log_callback,
ffmpeg_stderr_fd))
def test_generator_processor_video(tmpdir, bees_video, filelists_path, pipeline_config):
repo = Repository(str(tmpdir))
pipeline = Pipeline([Image, Timestamp], [PipelineResult], **pipeline_config)
gen_processor = GeneratorProcessor(
pipeline, lambda: BBBinaryRepoSink(repo, camId=0))
gen = video_generator(bees_video, ts_format='2015', path_filelists=filelists_path)
gen_processor(gen)
fnames = list(repo.iter_fnames())
assert len(fnames) == 1
last_ts = 0
num_frames = 0
for fname in repo.iter_fnames():
print("{}: {}".format(fname, os.path.getsize(fname)))
with open(fname, 'rb') as f:
fc = FrameContainer.read(f)
num_frames += len(list(fc.frames))
assert fc.dataSources[0].filename == os.path.basename(bees_video)
assert last_ts < fc.fromTimestamp
last_ts = fc.fromTimestamp
assert(num_frames == 3)
def process_video(video_path, repo_output_path, ts_format, text_root_path, rank):
info = lambda msg: logger.info('Process {}: {}'.format(rank, msg))
import theano
from pipeline import Pipeline
from pipeline.cmdline import logger
from pipeline.pipeline import GeneratorProcessor, get_auto_config
from pipeline.io import BBBinaryRepoSink, video_generator
from pipeline.objects import PipelineResult, Image, Timestamp
from bb_binary import Repository, parse_video_fname
repo_output_path = os.path.join(repo_output_path, 'process_{}'.format(rank))
info('Theano compile dir: {}'.format(theano.config.base_compiledir))
info('Output dir: {}'.format(repo_output_path))
config = get_auto_config()
info('Initializing pipeline')
pipeline = Pipeline([Image, Timestamp], [PipelineResult], **config)
info('Loading bb_binary repository {}'.format(repo_output_path))
repo = Repository(repo_output_path)
camId, _, _ = parse_video_fname(video_path)
info('Parsed camId = {}'.format(camId))
gen_processor = GeneratorProcessor(pipeline,
lambda: BBBinaryRepoSink(repo, camId=camId))
log_callback = lambda frame_idx: info('Processing frame {} from {}'.format(frame_idx,
video_path))
ffmpeg_stderr_fd = open('process_{}_ffmpeg_stderr.log'.format(rank), 'w')
info('Processing video frames from {}'.format(video_path))
gen_processor(video_generator(video_path, ts_format, text_root_path,
log_callback, ffmpeg_stderr_fd))