def test_generator_processor(tmpdir, bees_image, pipeline_config):
def image_generator():
ts = time.time()
data_source = DataSource.new_message(filename='bees.jpeg')
for i in range(2):
img = imread(bees_image)
yield data_source, img, ts + i
repo = Repository(str(tmpdir))
pipeline = Pipeline([Image, Timestamp], [PipelineResult], **pipeline_config)
gen_processor = GeneratorProcessor(
pipeline, lambda: BBBinaryRepoSink(repo, camId=2))
gen_processor(image_generator())
gen_processor(image_generator())
fnames = list(repo.iter_fnames())
assert len(fnames) == 2
last_ts = 0
for fname in repo.iter_fnames():
print("{}: {}".format(fname, os.path.getsize(fname)))
with open(fname, 'rb') as f:
fc = FrameContainer.read(f)
assert fc.dataSources[0].filename == 'bees.jpeg'
assert last_ts < fc.fromTimestamp
last_ts = fc.fromTimestamp
def test_generator_processor_threads(tmpdir, bees_video, filelists_path,
pipeline_config):
repo = Repository(str(tmpdir))
pipelines = [
Pipeline([Image, Timestamp], [PipelineResult], **pipeline_config)
for _ in range(3)
]
gen_processor = GeneratorProcessor(pipelines,
lambda: BBBinaryRepoSink(repo, camId=0))
gen = video_generator(bees_video,
ts_format="2015",
path_filelists=filelists_path)
gen_processor(gen)
fnames = list(repo.iter_fnames())
assert len(fnames) == 1
num_frames = 0
for fname in repo.iter_fnames():
with open(fname, "rb") as f:
fc = FrameContainer.read(f)
num_frames += len(list(fc.frames))
assert num_frames == 3
def test_generator_processor_video(tmpdir, bees_video, filelists_path,
pipeline_config):
repo = Repository(str(tmpdir))
pipeline = Pipeline([Image, Timestamp], [PipelineResult],
**pipeline_config)
gen_processor = GeneratorProcessor(pipeline,
lambda: BBBinaryRepoSink(repo, camId=0))
gen = video_generator(bees_video,
ts_format="2015",
path_filelists=filelists_path)
gen_processor(gen)
fnames = list(repo.iter_fnames())
assert len(fnames) == 1
last_ts = 0
num_frames = 0
for fname in repo.iter_fnames():
print("{}: {}".format(fname, os.path.getsize(fname)))
with open(fname, "rb") as f:
fc = FrameContainer.read(f)
num_frames += len(list(fc.frames))
assert fc.dataSources[0].filename == os.path.basename(bees_video)
assert last_ts < fc.fromTimestamp
last_ts = fc.fromTimestamp
assert num_frames == 3
def process_video(video_path, repo_output_path, ts_format, text_root_path,
rank):
info = lambda msg: logger.info('Process {}: {}'.format(rank, msg))
import theano
from pipeline import Pipeline
from pipeline.cmdline import logger
from pipeline.pipeline import GeneratorProcessor, get_auto_config
from pipeline.io import BBBinaryRepoSink, video_generator
from pipeline.objects import PipelineResult, Image, Timestamp
from bb_binary import Repository, parse_video_fname
repo_output_path = os.path.join(repo_output_path,
'process_{}'.format(rank))
info('Theano compile dir: {}'.format(theano.config.base_compiledir))
info('Output dir: {}'.format(repo_output_path))
config = get_auto_config()
info('Initializing pipeline')
pipeline = Pipeline([Image, Timestamp], [PipelineResult], **config)
info('Loading bb_binary repository {}'.format(repo_output_path))
repo = Repository(repo_output_path)
camId, _, _ = parse_video_fname(video_path)
info('Parsed camId = {}'.format(camId))
gen_processor = GeneratorProcessor(
pipeline, lambda: BBBinaryRepoSink(repo, camId=camId))
log_callback = lambda frame_idx: info('Processing frame {} from {}'.format(
frame_idx, video_path))
ffmpeg_stderr_fd = open('process_{}_ffmpeg_stderr.log'.format(rank), 'w')
info('Processing video frames from {}'.format(video_path))
gen_processor(
video_generator(video_path, ts_format, text_root_path, log_callback,
ffmpeg_stderr_fd))
def process_video(args):
config = get_auto_config()
logger.info('Initializing {} pipeline(s)'.format(args.num_threads))
plines = [
Pipeline([Image, Timestamp], [PipelineResult], **config)
for _ in range(args.num_threads)
]
logger.info('Loading bb_binary repository {}'.format(
args.repo_output_path))
repo = Repository(args.repo_output_path)
camId, _, _ = parse_video_fname(args.video_path)
logger.info('Parsed camId = {}'.format(camId))
gen_processor = GeneratorProcessor(
plines, lambda: BBBinaryRepoSink(repo, camId=camId))
logger.info('Processing video frames from {}'.format(args.video_path))
gen_processor(
video_generator(args.video_path, args.timestamp_format,
args.text_root_path))
def process_video(video_path, repo_output_path, ts_format, text_root_path,
rank):
info = lambda msg: logger.info(f"Process {rank}: {msg}")
import theano
from pipeline import Pipeline
from pipeline.cmdline import logger
from pipeline.pipeline import GeneratorProcessor, get_auto_config
from pipeline.io import BBBinaryRepoSink, video_generator
from pipeline.objects import PipelineResult, Image, Timestamp
from bb_binary import Repository, parse_video_fname
repo_output_path = os.path.join(repo_output_path, f"process_{rank}")
info(f"Theano compile dir: {theano.config.base_compiledir}")
info(f"Output dir: {repo_output_path}")
config = get_auto_config()
info("Initializing pipeline")
pipeline = Pipeline([Image, Timestamp], [PipelineResult], **config)
info(f"Loading bb_binary repository {repo_output_path}")
repo = Repository(repo_output_path)
camId, _, _ = parse_video_fname(video_path)
info(f"Parsed camId = {camId}")
gen_processor = GeneratorProcessor(
pipeline, lambda: BBBinaryRepoSink(repo, camId=camId))
log_callback = lambda frame_idx: info(
f"Processing frame {frame_idx} from {video_path}")
ffmpeg_stderr_fd = open(f"process_{rank}_ffmpeg_stderr.log", "w")
info(f"Processing video frames from {video_path}")
gen_processor(
video_generator(video_path, ts_format, text_root_path, log_callback,
ffmpeg_stderr_fd))