def verify(ephemeral, resource, informat, force_delete, rm_script):
ephemeral_ts = TimeStream(ephemeral, format=informat)
resource_ts = TimeStream(resource, format=informat)
to_delete = []
resource_images = resource_ts.instants
try:
for image in tqdm(ephemeral_ts, unit=" files"):
try:
res_img = resource_images[image.instant]
if image.md5sum == res_img.md5sum:
if not isinstance(image.fetcher, FileContentFetcher):
click.echo(
f"WARNING: can't delete {image.filename} as it is bundled",
err=True)
to_delete.append(image.fetcher.path)
res_img.clear_content()
except KeyError:
continue
except Exception as exc:
click.echo(
f"WARNING: error in resources lookup of {image.filename}: {str(exc)}",
err=True)
finally:
if rm_script is not None:
with open(rm_script, "w") as fh:
for f in to_delete:
print("rm -fv", realpath(f), file=fh)
else:
click.echo("will delete the following files:")
for f in to_delete:
click.echo("\t{}".format(f))
if force_delete or click.confirm("Is that OK?"):
for f in to_delete:
os.unlink(f)
def cp(informat, bundle, input, output, start_time, start_date, end_time,
end_date, interval):
tfilter = TimeFilter(start_date, end_date, start_time, end_time)
if interval is not None:
raise NotImplementedError("haven't done interval restriction yet")
output = TimeStream(output, bundle_level=bundle)
for image in tqdm(TimeStream(input, format=informat, timefilter=tfilter)):
with CatchSignalThenExit():
output.write(image)
def bundle(force, informat, bundle, input, output):
input = TimeStream(input, format=informat)
if os.path.exists(output) and not force:
click.echo(f"ERROR: output exists: {output}", err=True)
sys.exit(1)
output = TimeStream(output, bundle_level=bundle)
for image in input:
with CatchSignalThenExit():
output.write(image)
click.echo(f"Processing {image}")
def video(input, output, ffmpeg_path, ffmpeg_args, framerate, scaling, ncpus,
informat, segmented_by):
"""Creates a standard timelapse video from timestream"""
from pyts2.pipeline.video import VideoEncoder, ImageWatermarker, SegementedVideoEncoder
ints = TimeStream(input, format=informat)
pipe = TSPipeline(
DecodeImageFileStep(),
ImageWatermarker(),
EncodeImageFileStep(format="jpg"),
)
if segmented_by is None:
pipe.add_step(
VideoEncoder(output,
ffmpeg_args=ffmpeg_args,
ffmpeg_path=ffmpeg_path,
rate=framerate,
threads=ncpus,
scaling=scaling), )
else:
pipe.add_step(
SegementedVideoEncoder(output,
segmented_by=segmented_by,
ffmpeg_args=ffmpeg_args,
ffmpeg_path=ffmpeg_path,
rate=framerate,
threads=ncpus,
scaling=scaling), )
try:
for image in pipe.process(ints, ncpus=1):
pass
finally:
pipe.finish()
sys.exit(pipe.retcode)
def downsize(input, output, ncpus, informat, outformat, size, bundle, mode):
if mode == "resize":
downsizer = ResizeImageStep(geom=size)
elif mode == "centrecrop" or mode == "crop":
downsizer = CropCentreStep(geom=size)
pipe = TSPipeline(
DecodeImageFileStep(),
downsizer,
EncodeImageFileStep(format=outformat),
)
ints = TimeStream(input, format=informat)
outts = TimeStream(output, format=outformat, bundle_level=bundle)
try:
pipe.process_to(ints, outts, ncpus=ncpus)
finally:
click.echo(
f"{mode} {input}:{informat} to {output}:{outformat}, found {pipe.n} files"
)
def ingest(input, informat, output, bundle, ncpus, downsized_output,
downsized_size, downsized_bundle, audit_output):
ints = TimeStream(input, format=informat)
outts = TimeStream(output, bundle_level=bundle)
steps = [WriteFileStep(outts)]
# if downsized_output is not None or audit_output is not None:
# steps.append(DecodeImageFileStep())
if audit_output is not None:
audit_pipe = TSPipeline(
FileStatsStep(),
CalculateEVStep(),
DecodeImageFileStep(),
ImageMeanColourStep(),
ScanQRCodesStep(),
)
steps.append(audit_pipe)
if downsized_output is not None:
downsized_ts = TimeStream(downsized_output,
bundle_level=downsized_bundle,
add_subsecond_field=True)
downsize_pipeline = TSPipeline(
DecodeImageFileStep(),
ResizeImageStep(geom=downsized_size),
EncodeImageFileStep(format="jpg"),
WriteFileStep(downsized_ts),
)
steps.append(downsize_pipeline)
pipe = TSPipeline(*steps)
try:
for image in pipe.process(ints, ncpus=ncpus):
pass
finally:
pipe.finish()
if audit_output is not None:
pipe.report.save(audit_output)
ifmt = f":{informat}" if informat is not None else ""
click.echo(f"Ingested {input}{ifmt} to {output}, found {pipe.n} files")
sys.exit(pipe.retcode)
def audit(output, input, ncpus=1, informat=None):
pipe = TSPipeline(
FileStatsStep(),
DecodeImageFileStep(),
ImageMeanColourStep(),
ScanQRCodesStep(),
)
ints = TimeStream(input, format=informat)
try:
for image in pipe.process(ints, ncpus=ncpus):
if pipe.n % 1000 == 0:
pipe.report.save(output)
finally:
pipe.report.save(output)
click.echo(f"Audited {input}:{informat}, found {pipe.n} files")
def test_roundtrip_v1_to_v2(tmpdir):
outpath = tmpdir.join("test.ts2")
out = TSv2Stream(outpath, "w")
for i, image in enumerate(TimeStream("testdata/timestreams/flat/")):
out.write(image)
out.close()
stream = TSv2Stream(outpath, "r")
for image in stream:
# Instant
expect_inst = TSInstant(SMALL_TIMESTREAMS["expect_times"][i],
subsecond=0,
index=None)
assert isinstance(image.instant, TSInstant)
if stream.sorted:
assert image.instant == expect_inst
else:
assert image.instant.datetime in SMALL_TIMESTREAMS["expect_times"]
assert image.instant.subsecond == 0
assert image.instant.index is None
def audit(input,
output,
telegraf_host,
telegraf_port,
telegraf_metric,
ncpus=1,
informat=None):
from pyts2.pipeline.telegraf import TelegrafRecordStep
if output is None and telegraf_host is None:
print("ERROR: must give one of --output or --telegraf-host")
sys.exit(1)
pipe = TSPipeline(
FileStatsStep(),
CalculateEVStep(),
DecodeImageFileStep(),
ImageMeanColourStep(),
ScanQRCodesStep(),
)
if telegraf_host is not None:
pipe.add_step(
TelegrafRecordStep(
metric_name=telegraf_metric,
telegraf_host=telegraf_host,
telegraf_port=telegraf_port,
))
ints = TimeStream(input, format=informat)
try:
for image in pipe.process(ints, ncpus=ncpus):
if output is not None:
if pipe.n % 1000 == 0:
pipe.report.save(output)
finally:
if output is not None:
pipe.report.save(output)
fmt = "" if informat is None else f":{informat}"
click.echo(f"Audited {input}{fmt}, found {pipe.n} files")
def ls(informat, input, start_time, start_date, end_time, end_date, interval):
tfilter = TimeFilter(start_date, end_date, start_time, end_time)
if interval is not None:
raise NotImplementedError("haven't done interval restriction yet")
for image in TimeStream(input, format=informat, timefilter=tfilter):
print(image.instant)
def gvmosaic(input, informat, dims, order, audit_output, composite_bundling,
composite_format, composite_size, composite_output,
composite_centrecrop, bundle_output, bundle_level, recoded_output,
recoded_format, recoded_bundling, rm_script, mv_destination,
truncate_time):
from pyts2.pipeline.gigavision import GigavisionMosaicStep
ints = TimeStream(input, format=informat)
composite_ts = TimeStream(composite_output,
bundle_level=composite_bundling,
add_subsecond_field=True)
steps = []
if truncate_time is not None:
steps.append(TruncateTimeStep(truncate_time))
if bundle_output is not None:
verbatim_ts = TimeStream(bundle_output, bundle_level=bundle_level)
steps.append(WriteFileStep(verbatim_ts))
# decode image
steps.append(DecodeImageFileStep())
# run audit pipeline
if audit_output is not None:
audit_pipe = TSPipeline(
FileStatsStep(),
ImageMeanColourStep(),
ScanQRCodesStep(),
)
steps.append(audit_pipe)
if recoded_output is not None:
# run recode pipeline
recoded_ts = TimeStream(recoded_output, bundle_level=recoded_bundling)
recoded_pipe = TSPipeline(
EncodeImageFileStep(format=recoded_format),
WriteFileStep(recoded_ts),
)
steps.append(TeeStep(recoded_pipe))
# do mosaicing
steps.append(
TSPipeline(
GigavisionMosaicStep(
dims,
composite_ts,
subimgres=composite_size,
order=order,
output_format=composite_format,
centrecrop=composite_centrecrop,
rm_script=rm_script,
mv_destination=mv_destination,
)))
# assemble total pipeline
pipe = TSPipeline(*steps)
# run pipeline
try:
for image in pipe.process(ints):
pass
finally:
pipe.finish()
if audit_output is not None:
pipe.report.save(audit_output)
sys.exit(pipe.retcode)
def verify(ephemerals, resource, informat, force_delete, rm_script, move_dest,
pixel_distance, distance_file, only_check_exists):
"""
Verify images from each of EPHEMERAL, ensuring images are in --resources.
"""
resource_ts = TimeStream(resource, format=informat)
decoder = DecodeImageFileStep()
resource_ts.index()
if rm_script is not None:
with open(rm_script, "w") as fh:
print("# rmscript for", *ephemerals, file=fh)
with Removalist(rm_script=rm_script, mv_dest=move_dest,
force=force_delete) as rmer:
if distance_file is not None:
distance_file = open(distance_file, "w")
print("ephemeral_image\tresource_image\tdistance",
file=distance_file)
for ephemeral in ephemerals:
click.echo(f"Crawling ephemeral timestream: {ephemeral}")
ephemeral_ts = TimeStream(ephemeral, format=informat)
try:
for image in tqdm(ephemeral_ts, unit=" files"):
try:
res_img = resource_ts.getinstant(image.instant)
if not isinstance(image.fetcher, FileContentFetcher):
click.echo(
f"WARNING: can't delete {image.filename} as it is bundled",
err=True)
continue
if only_check_exists:
rmer.remove(image.fetcher.pathondisk)
elif pixel_distance is not None:
eimg = decoder.process_file(image)
rimg = decoder.process_file(res_img)
if eimg.pixels.shape != rimg.pixels.shape:
if distance_file is not None:
print(basename(image.filename),
basename(res_img.filename),
"NA",
file=distance_file)
continue
dist = np.mean(abs(eimg.pixels - rimg.pixels))
if distance_file is not None:
print(basename(image.filename),
basename(res_img.filename),
dist,
file=distance_file)
if dist < pixel_distance:
rmer.remove(realpath(image.fetcher.pathondisk))
elif image.md5sum == res_img.md5sum:
rmer.remove(realpath(image.fetcher.pathondisk))
except KeyError:
tqdm.write(f"{image.instant} not in {resource}")
if distance_file is not None:
print(basename(image.filename),
"",
"",
file=distance_file)
except Exception as exc:
click.echo(
f"WARNING: error in resources lookup of {image.filename}: {str(exc)}",
err=True)
if stderr.isatty():
traceback.print_exc(file=stderr)
except KeyboardInterrupt:
print("\n\nExiting cleanly", file=stderr)
break
if distance_file is not None:
distance_file.close()
def liveingest(
input,
informat,
output,
bundle,
inotify_watch,
nuke,
min_mean_luminance,
truncate_time,
downsized_output,
downsized_size,
downsized_bundle,
recoded_output,
recoded_format,
recoded_bundle,
centrecropped_output,
centrecropped_size,
centrecropped_bundle,
telegraf_host,
telegraf_port,
telegraf_metric,
telegraf_additional_tags,
):
from pyts2.pipeline.telegraf import TelegrafRecordStep
ifmt = f"{informat}s" if informat is not None else "images"
click.echo(f"Begin live ingest of {ifmt} to {output}...")
if telegraf_additional_tags is not None:
telegraf_additional_tags = json.loads(telegraf_additional_tags)
else:
telegraf_additional_tags = {}
ints = TimeStream(format=informat)
outts = TimeStream(output, bundle_level=bundle)
pipe = TSPipeline()
pipe.add_step(FileStatsStep())
pipe.add_step(DecodeImageFileStep())
pipe.add_step(ImageMeanColourStep())
pipe.add_step(CalculateEVStep())
pipe.add_step(ScanQRCodesStep())
pipe.add_step(
TelegrafRecordStep(
metric_name=telegraf_metric,
telegraf_host=telegraf_host,
telegraf_port=telegraf_port,
tags=telegraf_additional_tags,
))
if downsized_output is not None:
downsized_ts = TimeStream(downsized_output,
bundle_level=downsized_bundle,
add_subsecond_field=True)
downsize_pipeline = TSPipeline(
ResizeImageStep(geom=downsized_size),
EncodeImageFileStep(format="jpg"),
WriteFileStep(downsized_ts),
)
pipe.add_step(TeeStep(downsize_pipeline))
if centrecropped_output is not None:
centrecropped_ts = TimeStream(centrecropped_output,
bundle_level=centrecropped_bundle,
add_subsecond_field=True)
centrecrop_pipeline = TSPipeline(
DecodeImageFileStep(),
CropCentreStep(geom=centrecropped_size),
EncodeImageFileStep(format="jpg"),
WriteFileStep(centrecropped_ts),
)
pipe.add_step(TeeStep(centrecrop_pipeline))
if min_mean_luminance is not None:
pipe.add_step(
FilterStep(
callback=lambda x: x.report["ImageMean_L"
] > min_mean_luminance,
message=
"Image has low luminance, probable nighttime image. Skipping.")
)
pipe.add_step(WriteFileStep(outts))
if recoded_output is not None:
recoded_ts = TimeStream(recoded_output,
bundle_level=recoded_bundle,
add_subsecond_field=True)
recode_pipeline = TSPipeline(
EncodeImageFileStep(format=recoded_format),
WriteFileStep(downsized_ts),
)
pipe.add_step(TeeStep(recode_pipeline))
if nuke:
pipe.add_step(UnsafeNuker())
try:
if inotify_watch is not None:
instream = ints.from_inotify(inotify_watch)
else:
instream = ints.from_fofn(input)
for image in instream:
image = pipe.process_file(image)
click.echo(f"{image.filename} Done")
pipe.n += 1
finally:
pipe.finish()
click.echo(f"Ingested {ifmt} to {output}, found {pipe.n} files")