def __init__(self, tigraph_factory=None, model=None):
if tigraph_factory is None:
tigraph_factory = model.get_inference_graph()
assert isinstance(tigraph_factory, TFInferenceGraphFactory)
self.tigraph_factory = tigraph_factory
clazzname = self.__class__.__name__
self.overall_thruput = util.ThruputObserver(name=clazzname,
log_on_del=True)
self.tf_thruput = util.ThruputObserver(name=clazzname + '.tf_thruput',
log_on_del=True)
def __init__(self, target_hw=None, target_nchan=None, norm_func=None):
self.norm_func = norm_func
self.target_hw = target_hw
self.target_nchan = target_nchan
self.thruput = util.ThruputObserver(
name='FillNormalized',
log_on_del=True)
def test_thruput_observer():
t1 = util.ThruputObserver()
assert str(t1)
t2 = util.ThruputObserver()
import random
import time
MAX_WAIT = 0.1
for _ in range(10):
with t2.observe(n=1, num_bytes=1):
time.sleep(random.random() * MAX_WAIT)
assert str(t2)
u = util.ThruputObserver.union((t1, t2))
assert str(u) == str(t2)
def thruput_accumulator(spark, **thruputKwargs):
from pyspark.accumulators import AccumulatorParam
class ThruputObsAccumulator(AccumulatorParam):
def zero(self, v):
return v or util.ThruputObserver()
def addInPlace(self, value1, value2):
value1 += value2
return value1
return spark.sparkContext.accumulator(
util.ThruputObserver(**thruputKwargs),
ThruputObsAccumulator())
def __init__(self, name, data_proxy):
vformat = name.split('.')[-1]
# bdd100k videos have some odd dimension issue, so we silence:
# "the frame size for reading (1280, 720) is different from the source frame size (720, 1280)"
import imageio
import imageio.plugins.ffmpeg
imageio.plugins.ffmpeg.logging.warning = lambda m: True
self.reader = imageio.get_reader(data_proxy(), format=vformat)
self.name = name
self.lock = threading.Lock()
self.thruput = util.ThruputObserver(name='ExclusiveReader.' +
self.name,
log_on_del=True)
def gen_videometas(vidnames):
t = util.ThruputObserver()
with t.observe():
for vidname in vidnames:
video = cls.get_video(vidname)
if video:
videometa = video.video_meta
else:
# Just use the meta info in the telemetry dataset
meta = cls.INFO.get_meta_for_video(vidname)
videometa = VideoMeta.from_meta(meta)
yield videometa
t.update_tallies(n=1)
global _setup_thruput
_setup_thruput += t
def zero(self, v): return v or util.ThruputObserver()