def __init__(self, tigraph_factory=None, model=None): if tigraph_factory is None: tigraph_factory = model.get_inference_graph() assert isinstance(tigraph_factory, TFInferenceGraphFactory) self.tigraph_factory = tigraph_factory clazzname = self.__class__.__name__ self.overall_thruput = util.ThruputObserver(name=clazzname, log_on_del=True) self.tf_thruput = util.ThruputObserver(name=clazzname + '.tf_thruput', log_on_del=True)
def __init__(self, target_hw=None, target_nchan=None, norm_func=None): self.norm_func = norm_func self.target_hw = target_hw self.target_nchan = target_nchan self.thruput = util.ThruputObserver( name='FillNormalized', log_on_del=True)
def test_thruput_observer(): t1 = util.ThruputObserver() assert str(t1) t2 = util.ThruputObserver() import random import time MAX_WAIT = 0.1 for _ in range(10): with t2.observe(n=1, num_bytes=1): time.sleep(random.random() * MAX_WAIT) assert str(t2) u = util.ThruputObserver.union((t1, t2)) assert str(u) == str(t2)
def thruput_accumulator(spark, **thruputKwargs): from pyspark.accumulators import AccumulatorParam class ThruputObsAccumulator(AccumulatorParam): def zero(self, v): return v or util.ThruputObserver() def addInPlace(self, value1, value2): value1 += value2 return value1 return spark.sparkContext.accumulator( util.ThruputObserver(**thruputKwargs), ThruputObsAccumulator())
def __init__(self, name, data_proxy): vformat = name.split('.')[-1] # bdd100k videos have some odd dimension issue, so we silence: # "the frame size for reading (1280, 720) is different from the source frame size (720, 1280)" import imageio import imageio.plugins.ffmpeg imageio.plugins.ffmpeg.logging.warning = lambda m: True self.reader = imageio.get_reader(data_proxy(), format=vformat) self.name = name self.lock = threading.Lock() self.thruput = util.ThruputObserver(name='ExclusiveReader.' + self.name, log_on_del=True)
def gen_videometas(vidnames): t = util.ThruputObserver() with t.observe(): for vidname in vidnames: video = cls.get_video(vidname) if video: videometa = video.video_meta else: # Just use the meta info in the telemetry dataset meta = cls.INFO.get_meta_for_video(vidname) videometa = VideoMeta.from_meta(meta) yield videometa t.update_tallies(n=1) global _setup_thruput _setup_thruput += t
def zero(self, v): return v or util.ThruputObserver()