def connect(sensor, algorithm, uri): db=DBIngest(host=dbhost, index="algorithms",office=office) db.update(algorithm["_id"], { "sensor": sensor["_id"], }) db=DBIngest(host=dbhost, index="analytics", office=office) while True: counts=[] for i in range(100): zonecount={} for zonemap in sensor["_source"]["zonemap"]: zonecount["zone"+str(zonemap["zone"])]=int(random.random()*1000) counts.append({ "time": int(time.mktime(datetime.datetime.now().timetuple())*1000), "office": { "lat": office[0], "lon": office[1], }, "sensor": sensor["_id"], "algorithm": algorithm["_id"], "count": zonecount, }) db.ingest_bulk(counts) time.sleep(1000)
class RunVA(object): def __init__(self): super(RunVA, self).__init__() # remove HTTP_PROXY env = os.environ.copy() env.pop("http_proxy", None) env.pop("HTTP_PROXY", None) self._va = Popen(["/usr/bin/python3", "-m", "openapi_server"], cwd="/home/video-analytics/app/server", env=env) self._db = DBIngest(host=dbhost, index="algorithms", office=office) self._stop = None def stop(self): self._stop = True def loop(self, sensor, location, uri, algorithm, topic): req = { "source": { "uri": uri, "type": "uri" }, "destination": { "type": "mqtt", "host": mqtthost, "clientid": algorithm, "topic": topic }, "tags": { "sensor": sensor, "location": location, "algorithm": algorithm, "office": { "lat": office[0], "lon": office[1], }, }, "parameters": { "every-nth-frame": every_nth_frame, "recording_prefix": "recordings/" + sensor }, } while True: try: r = requests.post(vahost + "/object_detection/2", json=req, timeout=10) if r.status_code == 200: pid = int(r.text) break except Exception as e: print("Exception: " + str(e), flush=True) time.sleep(10) while not self._stop: r = requests.get(vahost + "/object_detection/2/" + str(pid) + "/status", timeout=10) if r.status_code != 200: print("pipeline status: " + str(r.status_code), flush=True) print(r.text, flush=True) break pinfo = r.json() print(pinfo, flush=True) state = pinfo["state"] if state == "COMPLETED" or state == "ABORTED" or state == "ERROR": print("pineline ended with " + str(state), flush=True) break if state == "RUNNING": if "avg_pipeline_latency" not in pinfo: pinfo["avg_pipeline_latency"] = 0 self._db.update( algorithm, { "sensor": sensor, "performance": pinfo["avg_fps"], "latency": pinfo["avg_pipeline_latency"] * 1000, }) time.sleep(10) print("exiting va pipeline", flush=True) self._va.terminate()
class RunVA(object): def __init__(self, pipeline, version="2"): super(RunVA, self).__init__() self._pipeline = pipeline self._version = version self._db = DBIngest(host=dbhost, index="algorithms", office=office) self._maincontext = GLib.MainLoop().get_context() self._stop = None ModelManager.load_config("/home/models", {}) PipelineManager.load_config("/home/pipelines", 1) def _noop(self): pass def stop(self): GLib.timeout_add(10, self._noop) self._stop = True def loop(self, sensor, location, uri, algorithm, algorithmName, resolution={ "width": 0, "height": 0 }, zonemap=[], topic="analytics"): pid, msg = PipelineManager.create_instance( self._pipeline, self._version, { "source": { "uri": uri, "type": "uri" }, "destination": { "type": "mqtt", "host": mqtthost, "clientid": algorithm, "topic": topic, }, "tags": { "sensor": sensor, "location": location, "algorithm": algorithm, "office": { "lat": office[0], "lon": office[1], }, }, "parameters": { "crowd_count": { # crowd-counting only "width": resolution["width"], "height": resolution["height"], "zonemap": zonemap }, "every-nth-frame": every_nth_frame, "recording_prefix": "/tmp/" + sensor, }, }) if pid is None: print("Exception: " + str(msg), flush=True) return while not self._stop: self._maincontext.iteration() pinfo = PipelineManager.get_instance_status( self._pipeline, self._version, pid) if pinfo is not None: print(pinfo, flush=True) state = pinfo["state"] if state == "COMPLETED" or state == "ABORTED" or state == "ERROR": print("pineline ended with " + str(state), flush=True) break if pinfo["avg_fps"] > 0 and state == "RUNNING": if "avg_pipeline_latency" not in pinfo: pinfo["avg_pipeline_latency"] = 0 self._db.update( algorithm, { "sensor": sensor, "performance": pinfo["avg_fps"], "latency": pinfo["avg_pipeline_latency"] * 1000, }) print("exiting va pipeline", flush=True) PipelineManager.stop_instance(self._pipeline, self._version, pid)
class Feeder(): def __init__(self): logger.debug("Initializing Feeder") self.office = list(map(float, os.environ["OFFICE"].split(","))) self.alg_id = None self.recording_volume = os.environ["STORAGE_VOLUME"] self.every_nth_frame = int(os.environ["EVERY_NTH_FRAME"]) #Hosts self.dbhost = os.environ["DBHOST"] self.vahost = "http://localhost:8080/pipelines" self.mqtthost = os.environ["MQTTHOST"] #Clients self.db_alg = DBIngest(host=self.dbhost, index="algorithms", office=self.office) self.db_inf = DBIngest(host=self.dbhost, index="analytics", office=self.office) self.db_sensors = DBQuery(host=self.dbhost, index="sensors", office=self.office) self.mqttclient = None self.mqtttopic = None self.observer = Observer() self.batchsize = 300 self.inference_cache = [] self._threadflag = False def start(self): logger.info(" ### Starting Feeder ### ") logger.debug("Waiting for VA startup") r = requests.Response() r.status_code = 400 while r.status_code != 200 and r.status_code != 201: try: r = requests.get(self.vahost) except Exception as e: r = requests.Response() r.status_code = 400 time.sleep(10) # Register Algorithm logger.debug("Registering as algorithm in the DB") while True: try: self.alg_id = self.db_alg.ingest({ "name": "object_detection", "office": { "lat": self.office[0], "lon": self.office[1] }, "status": "idle", "skip": self.every_nth_frame, })["_id"] break except Exception as e: logger.debug("Register algo exception: " + str(e)) time.sleep(10) self.mqtttopic = "smtc_va_inferences_" + self.alg_id camera_monitor_thread = Thread(target=self.monitor_cameras, daemon=True) logger.debug("Starting working threads") self._threadflag = True self.startmqtt() self.observer.start() camera_monitor_thread.start() logger.debug("Waiting for interrupt...") camera_monitor_thread.join() self.observer.join() def stop(self): logger.info(" ### Stopping Feeder ### ") self._threadflag = False logger.debug("Unregistering algorithm from DB") self.db_alg.delete(self.alg_id) self.mqttclient.loop_stop() self.observer.stop() def startmqtt(self): self.mqttclient = mqtt.Client("feeder_" + self.alg_id) self.mqttclient.connect(self.mqtthost) self.mqttclient.on_message = self.mqtt_handler self.mqttclient.loop_start() self.mqttclient.subscribe(self.mqtttopic) def mqtt_handler(self, client, userdata, message): m_in = json.loads(str(message.payload.decode("utf-8", "ignore"))) for tag in m_in["tags"]: m_in[tag] = m_in["tags"][tag] del m_in["tags"] m_in["time"] = m_in["real_base"] + m_in["timestamp"] # convert to milliseconds m_in["time"] = int(m_in["time"] / 1000000) self.inference_cache.append(m_in) if len(self.inference_cache) >= self.batchsize: try: self.db_inf.ingest_bulk(self.inference_cache[:self.batchsize]) self.inference_cache = self.inference_cache[self.batchsize:] except Exception as e: logger.debug("Ingest Error: " + str(e)) def monitor_cameras(self): logger.debug("Starting Sensor Monitor Thread") while self._threadflag: logger.debug("Searching for sensors...") try: for sensor in self.db_sensors.search( "sensor:'camera' and status:'idle' and office:[" + str(self.office[0]) + "," + str(self.office[1]) + "]"): logger.debug(sensor) try: fswatch = None logger.debug("Sensor found! " + sensor["_id"]) logger.debug("Setting sensor " + sensor["_id"] + " to streaming") r = self.db_sensors.update(sensor["_id"], {"status": "streaming"}, version=sensor["_version"]) logger.debug( "Setting algorithm to streaming from sensor " + sensor["_id"]) r = self.db_alg.update(self.alg_id, { "source": sensor["_id"], "status": "processing" }) # Attempt to POST to VA service jsonData = { "source": { "uri": sensor["_source"]["url"], "type": "uri" }, "tags": { "algorithm": self.alg_id, "sensor": sensor["_id"], "office": { "lat": self.office[0], "lon": self.office[1], }, }, "parameters": { "every-nth-frame": self.every_nth_frame, "recording_prefix": "recordings/" + sensor["_id"], "method": "mqtt", "address": self.mqtthost, "clientid": self.alg_id, "topic": self.mqtttopic }, } folderpath = os.path.join( os.path.realpath(self.recording_volume), sensor["_id"]) if not os.path.exists(folderpath): os.makedirs(folderpath) logger.debug("Adding folder watch for " + folderpath) filehandler = FSHandler( sensor=sensor["_id"], office=self.office, dbhost=self.dbhost, rec_volume=self.recording_volume) fswatch = self.observer.schedule(filehandler, folderpath, recursive=True) try: logger.info("Posting Request to VA Service") r = requests.post(self.vahost + "/object_detection/2", json=jsonData, timeout=10) r.raise_for_status() pipeline_id = None if r.status_code == 200: logger.debug("Started pipeline " + r.text) pipeline_id = int(r.text) while r.status_code == 200: logger.debug("Querying status of pipeline") r = requests.get(self.vahost + "/object_detection/2/" + str(pipeline_id) + "/status", timeout=10) r.raise_for_status() jsonValue = r.json() if "avg_pipeline_latency" not in jsonValue: jsonValue["avg_pipeline_latency"] = 0 state = jsonValue["state"] try: logger.debug("fps: ") logger.debug(str(jsonValue)) except: logger.debug("error") logger.debug("Pipeline state is " + str(state)) if state == "COMPLETED" or state == "ABORTED" or state == "ERROR": logger.debug("Pipeline ended") break self.db_alg.update( self.alg_id, { "performance": jsonValue["avg_fps"], "latency": jsonValue["avg_pipeline_latency"] * 1000 }) time.sleep(10) logger.debug("Setting sensor " + sensor["_id"] + " to disconnected") r = self.db_sensors.update( sensor["_id"], {"status": "disconnected"}) except requests.exceptions.RequestException as e: logger.error( "Feeder: Request to VA Service Failed: " + str(e)) logger.debug("Setting sensor " + sensor["_id"] + " to idle") r = self.db_sensors.update(sensor["_id"], {"status": "idle"}) except Exception as e: logger.error("Feeder Exception: " + str(e)) if fswatch: self.observer.unschedule(fswatch) del (filehandler) logger.debug("Setting algorithm to idle") r = self.db_alg.update(self.alg_id, {"status": "idle"}) break except Exception as e: print(e, flush=True) time.sleep(5) logger.debug("Sensor monitor thread done")
class RunVA(object): def _test_mqtt_connection(self): print("testing mqtt connection", flush=True) mqtt = Client() while True: try: mqtt.connect(mqtthost) break except: print("Waiting for mqtt...", flush=True) time.sleep(5) print("mqtt connected", flush=True) mqtt.disconnect() def __init__(self, pipeline, version="2"): super(RunVA, self).__init__() self._test_mqtt_connection() self._pipeline = pipeline self._version = version self._db = DBIngest(host=dbhost, index="algorithms", office=office) self._stop = None def stop(self): if self._stop: print("stopping", flush=True) self._stop.set() def loop(self, sensor, location, uri, algorithm, algorithmName, options={}, topic="analytics"): try: VAServing.start({ 'model_dir': '/home/models', 'pipeline_dir': '/home/pipelines', 'max_running_pipelines': 1, }) try: source = { "type": "uri", "uri": uri, } destination = { "type": "mqtt", "host": mqtthost, "clientid": algorithm, "topic": topic } tags = { "sensor": sensor, "location": location, "algorithm": algorithmName, "office": { "lat": office[0], "lon": office[1] }, } parameters = { "inference-interval": every_nth_frame, "recording_prefix": "/tmp/rec/" + sensor } parameters.update(options) pipeline = VAServing.pipeline(self._pipeline, self._version) instance_id = pipeline.start(source=source, destination=destination, tags=tags, parameters=parameters) if instance_id is None: raise Exception( "Pipeline {} version {} Failed to Start".format( self._pipeline, self._version)) self._stop = Event() while not self._stop.is_set(): status = pipeline.status() print(status, flush=True) if status.state.stopped(): print( "Pipeline {} Version {} Instance {} Ended with {}". format(self._pipeline, self._version, instance_id, status.state.name), flush=True) break if status.avg_fps > 0 and status.state is Pipeline.State.RUNNING: avg_pipeline_latency = status.avg_pipeline_latency if not avg_pipeline_latency: avg_pipeline_latency = 0 self._db.update( algorithm, { "sensor": sensor, "performance": status.avg_fps, "latency": avg_pipeline_latency * 1000, "cpu": psutil.cpu_percent(), "memory": psutil.virtual_memory().percent, }) self._stop.wait(3) self._stop = None pipeline.stop() except: print(traceback.format_exc(), flush=True) VAServing.stop() except: print(traceback.format_exc(), flush=True)