def connect(sensor, algorithm, uri):
db=DBIngest(host=dbhost, index="algorithms",office=office)
db.update(algorithm["_id"], {
"sensor": sensor["_id"],
})
db=DBIngest(host=dbhost, index="analytics", office=office)
while True:
counts=[]
for i in range(100):
zonecount={}
for zonemap in sensor["_source"]["zonemap"]:
zonecount["zone"+str(zonemap["zone"])]=int(random.random()*1000)
counts.append({
"time": int(time.mktime(datetime.datetime.now().timetuple())*1000),
"office": {
"lat": office[0],
"lon": office[1],
},
"sensor": sensor["_id"],
"algorithm": algorithm["_id"],
"count": zonecount,
})
db.ingest_bulk(counts)
time.sleep(1000)
class RunVA(object):
def __init__(self):
super(RunVA, self).__init__()
# remove HTTP_PROXY
env = os.environ.copy()
env.pop("http_proxy", None)
env.pop("HTTP_PROXY", None)
self._va = Popen(["/usr/bin/python3", "-m", "openapi_server"],
cwd="/home/video-analytics/app/server",
env=env)
self._db = DBIngest(host=dbhost, index="algorithms", office=office)
self._stop = None
def stop(self):
self._stop = True
def loop(self, sensor, location, uri, algorithm, topic):
req = {
"source": {
"uri": uri,
"type": "uri"
},
"destination": {
"type": "mqtt",
"host": mqtthost,
"clientid": algorithm,
"topic": topic
},
"tags": {
"sensor": sensor,
"location": location,
"algorithm": algorithm,
"office": {
"lat": office[0],
"lon": office[1],
},
},
"parameters": {
"every-nth-frame": every_nth_frame,
"recording_prefix": "recordings/" + sensor
},
}
while True:
try:
r = requests.post(vahost + "/object_detection/2",
json=req,
timeout=10)
if r.status_code == 200:
pid = int(r.text)
break
except Exception as e:
print("Exception: " + str(e), flush=True)
time.sleep(10)
while not self._stop:
r = requests.get(vahost + "/object_detection/2/" + str(pid) +
"/status",
timeout=10)
if r.status_code != 200:
print("pipeline status: " + str(r.status_code), flush=True)
print(r.text, flush=True)
break
pinfo = r.json()
print(pinfo, flush=True)
state = pinfo["state"]
if state == "COMPLETED" or state == "ABORTED" or state == "ERROR":
print("pineline ended with " + str(state), flush=True)
break
if state == "RUNNING":
if "avg_pipeline_latency" not in pinfo:
pinfo["avg_pipeline_latency"] = 0
self._db.update(
algorithm, {
"sensor": sensor,
"performance": pinfo["avg_fps"],
"latency": pinfo["avg_pipeline_latency"] * 1000,
})
time.sleep(10)
print("exiting va pipeline", flush=True)
self._va.terminate()
class RunVA(object):
def __init__(self, pipeline, version="2"):
super(RunVA, self).__init__()
self._pipeline = pipeline
self._version = version
self._db = DBIngest(host=dbhost, index="algorithms", office=office)
self._maincontext = GLib.MainLoop().get_context()
self._stop = None
ModelManager.load_config("/home/models", {})
PipelineManager.load_config("/home/pipelines", 1)
def _noop(self):
pass
def stop(self):
GLib.timeout_add(10, self._noop)
self._stop = True
def loop(self,
sensor,
location,
uri,
algorithm,
algorithmName,
resolution={
"width": 0,
"height": 0
},
zonemap=[],
topic="analytics"):
pid, msg = PipelineManager.create_instance(
self._pipeline,
self._version,
{
"source": {
"uri": uri,
"type": "uri"
},
"destination": {
"type": "mqtt",
"host": mqtthost,
"clientid": algorithm,
"topic": topic,
},
"tags": {
"sensor": sensor,
"location": location,
"algorithm": algorithm,
"office": {
"lat": office[0],
"lon": office[1],
},
},
"parameters": {
"crowd_count": { # crowd-counting only
"width": resolution["width"],
"height": resolution["height"],
"zonemap": zonemap
},
"every-nth-frame": every_nth_frame,
"recording_prefix": "/tmp/" + sensor,
},
})
if pid is None:
print("Exception: " + str(msg), flush=True)
return
while not self._stop:
self._maincontext.iteration()
pinfo = PipelineManager.get_instance_status(
self._pipeline, self._version, pid)
if pinfo is not None:
print(pinfo, flush=True)
state = pinfo["state"]
if state == "COMPLETED" or state == "ABORTED" or state == "ERROR":
print("pineline ended with " + str(state), flush=True)
break
if pinfo["avg_fps"] > 0 and state == "RUNNING":
if "avg_pipeline_latency" not in pinfo:
pinfo["avg_pipeline_latency"] = 0
self._db.update(
algorithm, {
"sensor": sensor,
"performance": pinfo["avg_fps"],
"latency": pinfo["avg_pipeline_latency"] * 1000,
})
print("exiting va pipeline", flush=True)
PipelineManager.stop_instance(self._pipeline, self._version, pid)
class Feeder():
def __init__(self):
logger.debug("Initializing Feeder")
self.office = list(map(float, os.environ["OFFICE"].split(",")))
self.alg_id = None
self.recording_volume = os.environ["STORAGE_VOLUME"]
self.every_nth_frame = int(os.environ["EVERY_NTH_FRAME"])
#Hosts
self.dbhost = os.environ["DBHOST"]
self.vahost = "http://localhost:8080/pipelines"
self.mqtthost = os.environ["MQTTHOST"]
#Clients
self.db_alg = DBIngest(host=self.dbhost,
index="algorithms",
office=self.office)
self.db_inf = DBIngest(host=self.dbhost,
index="analytics",
office=self.office)
self.db_sensors = DBQuery(host=self.dbhost,
index="sensors",
office=self.office)
self.mqttclient = None
self.mqtttopic = None
self.observer = Observer()
self.batchsize = 300
self.inference_cache = []
self._threadflag = False
def start(self):
logger.info(" ### Starting Feeder ### ")
logger.debug("Waiting for VA startup")
r = requests.Response()
r.status_code = 400
while r.status_code != 200 and r.status_code != 201:
try:
r = requests.get(self.vahost)
except Exception as e:
r = requests.Response()
r.status_code = 400
time.sleep(10)
# Register Algorithm
logger.debug("Registering as algorithm in the DB")
while True:
try:
self.alg_id = self.db_alg.ingest({
"name": "object_detection",
"office": {
"lat": self.office[0],
"lon": self.office[1]
},
"status": "idle",
"skip": self.every_nth_frame,
})["_id"]
break
except Exception as e:
logger.debug("Register algo exception: " + str(e))
time.sleep(10)
self.mqtttopic = "smtc_va_inferences_" + self.alg_id
camera_monitor_thread = Thread(target=self.monitor_cameras,
daemon=True)
logger.debug("Starting working threads")
self._threadflag = True
self.startmqtt()
self.observer.start()
camera_monitor_thread.start()
logger.debug("Waiting for interrupt...")
camera_monitor_thread.join()
self.observer.join()
def stop(self):
logger.info(" ### Stopping Feeder ### ")
self._threadflag = False
logger.debug("Unregistering algorithm from DB")
self.db_alg.delete(self.alg_id)
self.mqttclient.loop_stop()
self.observer.stop()
def startmqtt(self):
self.mqttclient = mqtt.Client("feeder_" + self.alg_id)
self.mqttclient.connect(self.mqtthost)
self.mqttclient.on_message = self.mqtt_handler
self.mqttclient.loop_start()
self.mqttclient.subscribe(self.mqtttopic)
def mqtt_handler(self, client, userdata, message):
m_in = json.loads(str(message.payload.decode("utf-8", "ignore")))
for tag in m_in["tags"]:
m_in[tag] = m_in["tags"][tag]
del m_in["tags"]
m_in["time"] = m_in["real_base"] + m_in["timestamp"]
# convert to milliseconds
m_in["time"] = int(m_in["time"] / 1000000)
self.inference_cache.append(m_in)
if len(self.inference_cache) >= self.batchsize:
try:
self.db_inf.ingest_bulk(self.inference_cache[:self.batchsize])
self.inference_cache = self.inference_cache[self.batchsize:]
except Exception as e:
logger.debug("Ingest Error: " + str(e))
def monitor_cameras(self):
logger.debug("Starting Sensor Monitor Thread")
while self._threadflag:
logger.debug("Searching for sensors...")
try:
for sensor in self.db_sensors.search(
"sensor:'camera' and status:'idle' and office:[" +
str(self.office[0]) + "," + str(self.office[1]) + "]"):
logger.debug(sensor)
try:
fswatch = None
logger.debug("Sensor found! " + sensor["_id"])
logger.debug("Setting sensor " + sensor["_id"] +
" to streaming")
r = self.db_sensors.update(sensor["_id"],
{"status": "streaming"},
version=sensor["_version"])
logger.debug(
"Setting algorithm to streaming from sensor " +
sensor["_id"])
r = self.db_alg.update(self.alg_id, {
"source": sensor["_id"],
"status": "processing"
})
# Attempt to POST to VA service
jsonData = {
"source": {
"uri": sensor["_source"]["url"],
"type": "uri"
},
"tags": {
"algorithm": self.alg_id,
"sensor": sensor["_id"],
"office": {
"lat": self.office[0],
"lon": self.office[1],
},
},
"parameters": {
"every-nth-frame": self.every_nth_frame,
"recording_prefix":
"recordings/" + sensor["_id"],
"method": "mqtt",
"address": self.mqtthost,
"clientid": self.alg_id,
"topic": self.mqtttopic
},
}
folderpath = os.path.join(
os.path.realpath(self.recording_volume),
sensor["_id"])
if not os.path.exists(folderpath):
os.makedirs(folderpath)
logger.debug("Adding folder watch for " + folderpath)
filehandler = FSHandler(
sensor=sensor["_id"],
office=self.office,
dbhost=self.dbhost,
rec_volume=self.recording_volume)
fswatch = self.observer.schedule(filehandler,
folderpath,
recursive=True)
try:
logger.info("Posting Request to VA Service")
r = requests.post(self.vahost +
"/object_detection/2",
json=jsonData,
timeout=10)
r.raise_for_status()
pipeline_id = None
if r.status_code == 200:
logger.debug("Started pipeline " + r.text)
pipeline_id = int(r.text)
while r.status_code == 200:
logger.debug("Querying status of pipeline")
r = requests.get(self.vahost +
"/object_detection/2/" +
str(pipeline_id) + "/status",
timeout=10)
r.raise_for_status()
jsonValue = r.json()
if "avg_pipeline_latency" not in jsonValue:
jsonValue["avg_pipeline_latency"] = 0
state = jsonValue["state"]
try:
logger.debug("fps: ")
logger.debug(str(jsonValue))
except:
logger.debug("error")
logger.debug("Pipeline state is " + str(state))
if state == "COMPLETED" or state == "ABORTED" or state == "ERROR":
logger.debug("Pipeline ended")
break
self.db_alg.update(
self.alg_id, {
"performance":
jsonValue["avg_fps"],
"latency":
jsonValue["avg_pipeline_latency"] *
1000
})
time.sleep(10)
logger.debug("Setting sensor " + sensor["_id"] +
" to disconnected")
r = self.db_sensors.update(
sensor["_id"], {"status": "disconnected"})
except requests.exceptions.RequestException as e:
logger.error(
"Feeder: Request to VA Service Failed: " +
str(e))
logger.debug("Setting sensor " + sensor["_id"] +
" to idle")
r = self.db_sensors.update(sensor["_id"],
{"status": "idle"})
except Exception as e:
logger.error("Feeder Exception: " + str(e))
if fswatch:
self.observer.unschedule(fswatch)
del (filehandler)
logger.debug("Setting algorithm to idle")
r = self.db_alg.update(self.alg_id, {"status": "idle"})
break
except Exception as e:
print(e, flush=True)
time.sleep(5)
logger.debug("Sensor monitor thread done")
class RunVA(object):
def _test_mqtt_connection(self):
print("testing mqtt connection", flush=True)
mqtt = Client()
while True:
try:
mqtt.connect(mqtthost)
break
except:
print("Waiting for mqtt...", flush=True)
time.sleep(5)
print("mqtt connected", flush=True)
mqtt.disconnect()
def __init__(self, pipeline, version="2"):
super(RunVA, self).__init__()
self._test_mqtt_connection()
self._pipeline = pipeline
self._version = version
self._db = DBIngest(host=dbhost, index="algorithms", office=office)
self._stop = None
def stop(self):
if self._stop:
print("stopping", flush=True)
self._stop.set()
def loop(self,
sensor,
location,
uri,
algorithm,
algorithmName,
options={},
topic="analytics"):
try:
VAServing.start({
'model_dir': '/home/models',
'pipeline_dir': '/home/pipelines',
'max_running_pipelines': 1,
})
try:
source = {
"type": "uri",
"uri": uri,
}
destination = {
"type": "mqtt",
"host": mqtthost,
"clientid": algorithm,
"topic": topic
}
tags = {
"sensor": sensor,
"location": location,
"algorithm": algorithmName,
"office": {
"lat": office[0],
"lon": office[1]
},
}
parameters = {
"inference-interval": every_nth_frame,
"recording_prefix": "/tmp/rec/" + sensor
}
parameters.update(options)
pipeline = VAServing.pipeline(self._pipeline, self._version)
instance_id = pipeline.start(source=source,
destination=destination,
tags=tags,
parameters=parameters)
if instance_id is None:
raise Exception(
"Pipeline {} version {} Failed to Start".format(
self._pipeline, self._version))
self._stop = Event()
while not self._stop.is_set():
status = pipeline.status()
print(status, flush=True)
if status.state.stopped():
print(
"Pipeline {} Version {} Instance {} Ended with {}".
format(self._pipeline, self._version, instance_id,
status.state.name),
flush=True)
break
if status.avg_fps > 0 and status.state is Pipeline.State.RUNNING:
avg_pipeline_latency = status.avg_pipeline_latency
if not avg_pipeline_latency: avg_pipeline_latency = 0
self._db.update(
algorithm, {
"sensor": sensor,
"performance": status.avg_fps,
"latency": avg_pipeline_latency * 1000,
"cpu": psutil.cpu_percent(),
"memory": psutil.virtual_memory().percent,
})
self._stop.wait(3)
self._stop = None
pipeline.stop()
except:
print(traceback.format_exc(), flush=True)
VAServing.stop()
except:
print(traceback.format_exc(), flush=True)
