def unify(self, req):
try:
print(req["unify"])
response = http.request("GET",
req["unify"] + '/api/2.0/recording',
fields=req)
items = json.loads(response.data.decode('utf8'))
debug(
"Connected to " + req["unify"] + "(" +
str(len(items['data'])) + " available videos)", 1)
except:
debug("Unable to connect to " + req["unify"], 0)
return None
for r in items['data']:
if "locked" in req:
if req["locked"] == True and r["locked"] == False:
continue
requests.append({
"name":
r["meta"]["cameraName"] + "-" + str(r["endTime"]),
"url":
req["unify"] + '/api/2.0/recording/' + r["_id"] +
'/download?apiKey=' + req["apiKey"],
"startTime":
r["startTime"],
"endTime":
r["endTime"]
})
def get_image(self):
self.psProcess.resume()
size = self.shape[0] * self.shape[1] * 3
raw_image = self.pipe.stdout.read(size)
while np.array_equal(self.raw_image_prev, raw_image) is True:
raw_image = self.pipe.stdout.read(size)
self.img_count += 1
debug("Drop frame from ffmpeg (duplicated)", 3)
self.raw_image_prev = np.copy(raw_image)
#raw_image = self.pipe.stdout.read(size)
image = np.fromstring(raw_image, dtype='uint8')
if image.shape[0] == 0:
return None
image = image.reshape((self.shape[1], self.shape[0], 3))
self.pipe.stdout.flush()
self.psProcess.suspend()
if (self.doResize):
image = np.array(
Image.fromarray(image, 'RGB').resize(self.resolution))
self.img_count += 1
return image
def _launchTarget(self):
try:
self._doJob()
except:
debug("Error from job thread", 0, True)
traceback.print_exc()
self.stop(1)
def startStreamer(self, streamerInfo):
if (not checkStreamerConfigSanity(json.dumps(streamerInfo))):
raise ValueError("Error in configuration")
resolution = getWithDefault(streamerInfo, "resolution",
self.options[self.DEFAULT_RESOLUTION_TAG])
img_rate = getWithDefault(streamerInfo, "img_rate",
self.options[self.DEFAULT_IMG_RATE_TAG])
name = getWithDefault(streamerInfo, "name",
"streamer" + str(self.streamerCount))
location = getWithDefault(streamerInfo, "url")
realtime = getWithDefault(streamerInfo, "realtime",
self.options[self.REALTIME_TAG])
if (location == None):
location = getWithDefault(streamerInfo, "path")
try:
if (realtime):
streamer = RealTimeStreamer(name, location, img_rate,
resolution)
else:
streamer = Streamer(name, location, img_rate, resolution)
except:
debug("Cannot start streamer " + name + " (" + str(location) + ")",
2, True)
if (_DEBUG_LEVEL >= 3):
traceback.print_exc()
return
self.streamers.append(streamer)
self.streamerCount += 1
def _listenTarget(self):
try:
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(('', self.port))
server.listen(4)
debug("Listening on " + str(self.port))
except:
traceback.print_exc()
self.stop(1)
return
self.server = server
try:
while (not self.workerShutdown.value):
client, addr = server.accept()
if (client != None):
debug("Input from: " + str(addr), 1)
Thread(target=self._clientInTarget,
args=(client, ),
daemon=True).start()
except:
if (self.workerShutdown.value):
return
traceback.print_exc()
if (not self.jobRunning.value):
return
def loop(self, data):
img = None
while type(img) == type(None):
if (self.streamerIndex >= len(self.streamers)):
self.streamerIndex = 0
if (len(self.streamers) == 0):
return None
streamer = self.streamers[self.streamerIndex]
img = streamer.get_image()
if (type(img) == type(None)):
self.streamers.remove(streamer)
print("Streamer " + streamer.name + " is done reading")
self._startNewStreamerFromExploration()
if (len(self.streamers) == 0):
debug("Reached end of multistreamer job, successfull exit",
2)
self.shouldStop = True
return None
img2 = np.array(Image.fromarray(img))
self.streamerIndex += 1
return {
"from": str(streamer.name),
"frame_count": str(streamer.img_count),
"img": img2
}
def _clientTarget(self, sock):
chan = sock.makefile("rwb")
while(self.running):
try:
j = network.readString(chan)
out = network.OK
try:
sa = self._detectSpecialAction(j)
if(sa != None):
out = sa
else:
self.handleConfigInput(j)
network.sendString(chan, out)
except Exception as e:
network.sendString(chan, repr(e))
raise e
except struct.error:
break
except:
traceback.print_exc()
break
debug("Closing client connection", 1, True)
sock.shutdown(socket.SHUT_RDWR)
sock.close()
def push(self, obj, throwOnError = True):
#prof.enter("PUSH")
debug("Pushing to "+self.name,2)
sock = self._connect()
chan = sock.makefile("rwb")
network.sendString(chan, json.dumps(obj))
debug("Awaiting reply...",2)
answ = network.readString(chan) or "SOCK_TIMEOUT"
try:
self._close(sock)
except:
pass
if(answ != network.OK):
if(throwOnError):
error("Push failure", 0)
error("Refer to supervisor log for details", 0)
error("Err was:"+answ,0)
return False
ok("Configuration "+ obj["workername"] + "("+ obj["jobname"]+") on " + self.name)
return answ
def handleClose(self):
debug(str(self.address) + ' closed', 2)
if self in clients:
clients.remove(self)
if self in clients_debug:
clients_debug.remove(self)
def setupJob(self, data):
if (self.job == None):
raise ValueError("This worker has no job")
self.job.setup(data)
self.jobSetup = True
self.job.shouldStop = False
debug("Pushing data", 1)
def setNetworkMethod(self, config):
if ("outputmethod" in config):
if (config['outputmethod'] == "distribute"):
debug("Output method is set to distribution")
self.outputmethod = self._distributeOverNetwork
else:
debug("Output method is set to duplication")
self.outputmethod = self._duplicateOverNetwork
def _testConnection(self):
debug("Testing connection to: " + str(self.addr))
s = self._connect()
if (s == None):
return False
self._close(s)
return True
def setup(self, data):
config = b"cfg/yolo9000.cfg"
weights = b"weights/yolo9000.weights"
meta = b"cfg/combine9k.data"
self.detector = Detector(config, weights, meta)
debug(
"Started detector cfg=" + str(config) + " weights=" +
str(weights) + " meta=" + str(meta), 3)
def loadSupervisors(cfg):
debug("Master config sanity check...")
if (not config_checker.checkMasterConfigSanity(cfg)):
return
objCfg = json.loads(cfg)
debug("Loading Supervisors...")
return loadUnits(objCfg['units'])
def checkAction(self, config):
try:
if ("action" in config.keys()):
debug("Handling action: " + config['action'])
action = getattr(self, "action_" + config['action'])
action()
return True
except:
pass
return False
def _startNewStreamerFromExploration(self):
debug("Cycling through videos...", 3)
try:
streamerInfo = self.streamerStartQueue.get(False)
except:
return None
try:
self.startStreamer(streamerInfo)
except:
traceback.print_exc()
async def sendImage(self, imagePacket):
url = "ws://" + self.config['url']
async with websockets.connect(url) as ws:
debug("Connected to " + str(url), 3)
#await ws.send(self.config['auth_key'])
#debug("Sent authentication key", 3)
await ws.send(json.dumps(imagePacket.data))
to_send = imagePacket.binObj
await ws.send(to_send)
debug("Image sent, " + str(len(to_send)) + " bytes", 3)
def download_video(url, path):
debug("Prepare video loading (" + url + ")", 3)
tmp_path = urllib.request.urlretrieve(url)[0]
video_info = pymediainfo.MediaInfo.parse(tmp_path)
video_type = video_info.to_data()["tracks"][0]["internet_media_type"]
if "video" not in video_type:
error("The downloaded file is not a valid video (" + url + ")")
return
path += "." + video_type.split("/")[1]
shutil.move(tmp_path, path)
debug("Video saved in " + path, 0)
def _connect(self):
#prof.enter("CONNECTION")
try:
sck = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sck.settimeout(2.5)
sck.connect(self.addr)
debug("Connected to " + str(self.addr))
return sck
except:
if (_DEBUG_LEVEL >= 3):
traceback.print_exc()
return None
def setup(self, data):
self.debug_boxing = False
if type(data) == type({}):
self.debug_boxing = data.get("debug_boxing")
config = b"cfg/yolo9000.cfg"
weights = b"weights/yolo9000.weights"
meta = b"cfg/combine9k.data"
self.detector = Detector(config, weights, meta)
ok("Starting boxing identifier.")
debug("cfg="+str(config)+" weights="+str(weights)+" meta="+str(meta), 3)
def launchJob(self):
if (self.job == None):
raise ValueError("This worker has no job")
if (self.jobRunning.value):
raise AssertionError("Process already running")
self.jobRunning.value = True
self.jobThread = Thread(target=self._launchTarget, daemon=True)
self.jobThread.start()
debug("Job is running", 1)
def _recFolderExploration(self, folderPath, streamerInfo):
for b in listdir(folderPath):
file = os.path.abspath(folderPath) + os.path.sep + b
if (os.path.isfile(file)):
if (os.path.splitext(file)[1]
in self.options[self.VIDEO_EXTENSION_TAG]):
info = copy.copy(streamerInfo)
info['path'] = file
info['name'] = info['name'] + os.path.basename(file)
self.streamerStartQueue.put(info)
debug("Found: " + file, 3)
else:
self._recFolderExploration(file, streamerInfo)
def handleConfigInput(self, workerConfig):
cfgObj = json.loads(workerConfig)
if(not 'workername' in cfgObj):
raise ValueError('The worker name is mandatory')
name = cfgObj['workername']
action = False
if("action" in cfgObj.keys()):
debug("Got action for worker, skipping config check")
action = True
else:
debug("Checking config...")
if(not config_checker.checkWorkerConfigSanity(workerConfig)):
return
debug("Config is OK")
if(name in self.workers):
debug("Worker found: "+name)
self._sendToWorker(name, workerConfig)
else:
if(action):
raise ValueError('The worker must exist to pass an action to it')
self.startWorker(workerConfig, name)
time.sleep(1)
def loadJob(self, jobName):
try:
self.jobName = str(jobName)
debug("Loading Job file: " + str(jobName), 1)
mod = importlib.import_module("jobs." + jobName)
shortName = jobName.split(".")[-1]
jobCl = getattr(mod, shortName.capitalize())
debug("[WARN] WARNING: Skipped subclass test", 0, True)
'''if(not issubclass(jobCl, Job)):
debug("The given job class is not a Job", 0, True)
debug("It is "+str(jobCl))
self.job = None
return '''
#difference btwn import error & load error
self.job = jobCl.__new__(jobCl)
self.job.__init__()
debug("Job loaded", 1)
except:
if (_DEBUG_LEVEL == 3):
traceback.print_exc()
debug("Could not load job", 1, True)
self.stop(1)
def loadWorkerDistributionSequence(workers, rsup):
#prof.enter("DISTRIB_LOAD")
def buildWorkerSequence(seq, workerDict, currentWorker):
if (currentWorker in seq):
return
if ("output" in workerDict[currentWorker].keys()):
output = workerDict[currentWorker]['output']
for i in range(len(output)):
out = output[i]
buildWorkerSequence(seq, workerDict, out)
trueOut = workerSuper[out].addr[0] + ":" + str(
workerDict[out]['port'])
debug("Resolved " + out + " to " + trueOut)
output[i] = trueOut
seq.append(currentWorker)
workerByName = {}
workerSuper = {}
workerSequence = []
for sup in workers.keys():
if (not sup in rsup):
raise ValueError("The requested unit is unknown: " + sup)
for worker in workers[sup]:
name = worker["workername"]
debug("Loading: " + name)
if (not checkWorkerConfig(json.dumps(worker))):
raise ValueError("Unable to load config for worker: " + name)
if (worker["workername"] in workerByName):
raise ValueError("Duplicate for worker name " + name)
workerByName[name] = worker
workerSuper[name] = rsup[sup]
debug("Got " + str(len(workerByName)) + " workers and " + str(len(rsup)) +
" supervisors")
try:
for w in workerByName.keys():
buildWorkerSequence(workerSequence, workerByName, w)
except KeyError as e:
debug("Unknown Worker: " + str(e), 0, True)
raise e
debug("Worker ignition sequence is " + str(workerSequence))
return (workerSequence, workerSuper, workerByName)
def buildWorkerSequence(seq, workerDict, currentWorker):
if(currentWorker in seq):
return
if("output" in workerDict[currentWorker].keys()):
output = workerDict[currentWorker]['output']
for i in range(len(output)):
out = output[i]
buildWorkerSequence(seq, workerDict, out)
trueOut = workerSuper[out].addr[0]+":"+str(workerDict[out]['port'])
debug("Resolved "+out+" to "+trueOut)
output[i] = trueOut
seq.append(currentWorker)
def _sendTo(self, sock, p):
binChan = self.outputs[sock]
try:
p.send(binChan)
#Network lock management
self.outputWorkerLocks[sock].clear()
self.globalOutputLock.clear()
except:
if (_DEBUG_LEVEL == 3):
traceback.print_exc()
debug("Output Connection was lost", 0, True)
self._closeSock(sock)
self.brokenOutputs.append(sock)
def _clientInTarget(self, sock):
binChan = sock.makefile("wrb") #w for sending back ack
self.inputConnections[sock] = binChan
try:
while (not self.workerShutdown.value):
p = Packet()
p.read(binChan)
self.inputQueue.put(p) #hold for next packet if Queue is full
except:
if (_DEBUG_LEVEL == 3):
traceback.print_exc()
debug("Incoming Connection was lost", 0, True)
finally:
self._closeSock(sock)
del self.inputConnections[sock]
def _listenTarget(self):
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.server.bind(('', config.SUPERVISOR_PORT))
self.server.listen(4)
ok("Started Supervisor Server")
while (self.running):
client, addr = self.server.accept()
debug("Connection from " + str(addr), 1)
Thread(target=self._clientTarget, args=(client, )).start()
except:
error("Supervisor Server Shutting down")
traceback.print_exc()
self.server.close()
suicide()
def _readBinObject(self, binChan, binSize):
b = b''
r = 0
debug("[NETWORK] Reading bin object of " + str(binSize) + " bytes", 3)
bufSize = binSize if self.BIN_RECV_FULL else self.BIN_READ_MAX
st = time.time()
while (r < binSize):
if (len(b) + bufSize > binSize):
bufSize = binSize - len(b)
a = binChan.read(bufSize)
b += a
r += len(a)
debug("READ " + str(len(b)) + " in " + str(time.time() - st), 3)
self.binObj = b
