def unify(self, req): try: print(req["unify"]) response = http.request("GET", req["unify"] + '/api/2.0/recording', fields=req) items = json.loads(response.data.decode('utf8')) debug( "Connected to " + req["unify"] + "(" + str(len(items['data'])) + " available videos)", 1) except: debug("Unable to connect to " + req["unify"], 0) return None for r in items['data']: if "locked" in req: if req["locked"] == True and r["locked"] == False: continue requests.append({ "name": r["meta"]["cameraName"] + "-" + str(r["endTime"]), "url": req["unify"] + '/api/2.0/recording/' + r["_id"] + '/download?apiKey=' + req["apiKey"], "startTime": r["startTime"], "endTime": r["endTime"] })
def get_image(self): self.psProcess.resume() size = self.shape[0] * self.shape[1] * 3 raw_image = self.pipe.stdout.read(size) while np.array_equal(self.raw_image_prev, raw_image) is True: raw_image = self.pipe.stdout.read(size) self.img_count += 1 debug("Drop frame from ffmpeg (duplicated)", 3) self.raw_image_prev = np.copy(raw_image) #raw_image = self.pipe.stdout.read(size) image = np.fromstring(raw_image, dtype='uint8') if image.shape[0] == 0: return None image = image.reshape((self.shape[1], self.shape[0], 3)) self.pipe.stdout.flush() self.psProcess.suspend() if (self.doResize): image = np.array( Image.fromarray(image, 'RGB').resize(self.resolution)) self.img_count += 1 return image
def _launchTarget(self): try: self._doJob() except: debug("Error from job thread", 0, True) traceback.print_exc() self.stop(1)
def startStreamer(self, streamerInfo): if (not checkStreamerConfigSanity(json.dumps(streamerInfo))): raise ValueError("Error in configuration") resolution = getWithDefault(streamerInfo, "resolution", self.options[self.DEFAULT_RESOLUTION_TAG]) img_rate = getWithDefault(streamerInfo, "img_rate", self.options[self.DEFAULT_IMG_RATE_TAG]) name = getWithDefault(streamerInfo, "name", "streamer" + str(self.streamerCount)) location = getWithDefault(streamerInfo, "url") realtime = getWithDefault(streamerInfo, "realtime", self.options[self.REALTIME_TAG]) if (location == None): location = getWithDefault(streamerInfo, "path") try: if (realtime): streamer = RealTimeStreamer(name, location, img_rate, resolution) else: streamer = Streamer(name, location, img_rate, resolution) except: debug("Cannot start streamer " + name + " (" + str(location) + ")", 2, True) if (_DEBUG_LEVEL >= 3): traceback.print_exc() return self.streamers.append(streamer) self.streamerCount += 1
def _listenTarget(self): try: server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server.bind(('', self.port)) server.listen(4) debug("Listening on " + str(self.port)) except: traceback.print_exc() self.stop(1) return self.server = server try: while (not self.workerShutdown.value): client, addr = server.accept() if (client != None): debug("Input from: " + str(addr), 1) Thread(target=self._clientInTarget, args=(client, ), daemon=True).start() except: if (self.workerShutdown.value): return traceback.print_exc() if (not self.jobRunning.value): return
def loop(self, data): img = None while type(img) == type(None): if (self.streamerIndex >= len(self.streamers)): self.streamerIndex = 0 if (len(self.streamers) == 0): return None streamer = self.streamers[self.streamerIndex] img = streamer.get_image() if (type(img) == type(None)): self.streamers.remove(streamer) print("Streamer " + streamer.name + " is done reading") self._startNewStreamerFromExploration() if (len(self.streamers) == 0): debug("Reached end of multistreamer job, successfull exit", 2) self.shouldStop = True return None img2 = np.array(Image.fromarray(img)) self.streamerIndex += 1 return { "from": str(streamer.name), "frame_count": str(streamer.img_count), "img": img2 }
def _clientTarget(self, sock): chan = sock.makefile("rwb") while(self.running): try: j = network.readString(chan) out = network.OK try: sa = self._detectSpecialAction(j) if(sa != None): out = sa else: self.handleConfigInput(j) network.sendString(chan, out) except Exception as e: network.sendString(chan, repr(e)) raise e except struct.error: break except: traceback.print_exc() break debug("Closing client connection", 1, True) sock.shutdown(socket.SHUT_RDWR) sock.close()
def push(self, obj, throwOnError = True): #prof.enter("PUSH") debug("Pushing to "+self.name,2) sock = self._connect() chan = sock.makefile("rwb") network.sendString(chan, json.dumps(obj)) debug("Awaiting reply...",2) answ = network.readString(chan) or "SOCK_TIMEOUT" try: self._close(sock) except: pass if(answ != network.OK): if(throwOnError): error("Push failure", 0) error("Refer to supervisor log for details", 0) error("Err was:"+answ,0) return False ok("Configuration "+ obj["workername"] + "("+ obj["jobname"]+") on " + self.name) return answ
def handleClose(self): debug(str(self.address) + ' closed', 2) if self in clients: clients.remove(self) if self in clients_debug: clients_debug.remove(self)
def setupJob(self, data): if (self.job == None): raise ValueError("This worker has no job") self.job.setup(data) self.jobSetup = True self.job.shouldStop = False debug("Pushing data", 1)
def setNetworkMethod(self, config): if ("outputmethod" in config): if (config['outputmethod'] == "distribute"): debug("Output method is set to distribution") self.outputmethod = self._distributeOverNetwork else: debug("Output method is set to duplication") self.outputmethod = self._duplicateOverNetwork
def _testConnection(self): debug("Testing connection to: " + str(self.addr)) s = self._connect() if (s == None): return False self._close(s) return True
def setup(self, data): config = b"cfg/yolo9000.cfg" weights = b"weights/yolo9000.weights" meta = b"cfg/combine9k.data" self.detector = Detector(config, weights, meta) debug( "Started detector cfg=" + str(config) + " weights=" + str(weights) + " meta=" + str(meta), 3)
def loadSupervisors(cfg): debug("Master config sanity check...") if (not config_checker.checkMasterConfigSanity(cfg)): return objCfg = json.loads(cfg) debug("Loading Supervisors...") return loadUnits(objCfg['units'])
def checkAction(self, config): try: if ("action" in config.keys()): debug("Handling action: " + config['action']) action = getattr(self, "action_" + config['action']) action() return True except: pass return False
def _startNewStreamerFromExploration(self): debug("Cycling through videos...", 3) try: streamerInfo = self.streamerStartQueue.get(False) except: return None try: self.startStreamer(streamerInfo) except: traceback.print_exc()
async def sendImage(self, imagePacket): url = "ws://" + self.config['url'] async with websockets.connect(url) as ws: debug("Connected to " + str(url), 3) #await ws.send(self.config['auth_key']) #debug("Sent authentication key", 3) await ws.send(json.dumps(imagePacket.data)) to_send = imagePacket.binObj await ws.send(to_send) debug("Image sent, " + str(len(to_send)) + " bytes", 3)
def download_video(url, path): debug("Prepare video loading (" + url + ")", 3) tmp_path = urllib.request.urlretrieve(url)[0] video_info = pymediainfo.MediaInfo.parse(tmp_path) video_type = video_info.to_data()["tracks"][0]["internet_media_type"] if "video" not in video_type: error("The downloaded file is not a valid video (" + url + ")") return path += "." + video_type.split("/")[1] shutil.move(tmp_path, path) debug("Video saved in " + path, 0)
def _connect(self): #prof.enter("CONNECTION") try: sck = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sck.settimeout(2.5) sck.connect(self.addr) debug("Connected to " + str(self.addr)) return sck except: if (_DEBUG_LEVEL >= 3): traceback.print_exc() return None
def setup(self, data): self.debug_boxing = False if type(data) == type({}): self.debug_boxing = data.get("debug_boxing") config = b"cfg/yolo9000.cfg" weights = b"weights/yolo9000.weights" meta = b"cfg/combine9k.data" self.detector = Detector(config, weights, meta) ok("Starting boxing identifier.") debug("cfg="+str(config)+" weights="+str(weights)+" meta="+str(meta), 3)
def launchJob(self): if (self.job == None): raise ValueError("This worker has no job") if (self.jobRunning.value): raise AssertionError("Process already running") self.jobRunning.value = True self.jobThread = Thread(target=self._launchTarget, daemon=True) self.jobThread.start() debug("Job is running", 1)
def _recFolderExploration(self, folderPath, streamerInfo): for b in listdir(folderPath): file = os.path.abspath(folderPath) + os.path.sep + b if (os.path.isfile(file)): if (os.path.splitext(file)[1] in self.options[self.VIDEO_EXTENSION_TAG]): info = copy.copy(streamerInfo) info['path'] = file info['name'] = info['name'] + os.path.basename(file) self.streamerStartQueue.put(info) debug("Found: " + file, 3) else: self._recFolderExploration(file, streamerInfo)
def handleConfigInput(self, workerConfig): cfgObj = json.loads(workerConfig) if(not 'workername' in cfgObj): raise ValueError('The worker name is mandatory') name = cfgObj['workername'] action = False if("action" in cfgObj.keys()): debug("Got action for worker, skipping config check") action = True else: debug("Checking config...") if(not config_checker.checkWorkerConfigSanity(workerConfig)): return debug("Config is OK") if(name in self.workers): debug("Worker found: "+name) self._sendToWorker(name, workerConfig) else: if(action): raise ValueError('The worker must exist to pass an action to it') self.startWorker(workerConfig, name) time.sleep(1)
def loadJob(self, jobName): try: self.jobName = str(jobName) debug("Loading Job file: " + str(jobName), 1) mod = importlib.import_module("jobs." + jobName) shortName = jobName.split(".")[-1] jobCl = getattr(mod, shortName.capitalize()) debug("[WARN] WARNING: Skipped subclass test", 0, True) '''if(not issubclass(jobCl, Job)): debug("The given job class is not a Job", 0, True) debug("It is "+str(jobCl)) self.job = None return ''' #difference btwn import error & load error self.job = jobCl.__new__(jobCl) self.job.__init__() debug("Job loaded", 1) except: if (_DEBUG_LEVEL == 3): traceback.print_exc() debug("Could not load job", 1, True) self.stop(1)
def loadWorkerDistributionSequence(workers, rsup): #prof.enter("DISTRIB_LOAD") def buildWorkerSequence(seq, workerDict, currentWorker): if (currentWorker in seq): return if ("output" in workerDict[currentWorker].keys()): output = workerDict[currentWorker]['output'] for i in range(len(output)): out = output[i] buildWorkerSequence(seq, workerDict, out) trueOut = workerSuper[out].addr[0] + ":" + str( workerDict[out]['port']) debug("Resolved " + out + " to " + trueOut) output[i] = trueOut seq.append(currentWorker) workerByName = {} workerSuper = {} workerSequence = [] for sup in workers.keys(): if (not sup in rsup): raise ValueError("The requested unit is unknown: " + sup) for worker in workers[sup]: name = worker["workername"] debug("Loading: " + name) if (not checkWorkerConfig(json.dumps(worker))): raise ValueError("Unable to load config for worker: " + name) if (worker["workername"] in workerByName): raise ValueError("Duplicate for worker name " + name) workerByName[name] = worker workerSuper[name] = rsup[sup] debug("Got " + str(len(workerByName)) + " workers and " + str(len(rsup)) + " supervisors") try: for w in workerByName.keys(): buildWorkerSequence(workerSequence, workerByName, w) except KeyError as e: debug("Unknown Worker: " + str(e), 0, True) raise e debug("Worker ignition sequence is " + str(workerSequence)) return (workerSequence, workerSuper, workerByName)
def buildWorkerSequence(seq, workerDict, currentWorker): if(currentWorker in seq): return if("output" in workerDict[currentWorker].keys()): output = workerDict[currentWorker]['output'] for i in range(len(output)): out = output[i] buildWorkerSequence(seq, workerDict, out) trueOut = workerSuper[out].addr[0]+":"+str(workerDict[out]['port']) debug("Resolved "+out+" to "+trueOut) output[i] = trueOut seq.append(currentWorker)
def _sendTo(self, sock, p): binChan = self.outputs[sock] try: p.send(binChan) #Network lock management self.outputWorkerLocks[sock].clear() self.globalOutputLock.clear() except: if (_DEBUG_LEVEL == 3): traceback.print_exc() debug("Output Connection was lost", 0, True) self._closeSock(sock) self.brokenOutputs.append(sock)
def _clientInTarget(self, sock): binChan = sock.makefile("wrb") #w for sending back ack self.inputConnections[sock] = binChan try: while (not self.workerShutdown.value): p = Packet() p.read(binChan) self.inputQueue.put(p) #hold for next packet if Queue is full except: if (_DEBUG_LEVEL == 3): traceback.print_exc() debug("Incoming Connection was lost", 0, True) finally: self._closeSock(sock) del self.inputConnections[sock]
def _listenTarget(self): self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: self.server.bind(('', config.SUPERVISOR_PORT)) self.server.listen(4) ok("Started Supervisor Server") while (self.running): client, addr = self.server.accept() debug("Connection from " + str(addr), 1) Thread(target=self._clientTarget, args=(client, )).start() except: error("Supervisor Server Shutting down") traceback.print_exc() self.server.close() suicide()
def _readBinObject(self, binChan, binSize): b = b'' r = 0 debug("[NETWORK] Reading bin object of " + str(binSize) + " bytes", 3) bufSize = binSize if self.BIN_RECV_FULL else self.BIN_READ_MAX st = time.time() while (r < binSize): if (len(b) + bufSize > binSize): bufSize = binSize - len(b) a = binChan.read(bufSize) b += a r += len(a) debug("READ " + str(len(b)) + " in " + str(time.time() - st), 3) self.binObj = b