def _launchTarget(self):
try:
self._doJob()
except:
traceback.print_exc()
error("Error from job thread")
self.stop(1)
def loadWorkerDistributionSequence(workers, rsup):
#prof.enter("DISTRIB_LOAD")
def buildWorkerSequence(seq, workerDict, currentWorker):
if(currentWorker in seq):
return
if("output" in workerDict[currentWorker].keys()):
output = workerDict[currentWorker]['output']
for i in range(len(output)):
out = output[i]
buildWorkerSequence(seq, workerDict, out)
trueOut = workerSuper[out].addr[0]+":"+str(workerDict[out]['port'])
debug("Resolved "+out+" to "+trueOut)
output[i] = trueOut
seq.append(currentWorker)
workerByName = {}
workerSuper = {}
workerSequence = []
for sup in workers.keys():
if(not sup in rsup):
raise ValueError("The requested unit is unknown: "+sup)
for worker in workers[sup]:
name = worker["workername"]
debug("Loading: "+name)
if(not checkWorkerConfig(json.dumps(worker))):
raise ValueError("Unable to load config for worker: "+name)
if(worker["workername"] in workerByName):
raise ValueError("Duplicate for worker name "+name)
workerByName[name] = worker
workerSuper[name] = rsup[sup]
debug("Got "+str(len(workerByName))+" workers and "+str(len(rsup))+" supervisors")
try:
for w in workerByName.keys():
buildWorkerSequence(workerSequence, workerByName, w)
except KeyError as e:
error("Unknown Worker: "+str(e), 0)
raise e
debug("Worker ignition sequence is "+str(workerSequence), 2)
return (workerSequence, workerSuper, workerByName)
def setup(self, data):
try:
if os.path.isdir(data["storage_path"]) is False:
raise Exception('Invalid storage path.')
except:
error(
"A valid storage path must be defined, please read the doc. ("
+ data["storage_path"] + ")")
sys.exit(-1)
self.storage_path = data["storage_path"]
ssl._create_default_https_context = ssl._create_unverified_context
def download_video(url, path):
debug("Prepare video loading (" + url + ")", 3)
tmp_path = urllib.request.urlretrieve(url)[0]
video_info = pymediainfo.MediaInfo.parse(tmp_path)
video_type = video_info.to_data()["tracks"][0]["internet_media_type"]
if "video" not in video_type:
error("The downloaded file is not a valid video (" + url + ")")
return
path += "." + video_type.split("/")[1]
shutil.move(tmp_path, path)
debug("Video saved in " + path, 0)
def _listenTarget(self):
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.server.bind(('', config.SUPERVISOR_PORT))
self.server.listen(4)
ok("Started Supervisor Server")
while (self.running):
client, addr = self.server.accept()
debug("Connection from " + str(addr), 1)
Thread(target=self._clientTarget, args=(client, )).start()
except:
error("Supervisor Server Shutting down")
traceback.print_exc()
self.server.close()
suicide()
def push(self, obj, throwOnError = True):
#prof.enter("PUSH")
debug("Pushing to "+self.name,2)
sock = self._connect()
chan = sock.makefile("rwb")
network.sendString(chan, json.dumps(obj))
debug("Awaiting reply...",2)
answ = network.readString(chan) or "SOCK_TIMEOUT"
try:
self._close(sock)
except:
pass
if(answ != network.OK):
if(throwOnError):
error("Push failure", 0)
error("Refer to supervisor log for details", 0)
error("Err was:"+answ,0)
return False
ok("Configuration "+ obj["workername"] + "("+ obj["jobname"]+") on " + self.name)
return answ
def loadJob(self, jobName):
try:
self.jobName = str(jobName)
debug("Loading Job file: " + str(jobName), 1)
mod = importlib.import_module("jobs." + jobName)
shortName = jobName.split(".")[-1]
jobCl = getattr(mod, shortName.capitalize())
#difference btwn import error & load error
self.job = jobCl.__new__(jobCl)
self.job.__init__()
if (not self.job.isJob):
error(str(jobCl) + " is not a valid Job")
self.job = None
self.action_halt()
return
debug("Job loaded", 1)
except:
if (_DEBUG_LEVEL == 3):
traceback.print_exc()
error("Could not load job")
self.stop(1)
except: #Queue has been closed
run = False
if (worker != None and worker._exitCode !=
None): #exception from the forced close on stdin
code = worker._exitCode
else:
debug("[WARNING] Caught CTRL+C event, stopping")
code = 35
sys.exit(code)
# getting input
#-------------------------------------
# handling input
config = json.loads(line)
if (worker == None):
worker = setupWithJsonConfig(config, inputQueue)
else:
worker.updateWithDiff(config)
except SystemExit as e:
raise e #must be transmitted
except:
worker = None
run = False
traceback.print_exc()
error("Uncaught exception, abort", 0)
suicide() #cannot stay in this state
_fields_ = [("w", c_int),
("h", c_int),
("c", c_int),
("data", POINTER(c_float))]
class METADATA(Structure):
_fields_ = [("classes", c_int),
("names", POINTER(c_char_p))]
#lib = CDLL("/home/pjreddie/documents/darknet/libdarknet.so", RTLD_GLOBAL)
try:
lib = CDLL(os.path.dirname(__file__)+"/libdarknet.so", RTLD_GLOBAL)
except:
error("Unable to load darknet: " + os.path.dirname(__file__)+"/libdarknet.so")
lib.network_width.argtypes = [c_void_p]
lib.network_width.restype = c_int
lib.network_height.argtypes = [c_void_p]
lib.network_height.restype = c_int
predict = lib.network_predict
predict.argtypes = [c_void_p, POINTER(c_float)]
predict.restype = POINTER(c_float)
set_gpu = lib.cuda_set_device
set_gpu.argtypes = [c_int]
make_image = lib.make_image
make_image.argtypes = [c_int, c_int, c_int]
ok("Starting Master...")
cfg = None
if(len(sys.argv) > 1):
debug("Using config file: "+str(sys.argv[1]), 2)
fil = sys.argv[1]
cfg = read(fil)
debug(cfg, 3)
if(cfg == None):
debug("Master started, awaiting config", 1)
cfg = input().strip()
debug("Reading config...")
try:
rSup = loadSupervisors(cfg)
objCfg = json.loads(cfg)
if("action" in objCfg.keys()):
answ = pushAction(objCfg, rSup)
print(answ)
else:
pushConfig(objCfg, rSup)
except Exception as e:
error("An exception occured when executing the config", 0)
error(repr(e), 0)
if(_DEBUG_LEVEL >= 3):
traceback.print_exc()