class ResourceProvider(WorkerResource):
def __init__(self, resourceQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._reqQ = resourceQ['reqQ']
self._routerQ = resourceQ['routerQ']
self._resourceQ = multiprocessing.Queue()
self._networkObj = None
WorkerResource.__init__(self, self._logger)
def __del__(self):
self._logger.warn('@ terminate process : %s' % PROCESS_NAME)
self.clearResource()
def doProcess(self):
try:
while True:
reqMsg = self._reqQ.get()
print "[Q] ResourceProvider :", reqMsg
protocol, statCode = parseProtocol(reqMsg)
if protocol == 'SYS_REQ_RESOURCE':
self._routerQ.put_nowait(
genResWorkerResource(reqMsg['workerId'],
self.getCurrentResource()))
except KeyboardInterrupt, e:
pass
class ResourceMaster(Resources):
def __init__(self, resourceQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._reqQ = resourceQ['reqQ']
self._routerQ = resourceQ['routerQ']
Resources.__init__(self, self._logger)
def __del__(self):
self._logger.warn('@ terminate process : %s' % PROCESS_NAME)
def doProcess(self):
try:
while True:
reqMsg = self._routerQ.get()
print "[Q] ResourceMaster :", reqMsg
protocol, statCode = parseProtocol(reqMsg)
if protocol == 'SYS_SET_RESOURCE':
self.initResource(reqMsg['workerId'], reqMsg['resource'])
elif protocol == 'SYS_DEL_RESOURCE':
self.delResource(reqMsg['workerId'])
print '[RESOURCE] resource size : %d' % (len(self._resources))
except KeyboardInterrupt, e:
pass
class DpuMaster:
def __init__(self):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' %PROCESS_NAME)
self._dpuProcesses = []
def _createMsgQueue(self):
clientQ = {
'reqQ' : multiprocessing.Queue(),
'routerQ' : multiprocessing.Queue()
}
deployQ = {
'reqQ' : multiprocessing.Queue(),
'routerQ' : multiprocessing.Queue()
}
jobQ = {
'reqQ' : multiprocessing.Queue(),
'routerQ' : multiprocessing.Queue()
}
resourceQ = {
'reqQ' : multiprocessing.Queue(),
'routerQ' : multiprocessing.Queue()
}
networkQ = {
'reqQ' : multiprocessing.Queue(),
'routerQ' : multiprocessing.Queue()
}
return clientQ, deployQ, jobQ, resourceQ, networkQ
def runDpuMaster(self):
clientQ, deployQ, jobQ, resourceQ, networkQ = self._createMsgQueue()
self._dpuProcesses.append(multiprocessing.Process(target=self._runClientListener, args=(clientQ,)))
self._dpuProcesses.append(multiprocessing.Process(target=self._runDeployManager, args=(deployQ,)))
self._dpuProcesses.append(multiprocessing.Process(target=self._runJobManager, args=(jobQ,)))
self._dpuProcesses.append(multiprocessing.Process(target=self._runResourceMaster, args=(resourceQ,)))
self._dpuProcesses.append(multiprocessing.Process(target=self._runNetworkManager, args=(networkQ,)))
for process in self._dpuProcesses:
process.daemon = True
process.start()
qRouter = QRouter(self._logger, self._dpuProcesses, clientQ, deployQ, jobQ, resourceQ, networkQ)
qRouter.doProcess()
def _runClientListener(self, clientQ):
ClientListener(clientQ).runServer()
def _runDeployManager(self, deployQ):
DeployManager(deployQ).doProcess()
def _runJobManager(self, jobQ):
JobManager(jobQ).doProcess()
def _runResourceMaster(self, resourceQ):
ResourceMaster(resourceQ).doProcess()
def _runNetworkManager(self, networkQ):
MasterNetworkManager(networkQ).doProcess()
def __init__(self, resourceQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._reqQ = resourceQ['reqQ']
self._routerQ = resourceQ['routerQ']
self._networkQ = multiprocessing.Queue()
self._tempProviders = {}
self._providers = {}
def __init__(self, workerId, networkQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._reqQ = networkQ['reqQ']
self._routerQ = networkQ['routerQ']
self._networkQ = multiprocessing.Queue()
self._workerId = workerId
self._networkObj = None
def __init__(self, logger=None):
if logger != None:
self._logger = logger
else:
self._logger = Logger().getLogger()
optionParser = OptParser(self._logger)
self._orderSheet = optionParser.getOrderSheet()
def __init__(self, resourceQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._reqQ = resourceQ['reqQ']
self._routerQ = resourceQ['routerQ']
self._resourceQ = multiprocessing.Queue()
self._networkObj = None
WorkerResource.__init__(self, self._logger)
class MainProcess:
def __init__(self):
self._logger = Logger(PROCESS_NAME).getLogger()
def doProcess(self):
self._logger.info("Show logger info format")
self._logger.debug("Show logger debug format")
self._logger.warn("Show logger warn format")
self._logger.critical("Show logger critical format")
self._logger.exception("Show logger except format")
class ClientListener:
def __init__(self, clientQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._clientQ = clientQ
self._reqQ = clientQ['reqQ']
self._routerQ = clientQ['routerQ']
def __del__(self):
self._logger.warn('@ terminate process : %s' % PROCESS_NAME)
def runServer(self):
try:
hostIp, hostPort = getHostInfo(PROCESS_NAME)
svrsock = socket(AF_INET, SOCK_STREAM)
svrsock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
svrsock.bind((hostIp, hostPort))
svrsock.listen(5)
while True:
socketObj, addr = svrsock.accept()
clientListener = Thread(target=self._bindClientListener,
args=(socketObj, addr))
clientListener.setDaemon(1)
clientListener.start()
self._logger.info("# connected client")
self._logger.info("- client addr : %s, port : %d" %
(addr[0], addr[1]))
except KeyboardInterrupt, ki:
pass
class ProcessManager:
def __init__(self, logger=None):
if logger != None:
self._logger = logger
else:
self._logger = Logger().getLogger()
optionParser = OptParser(self._logger)
self._orderSheet = optionParser.getOrderSheet()
def doProcess(self, recoveryOptions=None):
try:
self._logger.info("# doProcess")
except Exception, e:
self._logger.exception(e)
sys.exit(1)
self._logger.info("# job completed")
class DeployManager:
def __init__(self, deployQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' %PROCESS_NAME)
self._reqQ = deployQ['reqQ']
self._routerQ = deployQ['routerQ']
def __del__(self):
self._logger.warn('@ terminate process : %s' %PROCESS_NAME)
def doProcess(self):
try:
while True:
reqMsg = self._routerQ.get()
print "[Q] DeployManager :", reqMsg
print reqMsg
except KeyboardInterrupt, e:
pass
class JobWorkerManager:
def __init__(self, jobQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' %PROCESS_NAME)
self._reqQ = jobQ['reqQ']
self._routerQ = jobQ['routerQ']
def __del__(self):
self._logger.warn('@ terminate process : %s' %PROCESS_NAME)
def doProcess(self):
try:
self._reqQ.put_nowait("[%s] : message test" %PROCESS_NAME)
while True:
reqMsg = self._routerQ.get()
print "[Q] JobWorkerManager :", reqMsg
except KeyboardInterrupt, e:
pass
class JobManager:
def __init__(self, jobQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._reqQ = jobQ['reqQ']
self._routerQ = jobQ['routerQ']
def __del__(self):
self._logger.warn('@ terminate process : %s' % PROCESS_NAME)
def doProcess(self):
try:
self._loadJobs()
while True:
reqMsg = self._routerQ.get()
print "[Q] JobManager(SYS) :", reqMsg
protocol, statCode = parseProtocol(reqMsg)
except KeyboardInterrupt, e:
pass
class MasterNetworkManager:
def __init__(self, resourceQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._reqQ = resourceQ['reqQ']
self._routerQ = resourceQ['routerQ']
self._networkQ = multiprocessing.Queue()
self._tempProviders = {}
self._providers = {}
def __del__(self):
self._logger.warn('@ terminate process : %s' % PROCESS_NAME)
for tempId in self._tempProviders.keys():
self._tempProviders[tempId].close()
for workerId in self._providers.keys():
self._providers[workerId].close()
def _sendMsg(self, **args):
networkObj = None
try:
if args.has_key('tempId'):
networkObj = self._tempProviders[args['tempId']]
elif args.has_key('workerId'):
networkObj = self._providers[args['workerId']]
except KeyError, e:
self._logger.warn('# key error : %s' % (str(args)))
if not networkObj:
self._logger.warn('# network object not exist.')
return False
return networkObj.sendMsg(args)
class WorkerNetworkManager:
def __init__(self, workerId, networkQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._reqQ = networkQ['reqQ']
self._routerQ = networkQ['routerQ']
self._networkQ = multiprocessing.Queue()
self._workerId = workerId
self._networkObj = None
def __del__(self):
self._logger.warn('@ terminate process : %s' % PROCESS_NAME)
def doProcess(self):
try:
connGatewayQHandler = Thread(target=self._runConnectionGateway,
args=())
connGatewayQHandler.setDaemon(1)
connGatewayQHandler.start()
self._connectMasterServer()
while True:
reqMsg = self._reqQ.get()
print "[Q] WorkerNetworkManager(SYS) :", reqMsg
protocol, statCode = parseProtocol(reqMsg)
if protocol == 'SYS_GET_WID':
tempId = reqMsg['tempId']
self._workerId = reqMsg['workerId']
self._networkQ.put_nowait(
genSetWid(reqMsg['tempId'], reqMsg['workerId']))
elif protocol == 'SYS_RES_RESOURCE':
self._networkQ.put_nowait(
genSetWorkerResource(reqMsg['workerId'],
reqMsg['result']))
except KeyboardInterrupt, e:
pass
def __init__(self):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' %PROCESS_NAME)
self._dpuProcesses = []
def __init__(self, deployQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._reqQ = deployQ['reqQ']
self._routerQ = deployQ['routerQ']
def __init__(self): self._logger = Logger(PROCESS_NAME).getLogger()
class DpuWorker:
def __init__(self):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._workerId = "WK_%d%X" % (random.randrange(
1, 9) * random.randrange(100, 999), int(time.time() * 100000))
self._dpuProcesses = []
def _createMsgQueue(self):
deployQ = {
'reqQ': multiprocessing.Queue(),
'routerQ': multiprocessing.Queue()
}
jobQ = {
'reqQ': multiprocessing.Queue(),
'routerQ': multiprocessing.Queue()
}
resourceQ = {
'reqQ': multiprocessing.Queue(),
'routerQ': multiprocessing.Queue()
}
networkQ = {
'reqQ': multiprocessing.Queue(),
'routerQ': multiprocessing.Queue()
}
return deployQ, jobQ, resourceQ, networkQ
def runDpuWorker(self):
deployQ, jobQ, resourceQ, networkQ = self._createMsgQueue()
self._dpuProcesses.append(
multiprocessing.Process(target=self._runJobDeployer,
args=(deployQ, )))
self._dpuProcesses.append(
multiprocessing.Process(target=self._runJobWorker, args=(jobQ, )))
self._dpuProcesses.append(
multiprocessing.Process(target=self._runResourceProvider,
args=(resourceQ, )))
self._dpuProcesses.append(
multiprocessing.Process(target=self._runNetworkManager,
args=(networkQ, )))
for process in self._dpuProcesses:
process.daemon = True
process.start()
qRouter = QRouter(self._logger, self._dpuProcesses, deployQ, jobQ,
resourceQ, networkQ)
qRouter.doProcess(self._workerId)
def _runJobDeployer(self, deployQ):
JobDeployer(deployQ).doProcess()
def _runJobWorker(self, jobQ):
JobWorkerManager(jobQ).doProcess()
def _runResourceProvider(self, resourceQ):
ResourceProvider(resourceQ).doProcess()
def _runNetworkManager(self, networkQ):
WorkerNetworkManager(self._workerId, networkQ).doProcess()
class OutputManager:
def __init__(self, outputQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('---------- [ Output Manager ] ----------')
self._dataManager = DataServiceManager(self._logger)
self._outputQ = outputQ
def run(self):
while True:
self._logger.info("# Ready to run for nilm usage insert #")
outputMsg = self._outputQ.get()
try:
if outputMsg['jobType'] == 'USAGE' or outputMsg[
'jobType'] == 'usage':
dailyBasedTS = outputMsg['dbTS']
sid = outputMsg['sid']
did = outputMsg['did']
lfid = outputMsg['lfid']
hourlyUsage = outputMsg['hourlyUsage']
dailyUsage = outputMsg['dailyUsage']
hourlyAppOn = outputMsg['hourlyAppOn']
dailyAppOn = outputMsg['dailyAppOn']
self._logger.info(">" * 50)
self._logger.info(
'# Q size : %d, sid : %d, did : %d, lfid : %d' %
(self._outputQ.qsize(), sid, did, lfid))
self._logger.info("<" * 50)
# self._logger.debug(' - hourlyUsage : %s' %str(hourlyUsage))
# self._logger.debug(' - dailyUsage : %s' %str(dailyUsage))
# self._logger.debug(' - hourlyAppOn : %s' %str(hourlyAppOn))
# self._logger.debug(' - dailyAppOn : %s' %str(dailyAppOn))
self._dataManager.setNilmHourlyFeederUsage(
dailyBasedTS, sid, did, lfid, hourlyUsage, hourlyAppOn)
self._dataManager.setNilmDailyFeederUsage(
dailyBasedTS, sid, did, lfid, dailyUsage, dailyAppOn)
time.sleep(0.2)
except Exception, e:
self._logger.error("# output message : %s" % str(outputMsg))
self._logger.exception(e)
self._outputQ.put_nowait(outputMsg)
time.sleep(1)
def __init__(self, clientQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._clientQ = clientQ
self._reqQ = clientQ['reqQ']
self._routerQ = clientQ['routerQ']
def __init__(self):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._workerId = "WK_%d%X" % (random.randrange(
1, 9) * random.randrange(100, 999), int(time.time() * 100000))
self._dpuProcesses = []
def __init__(self, resourceQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('# process start : %s' % PROCESS_NAME)
self._reqQ = resourceQ['reqQ']
self._routerQ = resourceQ['routerQ']
Resources.__init__(self, self._logger)
def __init__(self, outputQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('---------- [ Output Manager ] ----------')
self._dataManager = DataServiceManager(self._logger)
self._outputQ = outputQ
