class NilmMetaSearch:
def __init__(self, logger, jobId, jobType):
self._logger = logger
self._jobId = jobId
self._jobType = jobType
self._dataManager = DataServiceManager(logger)
def doProcess(self, hz, sid, did, lfid, feederDataFilePath, dailyBasedTS = None):
self._logger.info("# [%s] Do nilm meta search, sid : %d, did : %d, lfid : %d, data : %s, freq : %d hz" %(self._jobId, sid, did, lfid, feederDataFilePath, hz))
metaDetector = AppDetectR(self._logger)
try:
metaDetector.set_data(feederDataFilePath, hz)
isDetected = metaDetector.detect_apps()
newAppMeta = json.loads(metaDetector.getAppInfo())
if isDetected:
nilmMetaInfo = {'sid': sid, 'did': did, 'lfid': lfid, 'metaSearch':{}}
nilmMetaInfo['metaSearch'][999] = self._genFidMetaSet(71, None)
if newAppMeta:
for nfid in [int(nfid) for nfid in newAppMeta.keys()]:
vfidMeta = newAppMeta[str(nfid)]
appType = self._getApplianceType(vfidMeta)
nilmMetaInfo['metaSearch'][nfid] = self._genFidMetaSet(appType, vfidMeta)
self._dataManager.setNilmMetaSearchInfo(nilmMetaInfo)
except Exception, e:
self._logger.exception(e)
finally:
def __init__(self, logger, options, args):
self._logger = logger
self._userOptions = self._setUserOptions(options, args)
self._serviceManager = DataServiceManager(self._logger)
if self._userOptions['debugMode']:
# self._logger.setLevel(ELogger.DEBUG)
self._logger.setLevel("DEBUG")
else:
# self._logger.setLevel(ELogger.INFO)
self._logger.setLevel("INFO")
class OutputManager:
def __init__(self, outputQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('---------- [ Output Manager ] ----------')
self._dataManager = DataServiceManager(self._logger)
self._outputQ = outputQ
def run(self):
while True:
self._logger.info("# Ready to run for nilm usage insert #")
outputMsg = self._outputQ.get()
try:
if outputMsg['jobType'] == 'USAGE' or outputMsg[
'jobType'] == 'usage':
dailyBasedTS = outputMsg['dbTS']
sid = outputMsg['sid']
did = outputMsg['did']
lfid = outputMsg['lfid']
hourlyUsage = outputMsg['hourlyUsage']
dailyUsage = outputMsg['dailyUsage']
hourlyAppOn = outputMsg['hourlyAppOn']
dailyAppOn = outputMsg['dailyAppOn']
self._logger.info(">" * 50)
self._logger.info(
'# Q size : %d, sid : %d, did : %d, lfid : %d' %
(self._outputQ.qsize(), sid, did, lfid))
self._logger.info("<" * 50)
# self._logger.debug(' - hourlyUsage : %s' %str(hourlyUsage))
# self._logger.debug(' - dailyUsage : %s' %str(dailyUsage))
# self._logger.debug(' - hourlyAppOn : %s' %str(hourlyAppOn))
# self._logger.debug(' - dailyAppOn : %s' %str(dailyAppOn))
self._dataManager.setNilmHourlyFeederUsage(
dailyBasedTS, sid, did, lfid, hourlyUsage, hourlyAppOn)
self._dataManager.setNilmDailyFeederUsage(
dailyBasedTS, sid, did, lfid, dailyUsage, dailyAppOn)
time.sleep(0.2)
except Exception, e:
self._logger.error("# output message : %s" % str(outputMsg))
self._logger.exception(e)
self._outputQ.put_nowait(outputMsg)
time.sleep(1)
class NilmAppUsage:
def __init__(self, logger, jobId, jobType):
self._dataManager = DataServiceManager(logger)
self._logger = logger
self._jobId = jobId
self._jobType = jobType
def doProcess(self,
hz,
sid,
did,
lfid,
feederDataFilePath,
dailyBasedTS=None):
self._logger.info(
"# [%s] Do nilm usages, sid : %d, did : %d, lfid : %d, based date : %s, freq : %d hz"
% (self._jobId, sid, did, lfid, convertTS2Date(dailyBasedTS), hz))
self._logger.info(" - file path : %s" % feederDataFilePath)
appUsage = AppDailyLoadR(self._logger)
resultMap = {}
try:
metaInfo = self._dataManager.getNilmMetaInfo(sid, did, lfid)
appUsage.set_data(feederDataFilePath, hz)
appUsage.set_app_info(metaInfo)
appUsage.do_compute()
appUsageMap = appUsage.get_usage()
if not appUsageMap:
self._logger.warn(
'- [%s] empty appUsageMap! sid : %d, file path : %s' %
(self._jobId, sid, feederDataFilePath))
return None
if appUsageMap:
hourlyUsage, dailyUsage = self._cvtUsageData(appUsageMap)
hourlyAppOn = self._checkHourlyAppOnOff(hourlyUsage)
dailyAppOn = self._checkDailyAppOnOff(dailyUsage)
resultMap = {
'dbTS': dailyBasedTS,
'jobType': self._jobType,
'sid': sid,
'did': did,
'lfid': lfid,
'hourlyUsage': hourlyUsage,
'dailyUsage': dailyUsage,
'hourlyAppOn': hourlyAppOn,
'dailyAppOn': dailyAppOn
}
except Exception, e:
self._logger.exception(e)
self._logger.error(appUsageMap)
finally:
def __init__(self, logger, jobId, jobType):
self._logger = logger
self._jobId = jobId
self._jobType = jobType
self._dataManager = DataServiceManager(logger)
class NilmMetaUpdate:
def __init__(self, logger, jobId, jobType):
self._logger = logger
self._jobId = jobId
self._jobType = jobType
self._dataManager = DataServiceManager(logger)
def doProcess(self,
hz,
sid,
did,
lfid,
feederDataFilePath,
dailyBasedTS=None):
self._logger.info(
"# [%s] Do nilm meta search, sid : %d, did : %d, lfid : %d, data : %s, freq : %d hz"
% (self._jobId, sid, did, lfid, feederDataFilePath, hz))
existedMeta = self._dataManager.getNilmMetaInfo(sid, did, lfid)
metaDetector = AppDetectR(self._logger)
if not existedMeta:
self._logger.warn('# Meta is not be existed. sid : %d' % (sid))
return self._genResult(dailyBasedTS, sid, did, lfid, hz)
try:
metaDetector.setExistedMeta(existedMeta)
metaDetector.set_data(feederDataFilePath, hz)
isDetected = metaDetector.detect_apps()
if isDetected:
newAppMeta = json.loads(metaDetector.getAppInfo())
nilmMetaInfo = {
'sid': sid,
'did': did,
'lfid': lfid,
'metaSearch': {},
'metaUpdate': {}
}
# new meta
insertNewMetaNfds = list(
set(newAppMeta.keys()).difference(existedMeta.keys()))
for nfid in insertNewMetaNfds:
vfidMeta = newAppMeta[str(nfid)]
appType = self._getApplianceType(vfidMeta)
nilmMetaInfo['metaSearch'][int(
nfid)] = self._genFidMetaSet(appType, vfidMeta)
# update meta info
updateNewMetaNfids = list(
set(newAppMeta.keys()).intersection(existedMeta.keys()))
for nfid in updateNewMetaNfids:
vfidMeta = newAppMeta[str(nfid)]
isEnabled = 1
nilmMetaInfo['metaUpdate'][int(
nfid)] = self._genUpdateMetaSet(vfidMeta, isEnabled)
# not use meta info
updateDisableNfids = list(
set(existedMeta.keys()).difference(newAppMeta.keys()))
for nfid in updateDisableNfids:
vfidMeta = existedMeta[str(nfid)]
isEnabled = 0
nilmMetaInfo['metaUpdate'][int(
nfid)] = self._genUpdateMetaSet(vfidMeta, isEnabled)
self._dataManager.setNilmMetaUpdateInfo(nilmMetaInfo)
except Exception, e:
self._logger.exception(e)
finally:
class NilmMetaUpdate:
def __init__(self, logger, jobId, jobType):
self._logger = logger
self._jobId = jobId
self._jobType = jobType
self._dataManager = DataServiceManager(logger)
def doProcess(self, hz, sid, did, lfid, feederDataFilePath, dailyBasedTS = None):
self._logger.info("# [%s] Do nilm meta update, sid : %d, did : %d, lfid : %d, data : %s, freq : %d hz" %(self._jobId, sid, did, lfid, feederDataFilePath, hz))
oldMetaInfo = self._dataManager.getNilmMetaInfo(sid, did, lfid)
hasMiscValue = False
metaDetector = AppDetectR(self._logger)
try:
metaDetector.set_data(feederDataFilePath, hz)
isDetected = metaDetector.detect_apps()
self._logger.info("- [%s] meta detected : %s, freq : %d hz" %(self._jobId, str(isDetected), hz))
newAppMeta = json.loads(metaDetector.getAppInfo())
# self._dataManager.setUsageRouteDataService(sid)
if not newAppMeta:
self._dataManager.setNilmVirtualFeederInfo(sid, did, lfid, 999)
return
if isDetected:
existingMetaInfo = self._dataManager.getNilmMetaInfo(sid, did, lfid)
if not existingMetaInfo:
for nfid in [int(nfid) for nfid in newAppMeta.keys()]:
vfidMeta = newAppMeta[str(nfid)]
applianceType = self._getApplianceType(vfidMeta)
vfid = self._dataManager.setNilmVirtualFeederInfo(sid, did, lfid, nfid, applianceType)
self._dataManager.setNilmMetaInfo(vfid, vfidMeta)
if not hasMiscValue:
self._dataManager.setNilmVirtualFeederInfo(sid, did, lfid, 999)
hasMiscValue = True
else:
if newAppMeta:
for nfid in [int(nfid) for nfid in newAppMeta.keys()]:
vfidMeta = newAppMeta[str(nfid)]
applianceType = self._getApplianceType(vfidMeta)
vfid = self._dataManager.setNilmVirtualFeederInfo(sid, did, lfid, nfid, applianceType)
if vfid:
self._dataManager.setNilmMetaInfo(vfid, newAppMeta[str(nfid)])
if not hasMiscValue:
self._dataManager.setNilmVirtualFeederInfo(sid, did, lfid, 999)
hasMiscValue = True
except Exception, e:
self._logger.exception(e)
finally:
def __init__(self, outputQ):
self._logger = Logger(PROCESS_NAME).getLogger()
self._logger.info('---------- [ Output Manager ] ----------')
self._dataManager = DataServiceManager(self._logger)
self._outputQ = outputQ
class OptionValidator:
def __init__(self, logger, options, args):
self._logger = logger
self._userOptions = self._setUserOptions(options, args)
self._serviceManager = DataServiceManager(self._logger)
if self._userOptions['debugMode']:
# self._logger.setLevel(ELogger.DEBUG)
self._logger.setLevel("DEBUG")
else:
# self._logger.setLevel(ELogger.INFO)
self._logger.setLevel("INFO")
def _setUserOptions(self, options, args):
user_options = {}
if options.sids != None:
user_options['sids'] = options.sids.split(',')
if options.job != None:
user_options['job'] = options.job
if options.period != None:
user_options['period'] = options.period
if options.deviceProtocol != None:
user_options['deviceProtocol'] = options.deviceProtocol.split(',')
if options.groups != None:
user_options['groups'] = options.groups.split(',')
if options.fileSave != None:
user_options['fileSave'] = options.fileSave
if options.injection != None:
user_options['injection'] = options.injection
if options.remove != None:
user_options['remove'] = options.remove.split(',')
if options.numOfProcess != None:
user_options['numOfProcess'] = options.numOfProcess
if options.hasFile != None:
user_options['hasFile'] = options.hasFile
if options.onlyRawdata != None:
user_options['onlyRawdata'] = options.onlyRawdata
if options.debugMode != None:
user_options['debugMode'] = options.debugMode
if options.usagePeriod != None:
user_options['usagePeriod'] = options.usagePeriod
if options.deviceMeta != None:
user_options['deviceMeta'] = options.deviceMeta
return user_options
def doCheckOptions(self):
userOptions = {}
userOptions['debugMode'] = self._checkLogMode()
userOptions['sids'] = self._checkSiteId()
userOptions['dids'] = ['*'] #self._checkDeviceId()
userOptions['job'] = self._checkJob()
startTS, endTS, userInputPeriod = self._checkPeriod()
userOptions['startTS'] = startTS
userOptions['endTS'] = endTS
userOptions['userInputPeriod'] = userInputPeriod
userOptions['deviceProtocol'] = self._checkDeviceProtocol(userOptions)
userOptions['fileSave'] = self._checkFileSave()
userOptions['numOfProcess'] = self._checkNumOfProcess()
userOptions['injection'] = self._checkInjection()
userOptions['remove'] = self._checkRemove()
userOptions['hasFile'] = self._checkHasFile()
userOptions['onlyRawdata'] = self._checkCollectingRawDataMode(
userOptions)
userOptions['usagePeriod'] = self._checkUsagePeriod()
userOptions['deviceMeta'] = self._checkDeviceMeta()
return userOptions
def _checkSiteId(self):
tarSids = []
sids = self._userOptions.get('sids')
def isNumber(string):
try:
int(string)
return True
except ValueError:
return False
for sid in sids:
if isNumber(sid):
pass
else:
if sid == '*':
groupNames = self._userOptions.get('groups')
if not groupNames:
groupNames = getNilmGroupNames()
sids = self._serviceManager.getNilmTargetSids(groupNames)
return sids
else:
self._logger.warn(
"# Wrong site ids in the site id. Device id will be integer number or '*'."
)
sys.exit(1)
return [int(sid) for sid in sids]
def _checkDeviceId(self):
tarDids = []
dids = self._userOptions.get('dids')
def isNumber(string):
try:
int(string)
return True
except ValueError:
return False
for did in dids:
if isNumber(did):
return [did]
else:
if did == '*':
return [did]
else:
self._logger.warn(
"# Wrong device ids in the site id. Device id will be integer number or '*'."
)
sys.exit(1)
def _checkPeriod(self):
def _checkUsagePeriod(period):
if period == 'hourly':
startTS, endTS = getHourlyDatetime()
return startTS, endTS
elif period == 'daily':
startTS, endTS = getDailyDatetime()
return startTS, endTS
else:
return None, None
def _checkUserPeriod(period):
if self._userOptions.get('remove'):
return 0, 0
if not period or not len(period) == 2:
return None, None
else:
startDT = period[0]
endDT = period[1]
startTS, endTS = getPeriodTS(startDT, endDT)
return startTS, endTS
def _error():
self._logger.warn(
"# Wrong timestamp format. check your time range formats.")
sys.exit(1)
period = self._userOptions.get('period')
if period:
startTS, endTS = _checkUserPeriod(period)
userInputPeriod = True
else:
period = self._userOptions.get('usagePeriod')
startTS, endTS = _checkUsagePeriod(period)
userInputPeriod = False
if not startTS or not endTS:
_error()
if startTS > endTS:
_error()
return startTS, endTS, userInputPeriod
def _checkUsagePeriod(self):
usagePeriod = self._userOptions.get('usagePeriod')
if usagePeriod == 'hourly' or usagePeriod == 'daily':
return usagePeriod
else:
self._logger.warn(
"""# Wrong usage period. (You have to use one such as 'hourly' or 'daily'"""
)
sys.exit(1)
def _checkDeviceProtocol(self, tempUserOptions):
deviceProtocols = []
ALLOWED_PROTOCOLS = getAllowedProtocol()
inputProtocols = self._userOptions.get('deviceProtocol')
# try:
# sid = int(tempUserOptions['sids'][0])
# instType = self._serviceManager.getEdmType(sid)
# envMode = getSystemEnv()
# if instType != None:
# for deviceProtocol in ALLOWED_PROTOCOLS:
# if instType.lower() == deviceProtocol.split('_')[0]:
# if len(deviceProtocol.split('_')) == 3 and envMode == 'staging':
# return [deviceProtocol]
# elif len(deviceProtocol.split('_')) == 2 and envMode == 'production':
# return [deviceProtocol]
# else:
# self._logger.warn("wrong site id : %d" %sid)
# sys.exit(1)
# except Exception, e:
# self._logger.exception(e)
if inputProtocols:
for inputProtocol in inputProtocols:
if inputProtocol not in ALLOWED_PROTOCOLS:
self._logger.warn(
"# %s is not in allowed protocols.' %dProtocol")
sys.exit(1)
else:
if inputProtocol == '*':
deviceProtocols = ['*']
return deviceProtocols
else:
deviceProtocols.append(inputProtocol)
deviceProtocols = list(set(deviceProtocols))
else:
deviceProtocols = ALLOWED_PROTOCOLS
return deviceProtocols
def _checkJob(self):
job = self._userOptions.get('job')
if job == 'app_search' or job == 'app_usage' or job == 'candidate':
return job
elif not job:
return None
else:
self._logger.warn(
"""# Wrong job request. (You have to use one such as 'app_search', 'app_usage' , 'candidate' or None)"""
)
sys.exit(1)
def _checkFileSave(self):
return self._userOptions.get('fileSave')
def _checkInjection(self):
return self._userOptions.get('injection')
def _checkNumOfProcess(self):
numOfProcess = self._userOptions.get('numOfProcess')
maxNumOfProcess = getMaxProcess()
if numOfProcess > maxNumOfProcess:
numOfProcess = maxNumOfProcess
return numOfProcess
def _checkRemove(self):
REMOVE_OPTIONS = ['appliance', 'meta', 'router', 'vfeeder', 'all', '*']
options = self._userOptions.get('remove')
if not options:
return None
options = list(set(options))
if 'vfeeder' in options or 'all' in options or '*' in options:
return ['vfeeder']
sids = self._checkSiteId()
if '*' in sids or len(sids) == 0 or len(sids) > 1:
self._logger.warn(
"# Wrong remove options. You have to use -s/--sids options.")
sys.exit(1)
for option in options:
if not option in REMOVE_OPTIONS:
self._logger.warn("# Wrong remove options.")
sys.exit(1)
return options
def _checkHasFile(self):
return self._userOptions.get('hasFile')
def _checkCollectingRawDataMode(self, userOptions):
isOnlyRawData = self._userOptions.get('onlyRawdata')
if isOnlyRawData:
sids = [int(sid) for sid in self._userOptions.get('sids')]
userOptions['sids'] = sids
return isOnlyRawData
return isOnlyRawData
# return self._userOptions.get('onlyRawdata')
def _checkLogMode(self):
return self._userOptions.get('debugMode')
def _checkDeviceMeta(self):
return self._userOptions.get('deviceMeta')