def get(self, parser, sourceType: SourceTypes, origin: str):
""" Returns all active data.
Params: Data
Retrive a object Data with structure:
{
'id' : self.id,
'born' : self.born,
'source_type' : str(self.source_type),
'source_name' : self.source_name,
'source_item' : self.source_item,
'package' : self.package,
'data' : self.serialize(self.data),
'aux' : self.serialize(self.aux),
'state' : str(self.state),
},
limit,
lastTime,
all
Controller and device could be an Exp Reg.
Ex. 'CamController' for single or 'CamController.*?' for all from CamController
"""
try:
#TODO: Si en lugar de data se reciben las variables id, package, source_name y source_item
# como se registra en la documentación. Además el tipo lo debe sacar de la url que se usa
parser = reqparse.RequestParser()
parser.add_argument('data')
parser.add_argument('limit')
parser.add_argument('lastTime')
parser.add_argument('onlyActive')
args = parser.parse_args()
dataIn = args.data
dataIn = '' if dataIn == None else dataIn
data = Data()
data.id = '-1'
if dataIn != '':
data = data.parse(dataIn)
limit = args.limit
lastTime = args.lastTime
onlyActive = True if args.onlyActive == '' else Misc.toBool(
args.onlyActive)
result = DataPool().getPool(data=data,
limit=limit,
lastTime=lastTime,
onlyActive=onlyActive)
result = list(map(lambda d: d.getJson(), result))
result.insert(0, {'msg': 'ok', 'queryTime': time()})
except:
message = Binnacle().errorDetail(Messages.error_pool_reading)
Binnacle().logFromCore(message, LogTypes.ERROR, origin)
result = [{'msg': message}]
return result
def start_pool(self):
""" Start data pool """
Binnacle().logFromCore(
Messages.system_pool_start + self.CONFIG['URL_BASE'],
LogTypes.INFO, self.__class__.__name__)
pool = DataPool()
pool.initialize(self.CONFIG)
self.poolThread = Process(target=pool.start, args=())
self.poolThread.daemon = True
self.poolThread.start()
del pool
sleep(2)
Binnacle().logFromCore(
Messages.system_pool_started + self.CONFIG['URL_BASE'],
LogTypes.INFO, self.__class__.__name__)
def __init__(self, cfg=None):
self.dp = DataPool() # Object to send information
self.URL = "" # Pool URL
self.Me_Path = "./" # Path of current component
self.Standalone = False # If a child start in standalone
self.lastTime = time() # Time of last petition to the pool
self.Config = cfg # Object with all config params
self.Classes = self.Config['CLASSES'] # Classes able to detect
self.NET = None # Neural Network to predict
self.Controller = '' # Controller to filter data
self.Device = '' # Device name to filter data
self.Limit = -1 # Amount of data to filter data
self.loggingLevel = None # logging level to write
self.loggingFile = None # Name of file where write log
self.loggingFormat = None # Format to show the log
def post(self, parser, sourceType: SourceTypes, origin: str):
""" Load data in pool. Params:
data: Data
Retrive a object Data with structure:
{
'id' : self.id,
'born' : self.born,
'source_type' : str(self.source_type),
'source_name' : self.source_name,
'source_item' : self.source_item,
'package' : self.package,
'data' : self.serialize(self.data),
'aux' : self.serialize(self.aux),
'state' : str(self.state),
}
"""
try:
#TODO: Validar tipo de dato y si no coiside emitir error
#TODO: E ltipo de dato esperado lo debe sacar de la clase en su instanciación
parser = reqparse.RequestParser()
parser.add_argument('data')
args = parser.parse_args()
dataIn = args.data
dataIn = '' if dataIn == None else dataIn
data = Data()
data.id = -1
if dataIn != '':
data = data.parse(dataIn)
if sourceType != SourceTypes.parse(data.source_type):
raise ValueError(
Messages.bad_source_type_deteiled.format(
sourceType, SourceTypes.parse(data.source_type)))
DataPool().append(data)
DataPool().pop()
result = [{'msg': 'ok', 'id': data.id}]
except:
message = Binnacle().errorDetail(Messages.error_pool_writing)
Binnacle().logFromCore(message, LogTypes.ERROR, origin)
result = [{'msg': message}]
return result
def put(self, parser, origin: str):
""" Exec commands on pool """
try:
parser = reqparse.RequestParser()
parser.add_argument('command')
args = parser.parse_args()
result = [{'msg': 'ok', 'res': DataPool().command(args.command)}]
except:
message = Binnacle().errorDetail(Messages.error_pool_command)
Binnacle().logFromCore(message, LogTypes.ERROR, origin)
result = [{'msg': message}]
return result
import sys sys.path.insert(0, './Core/') from DataPool import DataPool DP = DataPool() DP.start(prod=True) app = DP.app #from werkzeug.contrib.fixers import ProxyFix from werkzeug.middleware.proxy_fix import ProxyFix app.wsgi_app = ProxyFix(app.wsgi_app)
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from DataPool import DataPool
dp = DataPool('demo')
driver = webdriver.Chrome()
driver.get(dp.get('BAIDU_HOME_URL'))
driver.find_element_by_css_selector('#kw').send_keys("selenium")
driver.find_element_by_id('su').click()
#driver.maximize_window()
#driver.find_element_by_link_text('百度首页').click()
#print(driver.name,driver.title,driver.current_url)
WebDriverWait(driver, 5).until(lambda x: 'selenium' in x.title)
assert ('selenium_' in driver.title)
driver.close()
driver.quit()
class ClassifierHAR(abc.ABC):
""" Generic class that represents all the classifiers that can be loaded. """
def __init__(self, cfg=None):
self.dp = DataPool() # Object to send information
self.URL = "" # Pool URL
self.Me_Path = "./" # Path of current component
self.Standalone = False # If a child start in standalone
self.lastTime = time() # Time of last petition to the pool
self.Config = cfg # Object with all config params
self.Classes = self.Config['CLASSES'] # Classes able to detect
self.NET = None # Neural Network to predict
self.Controller = '' # Controller to filter data
self.Device = '' # Device name to filter data
self.Limit = -1 # Amount of data to filter data
self.loggingLevel = None # logging level to write
self.loggingFile = None # Name of file where write log
self.loggingFormat = None # Format to show the log
""" Abstract methods """
@abc.abstractmethod
def preLoad(self):
""" Implement me! :: Load everything need to proccess """
pass
@abc.abstractmethod
def loadModel(self):
""" Implement me! :: Load knowledge for prediction """
pass
@abc.abstractmethod
def predict(self, data):
""" Implement me! :: Do prediction and return class found """
pass
@abc.abstractmethod
def showData(self, data, classes, aux):
""" Implement me! :: To show data if this module start standalone.
set self.Standalone = True before start. """
pass
@abc.abstractmethod
def testData(self):
""" Implement me! :: Data t test if this module start standalone.
You must return an array as expected if you query the data pool.
set self.Standalone = True before start. """
pass
""" Real methods """
def start(self):
""" Start module and prediction """
self.activateLog()
self.preLoad()
self.running = True
logging.debug('Start detection of {} in {}.'.format(self.Config["CLASSES"],
self.__class__.__name__))
failedSend = 0
while self.running:
gdList = []
_classes = None
_idData = 0
try:
if not self.Standalone:
gdList = self.bring(controller=self.Controller, device=self.Device,
limit=self.Limit, lastTime=self.lastTime)
self.lastTime = gdList[0]['timeQuery']
else:
gdList = self.testData()
failedSend = 0
except:
failedSend += 1
logging.exception(
'Unexpected error getting data from pool: {}. Controller: {}, Device: {}, Limit: {}.'.format(
self.URL, self.Controller, self.Device, self.Limit))
if failedSend > 2 and not self.dp.isLive():
logging.error('Pool no found {} will shutdown.'.format(self.__class__.__name__))
self.stop()
break
continue
for gd in gdList[1:]:
_classes = []
try:
_classes, _aux = self.predict(gd)
_idData = gd['id']
except:
logging.exception(
'Unexpected error in prediction from classifier: {} ({}).'.format(
self.__class__.__name__, self.Config["MACHINE_NAME"]))
try:
if not self.Standalone and len(_classes) > 0:
self.sendDetection(_idData, _classes, _aux)
else:
self.showData(gd, _classes, _aux)
failedSend = 0
except:
failedSend += 1
logging.exception(
'Unexpected error sending data from classifier: {} ({}).'.format(
self.__class__.__name__, self.Config["MACHINE_NAME"]))
if failedSend > 2 and not self.dp.isLive():
logging.error('Pool no found {} will shutdown.'.format(self.__class__.__name__))
self.stop()
break
def stop(self):
""" Stop module and predicting """
self.running = False
def sendDetection(self, idData, classes, aux=None):
""" Send detection data to pool """
self.dp.URL = self.URL
self.dp.sendDetection(classifier=self.Config["MACHINE_NAME"],
idData=idData, classes=classes, aux=aux)
def bring(self, controller='', device='', limit=-1, lastTime=0):
""" Bring data from Pool """
self.dp.URL = self.URL
return self.dp.getData(controller=controller, device=device, limit=limit, lastTime=lastTime)
def activateLog(self):
""" Activate logging """
Misc.loggingConf(self.loggingLevel, self.loggingFile, self.loggingFormat)