def __init__(self, app):
super().__init__(app)
self.queue = MemoryQueue()
self.stop_evt = threading.Event()
self.push_to_es_async()
self.__class__.no += 1
self.store = ESStore(app.config["COMMAND_STORAGE"].get(
"HOSTS", self.default_hosts))
if not self.store.ping():
raise AssertionError("ESCommand storage init error")
class ESCommandRecorder(CommandRecorder, metaclass=Singleton):
batch_size = 10
timeout = 5
no = 0
default_hosts = ["http://localhost"]
def __init__(self, app):
super().__init__(app)
self.queue = MemoryQueue()
self.stop_evt = threading.Event()
self.push_to_es_async()
self.__class__.no += 1
self.store = ESStore(app.config["COMMAND_STORAGE"].get(
"HOSTS", self.default_hosts))
if not self.store.ping():
raise AssertionError("ESCommand storage init error")
def record(self, data):
if data and data['input']:
data['input'] = data['input'][:128]
data['output'] = data['output'][:1024]
data['timestamp'] = int(data['timestamp'])
self.queue.put(data)
def push_to_es_async(self):
def func():
while not self.stop_evt.is_set():
data_set = self.queue.mget(self.batch_size,
timeout=self.timeout)
logger.debug(
"<Session command recorder {}> queue size: {}".format(
self.no, self.queue.qsize()))
if not data_set:
continue
logger.debug("Send {} commands to server".format(
len(data_set)))
ok = self.store.bulk_save(data_set)
if not ok:
self.queue.mput(data_set)
thread = threading.Thread(target=func)
thread.daemon = True
thread.start()
def session_start(self, session_id):
pass
def session_end(self, session_id):
pass
def __del__(self):
print("{} has been gc".format(self))
def count(self, date_from=None, date_to=None,
user=None, asset=None, system_user=None,
input=None, session=None):
amount = ESStore.count(
self, date_from=date_from, date_to=date_to,
user=user, asset=asset, system_user=system_user,
input=input, session=session
)
return amount
def filter(self, date_from=None, date_to=None,
user=None, asset=None, system_user=None,
input=None, session=None):
data = ESStore.filter(
self, date_from=date_from, date_to=date_to,
user=user, asset=asset, system_user=system_user,
input=input, session=session
)
return AbstractSessionCommand.from_multi_dict(
[item["_source"] for item in data["hits"] if item]
)
def filter(self,
date_from=None,
date_to=None,
user=None,
asset=None,
system_user=None,
input=None,
session=None):
data = ESStore.filter(self,
date_from=date_from,
date_to=date_to,
user=user,
asset=asset,
system_user=system_user,
input=input,
session=session)
return [item["_source"] for item in data["hits"] if item]
def __init__(self, params):
hosts = params.get('HOSTS', ['http://localhost'])
ESStore.__init__(self, hosts=hosts)
def bulk_save(self, commands):
return ESStore.bulk_save(self, commands)
def save(self, command):
return ESStore.save(self, command)
def bulk_save(self, commands):
"""批量保存"""
return ESStore.bulk_save(self, commands)