class DataHandlerService:
def __init__(self):
self._threads: List[Thread] = []
self._queue = Queue()
# self._queue = collections.tsQueue()
self._event: Event = None
self._db: sql_engine.SQL_DB = None
@property
def is_active(self):
return len(self._threads) == 1
@property
def db_file(self):
return self._db.db_file
@property
def queue(self):
if self._event.isSet():
logger.warn(f"Stop invoked; new data cannot be enqueued")
return self._queue.__class__()
return self._queue
def init(self, location=None, file_name=DEFAULT_DB_FILE, cumulative=False):
self._db = sql_engine.SQL_DB(location, file_name, cumulative)
def start(self, event=Event()):
if self._db.is_new:
for name, table in TableSchemaService().tables.items():
try:
assert not self._db.table_exist(
table.name), f"Table '{name}' already exists"
self._db.execute(
sql_engine.create_table_sql(table.name, table.fields,
table.foreign_keys))
except AssertionError as e:
logger.warn(f"{e}")
except Exception as e:
logger.error(f"Cannot create table '{name}' -> Error: {e}")
raise
self._event = event
dh = Thread(name='DataHandler', target=self._data_handler, daemon=True)
dh.start()
self._threads.append(dh)
def stop(self, timeout=5):
if self._event:
self._event.set()
while len(self._threads) > 0:
th = self._threads.pop(0)
try:
th.join(timeout)
logger.debug(f"Thread '{th.name}' gracefully stopped")
except Exception as e:
logger.error(
f"Thread '{th.name}' gracefully stop failed; Error raised: {e}"
)
def execute(self, sql_text, *rows):
try:
return self._db.execute(sql_text, *rows)
except Exception as e:
logger.error("DB execute error: {}\n{}\n{}".format(
e, sql_text, '\n\t'.join([r for r in rows])))
raise
@property
def get_last_row_id(self):
return self._db.get_last_row_id
def add_data_unit(self, item: DataUnit):
if isinstance(item.table, PlugInTable):
last_tl_id = cache_timestamp(item.timestamp)
item(TL_ID=last_tl_id)
logger.debug(f"Item updated: {item.sql_data}")
self.queue.put(item)
logger.debug(
f"Item enqueued: '{item}' (Current queue size {self.queue.qsize()})"
)
# sleep(0.01)
# FIXME: Handle stdout should be moved in separate thread task; It should bw async with main data handler
def _data_handler(self):
logger.debug(
f"{self.__class__.__name__} Started with event {id(self._event)}")
while True:
if self.queue.empty():
if self._event.isSet():
break
else:
continue
item = self.queue.get()
try:
logger.debug(
f"Deque item: '{item}' (Current queue size {self.queue.qsize()})"
)
insert_sql_str, rows = item.sql_data
result = self.execute(
insert_sql_str,
rows) if rows else self.execute(insert_sql_str)
item.result = result
logger.debug("Insert item: {}\n\t{}\n\t{}".format(
type(item).__name__, insert_sql_str,
'\n\t'.join([str(r) for r in rows])))
except Exception as e:
f, l = get_error_info()
logger.error(
f"Unexpected error occurred on {type(item).__name__}: {e}; File: {f}:{l}"
)
else:
logger.debug(
f"Item {type(item).__name__} successfully handled")
logger.debug(f"Background task stopped invoked")
