class DataHandlerService: def __init__(self): self._threads: List[Thread] = [] self._queue = Queue() # self._queue = collections.tsQueue() self._event: Event = None self._db: sql_engine.SQL_DB = None @property def is_active(self): return len(self._threads) == 1 @property def db_file(self): return self._db.db_file @property def queue(self): if self._event.isSet(): logger.warn(f"Stop invoked; new data cannot be enqueued") return self._queue.__class__() return self._queue def init(self, location=None, file_name=DEFAULT_DB_FILE, cumulative=False): self._db = sql_engine.SQL_DB(location, file_name, cumulative) def start(self, event=Event()): if self._db.is_new: for name, table in TableSchemaService().tables.items(): try: assert not self._db.table_exist( table.name), f"Table '{name}' already exists" self._db.execute( sql_engine.create_table_sql(table.name, table.fields, table.foreign_keys)) except AssertionError as e: logger.warn(f"{e}") except Exception as e: logger.error(f"Cannot create table '{name}' -> Error: {e}") raise self._event = event dh = Thread(name='DataHandler', target=self._data_handler, daemon=True) dh.start() self._threads.append(dh) def stop(self, timeout=5): if self._event: self._event.set() while len(self._threads) > 0: th = self._threads.pop(0) try: th.join(timeout) logger.debug(f"Thread '{th.name}' gracefully stopped") except Exception as e: logger.error( f"Thread '{th.name}' gracefully stop failed; Error raised: {e}" ) def execute(self, sql_text, *rows): try: return self._db.execute(sql_text, *rows) except Exception as e: logger.error("DB execute error: {}\n{}\n{}".format( e, sql_text, '\n\t'.join([r for r in rows]))) raise @property def get_last_row_id(self): return self._db.get_last_row_id def add_data_unit(self, item: DataUnit): if isinstance(item.table, PlugInTable): last_tl_id = cache_timestamp(item.timestamp) item(TL_ID=last_tl_id) logger.debug(f"Item updated: {item.sql_data}") self.queue.put(item) logger.debug( f"Item enqueued: '{item}' (Current queue size {self.queue.qsize()})" ) # sleep(0.01) # FIXME: Handle stdout should be moved in separate thread task; It should bw async with main data handler def _data_handler(self): logger.debug( f"{self.__class__.__name__} Started with event {id(self._event)}") while True: if self.queue.empty(): if self._event.isSet(): break else: continue item = self.queue.get() try: logger.debug( f"Deque item: '{item}' (Current queue size {self.queue.qsize()})" ) insert_sql_str, rows = item.sql_data result = self.execute( insert_sql_str, rows) if rows else self.execute(insert_sql_str) item.result = result logger.debug("Insert item: {}\n\t{}\n\t{}".format( type(item).__name__, insert_sql_str, '\n\t'.join([str(r) for r in rows]))) except Exception as e: f, l = get_error_info() logger.error( f"Unexpected error occurred on {type(item).__name__}: {e}; File: {f}:{l}" ) else: logger.debug( f"Item {type(item).__name__} successfully handled") logger.debug(f"Background task stopped invoked")