def connect(self): self.connect_lock.acquire(True) try: if self.conn: self.log.debug("Already connected, connection ignored") return if self not in opened_dbs: opened_dbs.append(self) s = time.time() try: # Directory not exist yet os.makedirs(self.db_dir) self.log.debug("Created Db path: %s" % self.db_dir) except OSError as err: if err.errno != errno.EEXIST: raise err if not os.path.isfile(self.db_path): self.log.debug("Db file not exist yet: %s" % self.db_path) self.conn = sqlite3.connect(self.db_path, isolation_level="DEFERRED", check_same_thread=False) self.conn.row_factory = sqlite3.Row self.conn.set_progress_handler(self.progress, 5000000) self.conn.execute('PRAGMA journal_mode=WAL') if self.foreign_keys: self.conn.execute("PRAGMA foreign_keys = ON") self.cur = self.getCursor() self.log.debug( "Connected to %s in %.3fs (opened: %s, sqlite version: %s)..." % (self.db_path, time.time() - s, len(opened_dbs), sqlite3.version) ) self.log.debug("Connect by thread: %s" % threading.current_thread().ident) self.log.debug("Connect called by %s" % Debug.formatStack()) finally: self.connect_lock.release()
def close(self, reason="Unknown"): if not self.conn: return False self.connect_lock.acquire() s = time.time() if self.delayed_queue: self.processDelayed() if self in opened_dbs: opened_dbs.remove(self) self.need_commit = False self.commit("Closing: %s" % reason) self.log.debug("Close called by %s" % Debug.formatStack()) for i in range(5): if len(self.cursors) == 0: break self.log.debug("Pending cursors: %s" % len(self.cursors)) time.sleep(0.1 * i) if len(self.cursors): self.log.debug("Killing cursors: %s" % len(self.cursors)) self.conn.interrupt() if self.cur: self.cur.close() if self.conn: ThreadPool.main_loop.call(self.conn.close) self.conn = None self.cur = None self.log.debug("%s closed (reason: %s) in %.3fs, opened: %s" % (self.db_path, reason, time.time() - s, len(opened_dbs))) self.connect_lock.release() return True
def acquire(self, *args, **kwargs): s = time.time() res = self.lock.acquire(*args, **kwargs) time_taken = time.time() - s if time_taken >= self.log_after: logging.debug("%s: Waited %.3fs after called by %s" % (self.name, time_taken, Debug.formatStack())) return res
def atomicWrite(dest, content, mode="wb"): try: with open(dest + "-tmpnew", mode) as f: f.write(content) f.flush() os.fsync(f.fileno()) if os.path.isfile(dest + "-tmpold"): # Previous incomplete write os.rename(dest + "-tmpold", dest + "-tmpold-%s" % time.time()) if os.path.isfile(dest): # Rename old file to -tmpold os.rename(dest, dest + "-tmpold") os.rename(dest + "-tmpnew", dest) if os.path.isfile(dest + "-tmpold"): os.unlink(dest + "-tmpold") # Remove old file return True except Exception as err: from Debug import Debug logging.error("File %s write failed: %s, (%s) reverting..." % (dest, Debug.formatException(err), Debug.formatStack())) if os.path.isfile(dest + "-tmpold") and not os.path.isfile(dest): os.rename(dest + "-tmpold", dest) return False
def loadItem(self, key): try: self.num_loaded += 1 if self.num_loaded % 100 == 0: if config.verbose: self.log.debug( "Loaded json: %s (latest: %s) called by: %s" % (self.num_loaded, key, Debug.formatStack())) else: self.log.debug("Loaded json: %s (latest: %s)" % (self.num_loaded, key)) content = self.site.storage.loadJson(key) dict.__setitem__(self, key, content) except IOError: if dict.get(self, key): self.__delitem__(key) # File not exists anymore raise KeyError(key) self.addCachedKey(key) self.checkLimit() return content
def send(self, data): logging.debug("WsMock: Set result (data: %s) called by %s" % (data, Debug.formatStack())) self.result.set(json.loads(data)["result"])
def loadItem(self, key): try: self.num_loaded += 1 if self.num_loaded % 100 == 0: if config.verbose: self.log.debug("Loaded json: %s (latest: %s) called by: %s" % (self.num_loaded, key, Debug.formatStack())) else: self.log.debug("Loaded json: %s (latest: %s)" % (self.num_loaded, key)) content = self.site.storage.loadJson(key) dict.__setitem__(self, key, content) except IOError: if dict.get(self, key): self.__delitem__(key) # File not exists anymore raise KeyError(key) self.addCachedKey(key) self.checkLimit() return content