class TinyDBImageFileManifest(ImageManifest): """ Implementation for ImageManifest interface, it's implements database behavior with select, update, insert,... for file storage, based on JSON format. """ def __init__(self, manifest_path, timestamp, lock, db_write_cache_size=1, use_dr=False): self.__table_name = str(timestamp) path = "{0}/{1}".format(manifest_path, self.__table_name) logging.debug( "Creating or opening image manifest database {0}. It's uses tinydb project " "(https://pypi.python.org/pypi/tinydb) and has json format. Just copy-past content of this file " "to online web service, that can represent it in more user-friendly format, for example, " "http://jsonviewer.stack.hu/ http://www.jsoneditoronline.org/ http://codebeautify.org/jsonviewer" .format(path)) self.__db = None self.__table = None self.__storage = None self.__use_dr = use_dr try: # CachingMiddleware: Improves speed by reducing disk I/O. It caches all read operations and writes data # to disk every CachingMiddleware.WRITE_CACHE_SIZE write operations. CachingMiddleware.WRITE_CACHE_SIZE = db_write_cache_size self.__storage = CachingMiddleware(JSONStorage) self.__db = TinyDB(path, storage=self.__storage) # Creating new table for chunks self.__table = self.__db.table(self.__table_name) except Exception as e: logging.error( "!!!ERROR: Failed to create or open image manifest database: {0}" .format(e)) raise self.__manifest_path = manifest_path self.__db_write_cache_size = db_write_cache_size self.__lock = lock super(TinyDBImageFileManifest, self).__init__() @staticmethod def create(manifest_path, table_name, lock, db_write_cache_size, use_dr): return TinyDBImageFileManifest( manifest_path, "{0}.cloudscraper-manifest-tables".format(table_name), lock, db_write_cache_size, use_dr) @staticmethod def open(manifest_path, table_name, lock, db_write_cache_size, use_dr): return TinyDBImageFileManifest(manifest_path, table_name, lock, db_write_cache_size, use_dr) def flush(self): self.__storage.flush() def insert_db_meta(self, res): """ Writes database meta data with given res (dictionary) value. Default meta data table name is "_default" and can be found in manifest file. :param res: dictionary with meta data :type res: dict """ with self.__lock: self.__db.insert(res) def update(self, etag, offset, rec): """ Update record in database. Pair (etag, offset) has table unique key semantics, so in given table there are no records with same hash and offset. :param etag: etag to search :type etag: string :param offset: data chunk offset :type offset: int :param rec: dictionary with new values for given record :type rec: dict """ with self.__lock: key = Query() return self.__table.update( rec, key.etag == str(etag) and key.offset == str(offset)) def select(self, etag=None, part_name=None): """ Search records by given etag or (and) part name :param etag: etag to search :type etag: string :param part_name: part name to search :type part_name: string :return: list of records or empty [] """ key = Query() if etag and part_name: key = (key.etag == str(etag)) & (key.part_name == str(part_name)) elif etag: key = key.etag == str(etag) elif part_name: key = key.part_name == str(part_name) return self.__table.search(key) def insert(self, etag, local_hash, part_name, offset, size, status): """ Insert record in database. Pair (etag, offset) has table unique key semantics, so in given table there are no records with same hash and offset :param etag: etag (hash of remote storage data chunk) :type etag: string :param local_hash: (hash of local storage data chunk) :type local_hash: string :param part_name: name for uploaded part :type part_name: string :param offset: offset of data chunk in whole file :type offset: int :param size: size of data chunk :type size: int :param status: dictionary with new values for given record :type status: string """ res = { "uuid": str(uuid.uuid4()), "etag": str(etag), "local_hash": str(local_hash), "part_name": str(part_name), "offset": str(offset), "size": str(size), "status": str(status) } # # We can't insert record with same etag and offset (has unique key semantic) rec_list = self.select(res["etag"]) if any(d['offset'] == str(offset) for d in rec_list) is False: with self.__lock: self.__table.insert(res) def all(self): """ Returns all records from current (latest by timestamp) :return: list of all records """ with self.__lock: return self.__table.all() def get_name(self): """ Returns name of backup, in this case it's means manifest file name. :return: table (file) name """ return self.__table_name def get_path(self): if self.__use_dr: return "{0}/{1}".format(self.__manifest_path, self.__table_name)