class TinyDBImageFileManifest(ImageManifest):
"""
Implementation for ImageManifest interface, it's implements database behavior with select, update, insert,...
for file storage, based on JSON format.
"""
def __init__(self,
manifest_path,
timestamp,
lock,
db_write_cache_size=1,
use_dr=False):
self.__table_name = str(timestamp)
path = "{0}/{1}".format(manifest_path, self.__table_name)
logging.debug(
"Creating or opening image manifest database {0}. It's uses tinydb project "
"(https://pypi.python.org/pypi/tinydb) and has json format. Just copy-past content of this file "
"to online web service, that can represent it in more user-friendly format, for example, "
"http://jsonviewer.stack.hu/ http://www.jsoneditoronline.org/ http://codebeautify.org/jsonviewer"
.format(path))
self.__db = None
self.__table = None
self.__storage = None
self.__use_dr = use_dr
try:
# CachingMiddleware: Improves speed by reducing disk I/O. It caches all read operations and writes data
# to disk every CachingMiddleware.WRITE_CACHE_SIZE write operations.
CachingMiddleware.WRITE_CACHE_SIZE = db_write_cache_size
self.__storage = CachingMiddleware(JSONStorage)
self.__db = TinyDB(path, storage=self.__storage)
# Creating new table for chunks
self.__table = self.__db.table(self.__table_name)
except Exception as e:
logging.error(
"!!!ERROR: Failed to create or open image manifest database: {0}"
.format(e))
raise
self.__manifest_path = manifest_path
self.__db_write_cache_size = db_write_cache_size
self.__lock = lock
super(TinyDBImageFileManifest, self).__init__()
@staticmethod
def create(manifest_path, table_name, lock, db_write_cache_size, use_dr):
return TinyDBImageFileManifest(
manifest_path,
"{0}.cloudscraper-manifest-tables".format(table_name), lock,
db_write_cache_size, use_dr)
@staticmethod
def open(manifest_path, table_name, lock, db_write_cache_size, use_dr):
return TinyDBImageFileManifest(manifest_path, table_name, lock,
db_write_cache_size, use_dr)
def flush(self):
self.__storage.flush()
def insert_db_meta(self, res):
"""
Writes database meta data with given res (dictionary) value. Default meta data table name is "_default"
and can be found in manifest file.
:param res: dictionary with meta data
:type res: dict
"""
with self.__lock:
self.__db.insert(res)
def update(self, etag, offset, rec):
"""
Update record in database. Pair (etag, offset) has table unique key semantics,
so in given table there are no records with same hash and offset.
:param etag: etag to search
:type etag: string
:param offset: data chunk offset
:type offset: int
:param rec: dictionary with new values for given record
:type rec: dict
"""
with self.__lock:
key = Query()
return self.__table.update(
rec, key.etag == str(etag) and key.offset == str(offset))
def select(self, etag=None, part_name=None):
"""
Search records by given etag or (and) part name
:param etag: etag to search
:type etag: string
:param part_name: part name to search
:type part_name: string
:return: list of records or empty []
"""
key = Query()
if etag and part_name:
key = (key.etag == str(etag)) & (key.part_name == str(part_name))
elif etag:
key = key.etag == str(etag)
elif part_name:
key = key.part_name == str(part_name)
return self.__table.search(key)
def insert(self, etag, local_hash, part_name, offset, size, status):
"""
Insert record in database. Pair (etag, offset) has table unique key semantics,
so in given table there are no records with same hash and offset
:param etag: etag (hash of remote storage data chunk)
:type etag: string
:param local_hash: (hash of local storage data chunk)
:type local_hash: string
:param part_name: name for uploaded part
:type part_name: string
:param offset: offset of data chunk in whole file
:type offset: int
:param size: size of data chunk
:type size: int
:param status: dictionary with new values for given record
:type status: string
"""
res = {
"uuid": str(uuid.uuid4()),
"etag": str(etag),
"local_hash": str(local_hash),
"part_name": str(part_name),
"offset": str(offset),
"size": str(size),
"status": str(status)
}
# # We can't insert record with same etag and offset (has unique key semantic)
rec_list = self.select(res["etag"])
if any(d['offset'] == str(offset) for d in rec_list) is False:
with self.__lock:
self.__table.insert(res)
def all(self):
"""
Returns all records from current (latest by timestamp)
:return: list of all records
"""
with self.__lock:
return self.__table.all()
def get_name(self):
"""
Returns name of backup, in this case it's means manifest file name.
:return: table (file) name
"""
return self.__table_name
def get_path(self):
if self.__use_dr:
return "{0}/{1}".format(self.__manifest_path, self.__table_name)
