class TransactionCache(object):
def __init__(self, database_file):
self.logger = logging.getLogger("FR."+self.__class__.__name__)
self.db = SQLiteDB(database_file)
self.filename = database_file
self._recreate_db_if_not_exists()
def _recreate_db_if_not_exists(self):
if not os.path.exists(self.filename) or not self._check_database_file():
self._initialize()
def _query(self, statement, qargs):
self._recreate_db_if_not_exists()
return self.db.query(statement, qargs)
def insert(self, op_id, file_operation, transaction_timestamp):
# Pickled data is stored as binary data into a BLOB field
operation_str = buffer(pickle.dumps(file_operation))
timestamp_str = transaction_timestamp.strftime('%Y-%m-%d %H:%M:%S')
success = self.db.execute('INSERT INTO transaction_cache values (?,?,?)', (op_id, operation_str, timestamp_str))
# TODO: let SQLite raise its own exceptions and wrap them instead of using booleans!
if not success:
raise CachePersistenceException('transaction_cache insert')
def get_all(self):
records = self._query('SELECT * FROM transaction_cache', ())
result = []
for record in records:
op_id, operation_str, timestamp_str = record
# Pickled data is stored as binary data into a BLOB field
file_operation = pickle.loads(str(operation_str))
transaction_timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
result.append((op_id, file_operation, transaction_timestamp))
return result
def clear(self):
success = self.db.execute("DELETE FROM transaction_cache", [])
# TODO: let SQLite raise its own exceptions and wrap them instead of using booleans!
if not success:
raise CachePersistenceException('transaction_cache delete')
def _check_database_file(self):
try:
result = self.db.check_database_file("SELECT * from transaction_cache")
except:
# TODO: create backup
# self.logger.warning(u"Corrupted %s database file. Backupped as %s" % (self.filename, backup_name))
result = False
return result
def _initialize(self):
self.db.execute('CREATE TABLE transaction_cache (id int, file_operation blob, transaction_timestamp text)')
class LocalDatasetDB(object):
def __init__(self, database_file):
self.logger = logging.getLogger("FR."+self.__class__.__name__)
# self.logger = logging.getLogger("FR."+self.__class__.__name__)
self.logger.addHandler(logging.NullHandler())
self.db = SQLiteDB(database_file)
def select_all_records(self):
''' Returns either the list of tuples or False on error. '''
return self.db.query('select * from local_dataset;', ())
def is_there_record_for(self, pathname):
''' Returns either True or False, depending if there's a record with given pathname. '''
record = self.get_record(pathname)
if record == None: return False
else: return True
def get_record(self, pathname):
''' Returns record for the given pathname or None if such record does not exist. '''
record = self.db.query("select * from local_dataset where pathname=? ;", [pathname])
if isinstance(record, list) and len(record) == 0: return None
if isinstance(record, list) and len(record) == 1: return record[0]
elif isinstance(record, list) and len(record) > 1:
self.logger.warning(u'Whoops! More than one record found for pathname "%s"... returning the first...' % (pathname))
return record[0]
def get_last_modification_time(self, pathname):
''' Returns value of lmtime field of record for given pathname, None if it's empy, or False if there's no such record. '''
record = self.get_record(pathname)
if record == None: return False
lmtime = record[2]
self.logger.debug(u'Last modification time for pathname "%s" is "%s" according to local dataset DB.' % (pathname, lmtime))
if lmtime == '': return None
else: return lmtime
def get_all_records_starting_with(self, prefix):
''' Returns a list of all records (tuple) for which pathname starts with given prefix. '''
return self.db.query('select * from local_dataset where pathname LIKE ?', [ '%s%s' % (prefix,'%')] )
def insert_record(self, record):
''' Record is supposed to be a tuple like (pathname, etag, hash) '''
result = self.db.execute('insert into local_dataset values (?,?,?)', record)
if result: self.logger.debug(u'Record %s inserted in local dataset.' % (repr(record)))
return result
def delete_record(self, pathname):
statement = "delete from local_dataset where pathname=? ;"
eargs = [pathname]
result = self.db.execute(statement, eargs)
if result: self.logger.debug(u'Local dataset record successfully removed for pathname "%s"' % (pathname))
return result
def update_record(self, pathname, etag=None, lmtime=None):
if etag != None and lmtime != None: statement, eargs = self._update_etag_and_lmtime(pathname, etag, lmtime)
elif etag != None and lmtime == None: statement, eargs = self._update_etag(pathname, etag)
elif etag == None and lmtime != None: statement, eargs = self._update_lmtime(pathname, lmtime)
else:
self.logger.warning(u'Local dataset record update requested with invalid args (all None) for pathname "%s"' % (pathname))
return False
result = self.db.execute(statement, eargs)
if result: self.logger.debug(u'Local dataset record successfully updated for pathname "%s"' % (pathname))
return result
def rename_record(self, pathname, new_pathname):
result = self.db.execute("update local_dataset set pathname=? where pathname=? ;", [new_pathname, pathname])
if result: self.logger.debug(u'Record %s successfully renamed to %s' % (pathname, new_pathname))
return result
def clear(self):
statement = "delete from local_dataset"
eargs = []
result = self.db.execute(statement, eargs)
return result
def _update_etag_and_lmtime(self, pathname, etag, lmtime):
statement = "update local_dataset set etag=?, lmtime=? where pathname=? ;"
eargs = [etag, lmtime, pathname]
return (statement, eargs)
def _update_etag(self, pathname, etag):
statement = "update local_dataset set etag=? where pathname=? ;"
eargs = [etag, pathname]
return (statement, eargs)
def _update_lmtime(self, pathname, lmtime):
statement = "update local_dataset set lmtime=? where pathname=? ;"
eargs = [lmtime, pathname]
return (statement, eargs)
def check_database_file(self):
''' Check if database file is present and is a regulare sqlite3 database file. '''
query = 'select pathname, etag, lmtime from local_dataset;'
try:
result = self.db.check_database_file(query) # same method of self.db class ;)
except:
self.logger.warning(u'Something went wrong attempting default query, result is: %s' % (result))
return result
def initialize_new(self):
''' Initialize a new local dataset database. '''
self.logger.debug(u'Creating local_dataset table... ' )
self.db.execute('create table local_dataset (pathname text, etag text, lmtime text) ;')
return True
class DownloadCacheDB(object):
def __init__(self, database_file):
self.logger = logging.getLogger("FR." + self.__class__.__name__)
self.db = SQLiteDB(database_file)
def check_database_file(self):
'''
Check if database file is present
and is a regular sqlite3 database file.
'''
query = "SELECT * FROM %s LIMIT 1 ;" % (TABLENAME)
try:
result = self.db.check_database_file(query)
except:
self.logger.warning(u'Something went wrong attempting default \
query, result is: %s' % (result)
)
return result
def initialize_new(self):
''' Initialize a new local dataset database. '''
self.logger.debug(u'Creating local_dataset table... ')
result = self.db.execute('create table %s (%s) ;' % (TABLENAME,
TABLESCHEMA))
if result:
self.logger.debug(u'%s database table successfully created.' %
(TABLENAME))
else:
self.logger.warning(u'Something went wrong creating \
%s database table... ' % (TABLENAME))
return False
unix_gmtime, string_localtime = get_unix_and_local_timestamp()
result = self.db.execute("INSERT INTO %s VALUES (?,?,?,?,?) ;" %
(TABLENAME), [unix_gmtime,
string_localtime,
'autoinsert',
'',
UNKNOWN_BASIS_STRING
]
)
if result:
self.logger.debug(u'Default entry successfully inserted in \
%s database table.' % (TABLENAME))
else:
self.logger.warning(u'Something went wrong inserting default \
entry into %s database table... ' %
(TABLENAME))
return False
return True
def add(self, prev_hash, next_hash, user_accepted=False):
self.logger.debug(u'Saving following hashes %s %s' % (prev_hash,
next_hash))
if user_accepted:
record_type = 'useraccept'
else:
record_type = 'commit'
if prev_hash is None:
prev_hash = UNKNOWN_BASIS_STRING
unix_gmtime, string_localtime = get_unix_and_local_timestamp()
if not self.check_database_file():
self.initialize_new()
result = self.db.execute("INSERT INTO %s VALUES (?,?,?,?,?) ;" %
(TABLENAME), [unix_gmtime,
string_localtime,
record_type,
prev_hash,
next_hash])
if result:
self.logger.debug(u'New hash couple saved (%s, %s, %s)' %
(record_type, prev_hash, next_hash))
else:
self.logger.warning(u'Something went wrong saving hashes \
(%s, %s, %s)' % (record_type,
prev_hash,
next_hash))
return False
self.logger.info('Commit data saved to history')
return True
def list(self):
records = self.db.query('SELECT * FROM %s' % (TABLENAME), [])
return records
class MetadataDB(object):
"""
A key-value store
It manage a locally persistent that keep whatever data
that should be available among different run of the client.
"""
def __init__(self, database_file):
self.logger = logging.getLogger("FR." + self.__class__.__name__)
self.db = SQLiteDB(database_file)
self.filename = database_file
def _file_exists(self):
return os.path.exists(self.filename)
def delete(self):
""" Deletes the db file from disk """
if self._file_exists():
os.unlink(self.filename)
def _recreate_db_if_not_exists(self):
""" Recreate the db file if not exists """
if not os.path.exists(self.filename) or not self._check_database_file():
self.logger.debug(u"Local metadata database not found. Initializing " "new local metadata DB...")
self.initialize_new()
def _query(self, statement, qargs):
""" Executes a query on database and return its result """
self._recreate_db_if_not_exists()
return self.db.query(statement, qargs)
def _check_database_file(self):
"""
Check if database file is present and is a regular
sqlite3 database file.
"""
query = "select value from metadata where key='last_connected_datetime' ;"
try:
result = self.db.check_database_file(query)
except:
self.logger.warning(u"Something went wrong attempting default query, result is: %s" % (result))
return result
def initialize_new(self):
""" Initialize a new local dataset database. """
self.logger.debug(u"Creating local_dataset table... ")
result = self.db.execute("create table metadata (key text, value text) ;")
if result:
self.logger.debug(u"Metadata database table successfully created.")
else:
self.logger.warning(u"Something went wrong creating metadata database table... ")
return False
result = self.db.execute(
"insert into metadata values ('last_connected_datetime','Thu, 01 Jan 1970 00:00:00 GMT') ;"
)
if result:
self.logger.debug(u"Default entry successfully inserted in metadata database table.")
else:
self.logger.warning(u"Something went wrong inserting default entry into metadata database table... ")
return False
return True
def exists_record(self, key):
"""
Returns true if there is the key
@param key: the value of the key you are looking for
@return: boolean
"""
res = self._query("SELECT COUNT(*) FROM metadata WHERE key = ?", [key])
count = compose(fst, fst)(res) # Damn tuples
return count > 0
def get(self, key):
"""
Looks for the record with the given key
@param key: the key value
@return: the value associated with the given key
"""
statement = "SELECT value FROM metadata WHERE key=?"
result = self._query(statement, [key])
if isinstance(result, list) and len(result) == 1:
return result[0][0]
elif isinstance(result, list) and len(result) > 1:
self.logger.warning(u"More than one entry in metadata DB for %s!" % key)
return result[0][0]
else:
self.logger.warning(u"Something went wrong getting %s." % key)
raise Exception('No key "%s" in metadataDB' % key)
def _get_value_or_None(self, key):
"""
Looks for the record with the given key
@param key: the key value
@return: the value associated with the given key
"""
if self.exists_record(key):
value = self.get(key)
else:
value = None
return value
def try_get(self, key):
"""
Tries to get the value associated with the given key
@return: the value or None
"""
return self._get_value_or_None(key)
def set(self, key, value):
"""
Adds a key or updates the value of a key
@param key: the key name
@param value: the key value
"""
if self.exists_record(key):
self._update_record(key, value)
else:
self._insert_record(key, value)
def _update_record(self, key, value):
"""
Updates the value of a key
@param key: key value
@param value: the value
"""
values = [value, key]
self.db.execute("UPDATE metadata SET value = ? WHERE key = ?", values)
def delete_key(self, keyvalue):
"""
Deletes a key
@param keyvalue: the key to delete
"""
self.logger.debug("Deleting key %s from metadatadb" % keyvalue)
self.db.execute("DELETE FROM metadata WHERE key = ?", [keyvalue])
def _insert_record(self, key, value):
"""
Adds the key
@param key: the key
@param value: the value
"""
values = [key, value]
self.db.execute("INSERT INTO metadata VALUES (?, ?)", values)
def update_last_connected_datetime(self, value=""):
"""
Update metadata record for last_connected_timestamp with given value.
@return: result of the update
"""
if value == "":
value = formatdate(int(time()), localtime=False, usegmt=True)
statement = "update metadata set value=? where key=? ;"
eargs = [value, "last_connected_datetime"]
result = self.db.execute(statement, eargs)
# This log is too verbose
# if result: self.logger.debug(u'Metadata database updated with "last_connected_datetime"="%s"' % (value))
# We're not even using this in the one-way default version, so let's just log errors if any
if not result:
self.logger.warning(u"Something went wrong updating last_connected_datetime! ")
return result
def get_last_connected_datetime(self):
"""
Get last connected timestamp set.
@return: last_connected_date as string or None
"""
statement = "select value from metadata where key=? ;"
result = self._query(statement, ["last_connected_datetime"])
if isinstance(result, list) and len(result) == 1:
return result[0][0]
elif isinstance(result, list) and len(result) > 1:
self.logger.warning(u"More than one entry in metadata DB for last_connected_datetime!")
return result[0][0]
else:
self.logger.warning(u"Something went wrong getting last_connected_datetime.")
return None
def get_all(self):
"""
Returns a list of tuples
Any tuple will have the format (key, value)
@return: a list of tuples
"""
records = self._query("SELECT * FROM metadata", [])
result = []
for record in records:
key, value = record
result.append((key, value))
return result