class TransactionCache(object):

    def __init__(self, database_file):
        self.logger = logging.getLogger("FR."+self.__class__.__name__)
        self.db = SQLiteDB(database_file)
        self.filename = database_file
        self._recreate_db_if_not_exists()

    def _recreate_db_if_not_exists(self):
        if not os.path.exists(self.filename) or not self._check_database_file():
            self._initialize()

    def _query(self, statement, qargs):
        self._recreate_db_if_not_exists()
        return self.db.query(statement, qargs)

    def insert(self, op_id, file_operation, transaction_timestamp):
        # Pickled data is stored as binary data into a BLOB field
        operation_str = buffer(pickle.dumps(file_operation))
        timestamp_str = transaction_timestamp.strftime('%Y-%m-%d %H:%M:%S')
        success = self.db.execute('INSERT INTO transaction_cache values (?,?,?)', (op_id, operation_str, timestamp_str))
        # TODO: let SQLite raise its own exceptions and wrap them instead of using booleans!
        if not success:
            raise CachePersistenceException('transaction_cache insert')

    def get_all(self):
        records = self._query('SELECT * FROM transaction_cache', ())
        result = []
        for record in records:
            op_id, operation_str, timestamp_str = record
            # Pickled data is stored as binary data into a BLOB field
            file_operation = pickle.loads(str(operation_str))
            transaction_timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
            result.append((op_id, file_operation, transaction_timestamp))
        return result

    def clear(self):
        success = self.db.execute("DELETE FROM transaction_cache", [])
        # TODO: let SQLite raise its own exceptions and wrap them instead of using booleans!
        if not success:
            raise CachePersistenceException('transaction_cache delete')

    def _check_database_file(self):
        try:
            result = self.db.check_database_file("SELECT * from transaction_cache")
        except:
            # TODO: create backup
#            self.logger.warning(u"Corrupted %s database file. Backupped as %s" % (self.filename, backup_name))
            result = False
        return result

    def _initialize(self):
        self.db.execute('CREATE TABLE transaction_cache (id int, file_operation blob, transaction_timestamp text)')
class LocalDatasetDB(object):

    def __init__(self, database_file):
        self.logger = logging.getLogger("FR."+self.__class__.__name__)
#        self.logger = logging.getLogger("FR."+self.__class__.__name__)
        self.logger.addHandler(logging.NullHandler())
        self.db = SQLiteDB(database_file)

    def select_all_records(self):
        ''' Returns either the list of tuples or False on error. '''
        return self.db.query('select * from local_dataset;', ())

    def is_there_record_for(self, pathname):
        ''' Returns either True or False, depending if there's a record with given pathname. '''
        record = self.get_record(pathname)
        if record == None: return False
        else: return True

    def get_record(self, pathname):
        ''' Returns record for the given pathname or None if such record does not exist. '''
        record = self.db.query("select * from local_dataset where pathname=? ;", [pathname])
        if isinstance(record, list) and len(record) == 0: return None
        if isinstance(record, list) and len(record) == 1: return record[0]
        elif isinstance(record, list) and len(record) > 1:
            self.logger.warning(u'Whoops! More than one record found for pathname "%s"... returning the first...' % (pathname))
            return record[0]

    def get_last_modification_time(self, pathname):
        ''' Returns value of lmtime field of record for given pathname, None if it's empy, or False if there's no such record. '''
        record = self.get_record(pathname)
        if record == None: return False
        lmtime = record[2]
        self.logger.debug(u'Last modification time for pathname "%s" is "%s" according to local dataset DB.' % (pathname, lmtime))
        if lmtime == '': return None
        else: return lmtime

    def get_all_records_starting_with(self, prefix):
        ''' Returns a list of all records (tuple) for which pathname starts with given prefix. '''
        return self.db.query('select * from local_dataset where pathname LIKE ?', [ '%s%s' % (prefix,'%')] )

    def insert_record(self, record):
        ''' Record is supposed to be a tuple like (pathname, etag, hash) '''

        result = self.db.execute('insert into local_dataset values (?,?,?)', record)
        if result: self.logger.debug(u'Record %s inserted in local dataset.' % (repr(record)))
        return result

    def delete_record(self, pathname):
        statement = "delete from local_dataset where pathname=? ;"
        eargs = [pathname]
        result = self.db.execute(statement, eargs)
        if result: self.logger.debug(u'Local dataset record successfully removed for pathname "%s"' % (pathname))
        return result

    def update_record(self, pathname, etag=None, lmtime=None):
        if   etag != None and lmtime != None: statement, eargs = self._update_etag_and_lmtime(pathname, etag, lmtime)
        elif etag != None and lmtime == None: statement, eargs = self._update_etag(pathname, etag)
        elif etag == None and lmtime != None: statement, eargs = self._update_lmtime(pathname, lmtime)
        else:
            self.logger.warning(u'Local dataset record update requested with invalid args (all None) for pathname "%s"' % (pathname))
            return False
        result = self.db.execute(statement, eargs)
        if result: self.logger.debug(u'Local dataset record successfully updated for pathname "%s"' % (pathname))
        return result

    def rename_record(self, pathname, new_pathname):
        result = self.db.execute("update local_dataset set pathname=? where pathname=? ;", [new_pathname, pathname])
        if result: self.logger.debug(u'Record %s successfully renamed to %s' % (pathname, new_pathname))
        return result

    def clear(self):
        statement = "delete from local_dataset"
        eargs = []
        result = self.db.execute(statement, eargs)
        return result

    def _update_etag_and_lmtime(self, pathname, etag, lmtime):
        statement = "update local_dataset set etag=?, lmtime=? where pathname=? ;"
        eargs = [etag, lmtime, pathname]
        return (statement, eargs)

    def _update_etag(self, pathname, etag):
        statement = "update local_dataset set etag=? where pathname=? ;"
        eargs = [etag, pathname]
        return (statement, eargs)

    def _update_lmtime(self, pathname, lmtime):
        statement = "update local_dataset set lmtime=? where pathname=? ;"
        eargs = [lmtime, pathname]
        return (statement, eargs)

    def check_database_file(self):
        ''' Check if database file is present and is a regulare sqlite3 database file. '''
        query = 'select pathname, etag, lmtime from local_dataset;'
        try:
            result = self.db.check_database_file(query) # same method of self.db class ;)
        except:
            self.logger.warning(u'Something went wrong attempting default query, result is: %s' % (result))
        return result

    def initialize_new(self):
        ''' Initialize a new local dataset database. '''
        self.logger.debug(u'Creating local_dataset table... ' )
        self.db.execute('create table local_dataset (pathname text, etag text, lmtime text) ;')
        return True
class DownloadCacheDB(object):

    def __init__(self, database_file):
        self.logger = logging.getLogger("FR." + self.__class__.__name__)
        self.db = SQLiteDB(database_file)

    def check_database_file(self):
        '''
        Check if database file is present
        and is a regular sqlite3 database file.
        '''
        query = "SELECT * FROM %s LIMIT 1 ;" % (TABLENAME)
        try:
            result = self.db.check_database_file(query)
        except:
            self.logger.warning(u'Something went wrong attempting default \
                                  query, result is: %s' % (result)
                                )
        return result

    def initialize_new(self):
        ''' Initialize a new local dataset database. '''
        self.logger.debug(u'Creating local_dataset table... ')
        result = self.db.execute('create table %s (%s) ;' % (TABLENAME,
                                                             TABLESCHEMA))
        if result:
            self.logger.debug(u'%s database table successfully created.' %
                              (TABLENAME))
        else:
            self.logger.warning(u'Something went wrong creating \
                                  %s database table... ' % (TABLENAME))
            return False
        unix_gmtime, string_localtime = get_unix_and_local_timestamp()
        result = self.db.execute("INSERT INTO %s VALUES (?,?,?,?,?) ;" %
                                 (TABLENAME), [unix_gmtime,
                                               string_localtime,
                                               'autoinsert',
                                               '',
                                               UNKNOWN_BASIS_STRING
                                               ]
                                 )

        if result:
            self.logger.debug(u'Default entry successfully inserted in \
                                %s database table.' % (TABLENAME))
        else:
            self.logger.warning(u'Something went wrong inserting default \
                                  entry into %s database table... ' %
                                  (TABLENAME))
            return False
        return True

    def add(self, prev_hash, next_hash, user_accepted=False):
        self.logger.debug(u'Saving following hashes %s %s' % (prev_hash,
                                                              next_hash))
        if user_accepted:
            record_type = 'useraccept'
        else:
            record_type = 'commit'

        if prev_hash is None:
            prev_hash = UNKNOWN_BASIS_STRING

        unix_gmtime, string_localtime = get_unix_and_local_timestamp()

        if not self.check_database_file():
            self.initialize_new()

        result = self.db.execute("INSERT INTO %s VALUES (?,?,?,?,?) ;" %
                                 (TABLENAME), [unix_gmtime,
                                               string_localtime,
                                               record_type,
                                               prev_hash,
                                               next_hash])
        if result:
            self.logger.debug(u'New hash couple saved (%s, %s, %s)' %
                                     (record_type, prev_hash, next_hash))
        else:
            self.logger.warning(u'Something went wrong saving hashes \
                                    (%s, %s, %s)' % (record_type,
                                                     prev_hash,
                                                     next_hash))
            return False
        self.logger.info('Commit data saved to history')
        return True

    def list(self):
        records = self.db.query('SELECT * FROM %s' % (TABLENAME), [])
        return records
Esempio n. 4
0
class MetadataDB(object):
    """
    A key-value store
    It manage a locally persistent that keep whatever data
    that should be available among different run of the client.
    """

    def __init__(self, database_file):
        self.logger = logging.getLogger("FR." + self.__class__.__name__)
        self.db = SQLiteDB(database_file)
        self.filename = database_file

    def _file_exists(self):
        return os.path.exists(self.filename)

    def delete(self):
        """ Deletes the db file from disk """
        if self._file_exists():
            os.unlink(self.filename)

    def _recreate_db_if_not_exists(self):
        """ Recreate the db file if not exists """
        if not os.path.exists(self.filename) or not self._check_database_file():
            self.logger.debug(u"Local metadata database not found. Initializing " "new local metadata DB...")
            self.initialize_new()

    def _query(self, statement, qargs):
        """ Executes a query on database and return its result """
        self._recreate_db_if_not_exists()
        return self.db.query(statement, qargs)

    def _check_database_file(self):
        """
        Check if database file is present and is a regular
        sqlite3 database file.
        """
        query = "select value from metadata where key='last_connected_datetime' ;"
        try:
            result = self.db.check_database_file(query)
        except:
            self.logger.warning(u"Something went wrong attempting default query, result is: %s" % (result))
        return result

    def initialize_new(self):
        """ Initialize a new local dataset database. """
        self.logger.debug(u"Creating local_dataset table... ")
        result = self.db.execute("create table metadata (key text, value text) ;")
        if result:
            self.logger.debug(u"Metadata database table successfully created.")
        else:
            self.logger.warning(u"Something went wrong creating metadata database table... ")
            return False
        result = self.db.execute(
            "insert into metadata values ('last_connected_datetime','Thu, 01 Jan 1970 00:00:00 GMT') ;"
        )
        if result:
            self.logger.debug(u"Default entry successfully inserted in metadata database table.")
        else:
            self.logger.warning(u"Something went wrong inserting default entry into metadata database table... ")
            return False
        return True

    def exists_record(self, key):
        """
        Returns true if there is the key

        @param key: the value of the key you are looking for
        @return: boolean
        """
        res = self._query("SELECT COUNT(*) FROM metadata WHERE key = ?", [key])
        count = compose(fst, fst)(res)  # Damn tuples
        return count > 0

    def get(self, key):
        """
        Looks for the record with the given key

        @param key: the key value
        @return: the value associated with the given key
        """
        statement = "SELECT value FROM metadata WHERE key=?"
        result = self._query(statement, [key])
        if isinstance(result, list) and len(result) == 1:
            return result[0][0]
        elif isinstance(result, list) and len(result) > 1:
            self.logger.warning(u"More than one entry in metadata DB for %s!" % key)
            return result[0][0]
        else:
            self.logger.warning(u"Something went wrong getting %s." % key)
            raise Exception('No key "%s" in metadataDB' % key)

    def _get_value_or_None(self, key):
        """
        Looks for the record with the given key

        @param key: the key value
        @return: the value associated with the given key
        """
        if self.exists_record(key):
            value = self.get(key)
        else:
            value = None
        return value

    def try_get(self, key):
        """
        Tries to get the value associated with the given key

        @return: the value or None
        """
        return self._get_value_or_None(key)

    def set(self, key, value):
        """
        Adds a key or updates the value of a key

        @param key: the key name
        @param value: the key value
        """
        if self.exists_record(key):
            self._update_record(key, value)
        else:
            self._insert_record(key, value)

    def _update_record(self, key, value):
        """
        Updates the value of a key

        @param key: key value
        @param value: the value
        """
        values = [value, key]
        self.db.execute("UPDATE metadata SET value = ? WHERE key = ?", values)

    def delete_key(self, keyvalue):
        """
        Deletes a key

        @param keyvalue: the key to delete
        """
        self.logger.debug("Deleting key %s from metadatadb" % keyvalue)
        self.db.execute("DELETE FROM metadata WHERE key = ?", [keyvalue])

    def _insert_record(self, key, value):
        """
        Adds the key

        @param key: the key
        @param value: the value
        """
        values = [key, value]
        self.db.execute("INSERT INTO metadata VALUES (?, ?)", values)

    def update_last_connected_datetime(self, value=""):
        """
        Update metadata record for last_connected_timestamp with given value.

        @return: result of the update
        """
        if value == "":
            value = formatdate(int(time()), localtime=False, usegmt=True)
        statement = "update metadata set value=? where key=? ;"
        eargs = [value, "last_connected_datetime"]
        result = self.db.execute(statement, eargs)

        # This log is too verbose
        # if result: self.logger.debug(u'Metadata database updated with "last_connected_datetime"="%s"' % (value))
        # We're not even using this in the one-way default version, so let's just log errors if any

        if not result:
            self.logger.warning(u"Something went wrong updating last_connected_datetime! ")
        return result

    def get_last_connected_datetime(self):
        """
        Get last connected timestamp set.

        @return: last_connected_date as string or None
        """
        statement = "select value from metadata where key=? ;"
        result = self._query(statement, ["last_connected_datetime"])
        if isinstance(result, list) and len(result) == 1:
            return result[0][0]
        elif isinstance(result, list) and len(result) > 1:
            self.logger.warning(u"More than one entry in metadata DB for last_connected_datetime!")
            return result[0][0]
        else:
            self.logger.warning(u"Something went wrong getting last_connected_datetime.")
            return None

    def get_all(self):
        """
        Returns a list of tuples

        Any tuple will have the format (key, value)

        @return: a list of tuples
        """
        records = self._query("SELECT * FROM metadata", [])
        result = []
        for record in records:
            key, value = record
            result.append((key, value))
        return result