示例#1
0
def migrate_from_0():
    logging.info("Migrating datastore from version 0 to version 1")

    root_path = layoutmanager.get_instance().get_root_path()
    old_root_path = os.path.join(root_path, "store")
    if not os.path.exists(old_root_path):
        return

    for f in os.listdir(old_root_path):
        uid, ext = os.path.splitext(f)
        if ext != ".metadata":
            continue

        logging.debug("Migrating entry %r", uid)

        new_entry_dir = layoutmanager.get_instance().get_metadata_path(uid)
        if not os.path.exists(new_entry_dir):
            os.makedirs(new_entry_dir)

        try:
            _migrate_metadata(root_path, old_root_path, uid)
            _migrate_file(root_path, old_root_path, uid)
            _migrate_preview(root_path, old_root_path, uid)
        except Exception:
            logging.exception("Error while migrating entry %r", uid)

    # Just be paranoid, it's cheap.
    if old_root_path.endswith("datastore/store"):
        shutil.rmtree(old_root_path)

    logging.info("Migration finished")
示例#2
0
def migrate_from_0():
    logging.info('Migrating datastore from version 0 to version 1')

    root_path = layoutmanager.get_instance().get_root_path()
    old_root_path = os.path.join(root_path, 'store')
    if not os.path.exists(old_root_path):
        return

    for f in os.listdir(old_root_path):
        uid, ext = os.path.splitext(f)
        if ext != '.metadata':
            continue

        logging.debug('Migrating entry %r', uid)

        new_entry_dir = layoutmanager.get_instance().get_metadata_path(uid)
        if not os.path.exists(new_entry_dir):
            os.makedirs(new_entry_dir)

        try:
            _migrate_metadata(root_path, old_root_path, uid)
            _migrate_file(root_path, old_root_path, uid)
            _migrate_preview(root_path, old_root_path, uid)
        except Exception:
            logging.exception('Error while migrating entry %r', uid)

    # Just be paranoid, it's cheap.
    if old_root_path.endswith('datastore/store'):
        shutil.rmtree(old_root_path)

    logging.info('Migration finished')
示例#3
0
 def __init__(self):
     self._database = None
     self._flush_timeout = None
     self._pending_writes = 0
     root_path = layoutmanager.get_instance().get_root_path()
     self._index_updated_path = os.path.join(root_path, 'index_updated')
     self._std_index_path = layoutmanager.get_instance().get_index_path()
     self._index_path = self._std_index_path
示例#4
0
 def __init__(self):
     self._database = None
     self._flush_timeout = None
     self._pending_writes = 0
     root_path=layoutmanager.get_instance().get_root_path()
     self._index_updated_path = os.path.join(root_path,
                                             'index_updated')
     self._std_index_path = layoutmanager.get_instance().get_index_path()
     self._index_path = self._std_index_path
示例#5
0
    def hard_link_entry(self, new_uid, existing_uid):
        existing_file = layoutmanager.get_instance().get_data_path(
            existing_uid)
        new_file = layoutmanager.get_instance().get_data_path(new_uid)

        logging.debug('removing %r', new_file)
        os.remove(new_file)

        logging.debug('hard linking %r -> %r', new_file, existing_file)
        os.link(existing_file, new_file)
示例#6
0
    def store(self, uid, file_path, transfer_ownership, completion_cb):
        """Store a file for a given entry.

        """
        dir_path = layoutmanager.get_instance().get_entry_path(uid)
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

        destination_path = layoutmanager.get_instance().get_data_path(uid)
        if file_path:
            if not os.path.isfile(file_path):
                raise ValueError('No file at %r' % file_path)

            if os.path.islink(file_path):
                # Can't keep symlinks (especially pointed to removable medias).
                # Later, optimizer will help with saving duplicates
                file_path = os.path.realpath(file_path)
                # We should not move original file
                transfer_ownership = False

            if transfer_ownership:
                try:
                    logger.debug('FileStore moving from %r to %r', file_path,
                                 destination_path)
                    os.rename(file_path, destination_path)
                    completion_cb()
                except OSError as e:
                    if e.errno == errno.EXDEV:
                        self._async_copy(file_path,
                                         destination_path,
                                         completion_cb,
                                         unlink_src=True)
                    else:
                        raise
            else:
                self._async_copy(file_path,
                                 destination_path,
                                 completion_cb,
                                 unlink_src=False)
            """
        TODO: How can we support deleting the file of an entry?
        elif not file_path and os.path.exists(destination_path):
            logger.debug('FileStore: deleting %r' % destination_path)
            os.remove(destination_path)
            completion_cb()
            """
        else:
            logger.debug('FileStore: Nothing to do')
            completion_cb()
示例#7
0
    def find(self, query, properties):
        logging.debug('datastore.find %r', query)
        t = time.time()

        if not self._index_updating:
            try:
                uids, count = self._index_store.find(query)
                uids = [uid.decode() for uid in uids]
            except Exception:
                logging.exception('Failed to query index, will rebuild')
                self._rebuild_index()

        if self._index_updating:
            logging.warning('Index updating, returning all entries')
            return self._find_all(query, properties)

        entries = []
        for uid in uids:
            entry_path = layoutmanager.get_instance().get_entry_path(uid)
            if not os.path.exists(entry_path):
                logging.warning(
                    'Inconsistency detected, returning all entries')
                self._rebuild_index()
                return self._find_all(query, properties)

            metadata = self._metadata_store.retrieve(uid, properties)
            self._fill_internal_props(metadata, uid, properties)
            entries.append(metadata)

        logger.debug('find(): %r', time.time() - t)

        return entries, count
示例#8
0
    def store(self, uid, metadata):
        '''
        Create or update an entry to the journal
        '''
        # Name of the entry
        entity_name = layoutmanager.get_instance().get_entity_name(uid)

        # Get or create the entity
        update_stats = False
        entity = None
        if self._ers.entity_exist(entity_name):
            entity = self._ers.get_entity(entity_name, local=True)
        else:
            entity = self._ers.create_entity(entity_name)
            update_stats = True
        
        # If we need to update the stats, do it
        if update_stats and 'activity' in metadata:
            stats_entity_name = 'urn:ers:app:{}:activityStats'.format(metadata['activity'])
            stats_entity = None
            if self._ers.entity_exist(stats_entity_name):
                stats_entity = self._ers.get_entity(stats_entity_name, local=True)
            else:
                stats_entity = self._ers.create_entity(stats_entity_name)
                stats_entity.set_property_value('activity', metadata['activity'])
            stats_entity.add_property_value('usage', entity_name)
            self._ers.persist_entity(stats_entity)
        
        # Update the description of the entity
        metadata['uid'] = str(uid)
        for key, value in metadata.items():
            entity.set_property_value(key, value, private=True)
    
        # Persist the result
        self._ers.persist_entity(entity)
示例#9
0
 def get_property(self, uid, key):
     metadata_path = layoutmanager.get_instance().get_metadata_path(uid)
     property_path = os.path.join(metadata_path, key)
     if os.path.exists(property_path):
         return open(property_path, 'r').read()
     else:
         return None
示例#10
0
 def get_property(self, uid, key):
     metadata_path = layoutmanager.get_instance().get_metadata_path(uid)
     property_path = os.path.join(metadata_path, key)
     if os.path.exists(property_path):
         return open(property_path, 'r').read()
     else:
         return None
示例#11
0
    def delete(self, uid):
        """Remove the file associated to a given entry.

        """
        file_path = layoutmanager.get_instance().get_data_path(uid)
        if os.path.exists(file_path):
            os.remove(file_path)
示例#12
0
    def _set_property(self, uid, key, value, md_path=False):
        if not md_path:
            md_path = layoutmanager.get_instance().get_metadata_path(uid)
        # Hack to support activities that still pass properties named as
            # for example title:text.
        if ':' in key:
            key = key.split(':', 1)[0]

        changed = True
        fpath = os.path.join(md_path, key)
        tpath = os.path.join(md_path, '.' + key)
        # FIXME: this codepath handles raw image data
        # str() is 8-bit clean right now, but
        # this won't last. We will need more explicit
        # handling of strings, int/floats vs raw data
        if isinstance(value, unicode):
            value = value.encode('utf-8')
        elif not isinstance(value, basestring):
            value = str(value)

        # avoid pointless writes; replace atomically
        if os.path.exists(fpath):
            stored_val = open(fpath, 'r').read()

            if stored_val == value:
                changed = False
        if changed:
            f = open(tpath, 'w')
            f.write(value)
            f.close()
            os.rename(tpath, fpath)
示例#13
0
    def _set_property(self, uid, key, value, md_path=False):
        if not md_path:
            md_path = layoutmanager.get_instance().get_metadata_path(uid)
        # Hack to support activities that still pass properties named as
        # for example title:text.
        if ':' in key:
            key = key.split(':', 1)[0]

        changed = True
        fpath = os.path.join(md_path, key)
        tpath = os.path.join(md_path, '.' + key)
        # FIXME: this codepath handles raw image data
        # str() is 8-bit clean right now, but
        # this won't last. We will need more explicit
        # handling of strings, int/floats vs raw data
        if isinstance(value, unicode):
            value = value.encode('utf-8')
        elif not isinstance(value, basestring):
            value = str(value)

        # avoid pointless writes; replace atomically
        if os.path.exists(fpath):
            stored_val = open(fpath, 'r').read()

            if stored_val == value:
                changed = False
        if changed:
            f = open(tpath, 'w')
            f.write(value)
            f.close()
            os.rename(tpath, fpath)
示例#14
0
def _migrate_metadata(root_path, old_root_path, uid):
    metadata_path = layoutmanager.get_instance().get_metadata_path(uid)
    old_metadata_path = os.path.join(old_root_path, uid + '.metadata')
    metadata = json.load(open(old_metadata_path, 'r'))

    if 'uid' not in metadata:
        metadata['uid'] = uid

    if 'timestamp' not in metadata and 'mtime' in metadata:
        metadata['timestamp'] = \
                time.mktime(time.strptime(metadata['mtime'], DATE_FORMAT))

    for key, value in metadata.items():
        try:
            f = open(os.path.join(metadata_path, key), 'w')
            try:
                if isinstance(value, unicode):
                    value = value.encode('utf-8')
                if not isinstance(value, basestring):
                    value = str(value)
                f.write(value)
            finally:
                f.close()
        except Exception:
            logging.exception(
                    'Error while migrating property %s of entry %s', key, uid)
示例#15
0
    def find(self, query, properties):
        logging.debug('datastore.find %r', query)
        t = time.time()

        if not self._index_updating:
            try:
                uids, count = self._index_store.find(query)
            except Exception:
                logging.exception('Failed to query index, will rebuild')
                self._rebuild_index()

        if self._index_updating:
            logging.warning('Index updating, returning all entries')
            return self._find_all(query, properties)

        entries = []
        for uid in uids:
            entry_path = layoutmanager.get_instance().get_entry_path(uid)
            if not os.path.exists(entry_path):
                logging.warning(
                    'Inconsistency detected, returning all entries')
                self._rebuild_index()
                return self._find_all(query, properties)

            metadata = self._metadata_store.retrieve(uid, properties)
            self._fill_internal_props(metadata, uid, properties)
            entries.append(metadata)

        logger.debug('find(): %r', time.time() - t)

        return entries, count
示例#16
0
    def _identical_file_already_exists(self, checksum):
        """Check if we already have files with this checksum.

        """
        checksums_dir = layoutmanager.get_instance().get_checksums_dir()
        checksum_path = os.path.join(checksums_dir, checksum)
        return os.path.exists(checksum_path)
示例#17
0
def _migrate_metadata(root_path, old_root_path, uid):
    metadata_path = layoutmanager.get_instance().get_metadata_path(uid)
    old_metadata_path = os.path.join(old_root_path, uid + '.metadata')
    metadata = json.load(open(old_metadata_path, 'r'))

    if 'uid' not in metadata:
        metadata['uid'] = uid

    if 'timestamp' not in metadata and 'mtime' in metadata:
        metadata['timestamp'] = \
                time.mktime(time.strptime(metadata['mtime'], DATE_FORMAT))

    for key, value in list(metadata.items()):
        try:
            f = open(os.path.join(metadata_path, key), 'w')
            try:
                if isinstance(value, str):
                    value = value.encode('utf-8')
                if not isinstance(value, str):
                    value = str(value)
                f.write(value)
            finally:
                f.close()
        except Exception:
            logging.exception(
                    'Error while migrating property %s of entry %s', key, uid)
示例#18
0
    def _identical_file_already_exists(self, checksum):
        """Check if we already have files with this checksum.

        """
        checksums_dir = layoutmanager.get_instance().get_checksums_dir()
        checksum_path = os.path.join(checksums_dir, checksum)
        return os.path.exists(checksum_path)
示例#19
0
    def _get_uid_from_checksum(self, checksum):
        """Get an existing entry which file matches checksum.

        """
        checksums_dir = layoutmanager.get_instance().get_checksums_dir()
        checksum_path = os.path.join(checksums_dir, checksum)
        first_uid = os.listdir(checksum_path)[0]
        return first_uid
示例#20
0
    def _create_checksum_dir(self, checksum):
        """Create directory that tracks files with this same checksum.

        """
        checksums_dir = layoutmanager.get_instance().get_checksums_dir()
        checksum_path = os.path.join(checksums_dir, checksum)
        logging.debug('create dir %r', checksum_path)
        os.mkdir(checksum_path)
示例#21
0
    def _already_linked(self, uid, checksum):
        """Check if this entry's file is already a hard link to the checksums
           dir.

        """
        checksums_dir = layoutmanager.get_instance().get_checksums_dir()
        checksum_path = os.path.join(checksums_dir, checksum)
        return os.path.exists(os.path.join(checksum_path, uid))
示例#22
0
 def _update_index(self):
     """Find entries that are not yet in the index and add them."""
     uids = layoutmanager.get_instance().find_all()
     logging.debug('Going to update the index with object_ids %r',
                   uids)
     self._index_updating = True
     GObject.idle_add(lambda: self.__update_index_cb(uids),
                      priority=GObject.PRIORITY_LOW)
示例#23
0
    def _already_linked(self, uid, checksum):
        """Check if this entry's file is already a hard link to the checksums
           dir.

        """
        checksums_dir = layoutmanager.get_instance().get_checksums_dir()
        checksum_path = os.path.join(checksums_dir, checksum)
        return os.path.exists(os.path.join(checksum_path, uid))
示例#24
0
 def _update_index(self):
     """Find entries that are not yet in the index and add them."""
     uids = layoutmanager.get_instance().find_all()
     logging.debug('Going to update the index with object_ids %r',
                   uids)
     self._index_updating = True
     GLib.idle_add(lambda: self.__update_index_cb(uids),
                      priority=GLib.PRIORITY_LOW)
示例#25
0
    def _create_checksum_dir(self, checksum):
        """Create directory that tracks files with this same checksum.

        """
        checksums_dir = layoutmanager.get_instance().get_checksums_dir()
        checksum_path = os.path.join(checksums_dir, checksum)
        logging.debug('create dir %r', checksum_path)
        os.mkdir(checksum_path)
示例#26
0
    def _get_uid_from_checksum(self, checksum):
        """Get an existing entry which file matches checksum.

        """
        checksums_dir = layoutmanager.get_instance().get_checksums_dir()
        checksum_path = os.path.join(checksums_dir, checksum)
        first_uid = os.listdir(checksum_path)[0]
        return first_uid
示例#27
0
    def __init__(self, **options):
        bus_name = dbus.service.BusName(DS_SERVICE,
                                        bus=dbus.SessionBus(),
                                        replace_existing=False,
                                        allow_replacement=False)
        dbus.service.Object.__init__(self, bus_name, DS_OBJECT_PATH)

        migrated, initiated = self._open_layout()

        self._metadata_store = MetadataStore()
        self._file_store = FileStore()
        self._optimizer = Optimizer(self._file_store, self._metadata_store)
        self._index_store = IndexStore()
        self._index_updating = False

        root_path = layoutmanager.get_instance().get_root_path()
        self._cleanflag = os.path.join(root_path, 'ds_clean')

        if initiated:
            logging.debug('Initiate datastore')
            self._rebuild_index()
            self._index_store.flush()
            self._mark_clean()
            return

        if migrated:
            self._rebuild_index()
            self._mark_clean()
            return

        rebuild = False
        stat = os.statvfs(root_path)
        da = stat.f_bavail * stat.f_bsize

        if not self._index_store.index_updated:
            logging.warn('Index is not up-to-date')
            rebuild = True
        elif not os.path.exists(self._cleanflag):
            logging.warn('DS state is not clean')
            rebuild = True
        elif da < MIN_INDEX_FREE_BYTES:
            logging.warn('Disk space tight for index')
            rebuild = True

        if rebuild:
            logging.warn('Trigger index rebuild')
            self._rebuild_index()
        else:
            # fast path
            try:
                self._index_store.open_index()
            except BaseException:
                logging.exception('Failed to open index')
                # try...
                self._rebuild_index()

        self._mark_clean()
        return
示例#28
0
    def __init__(self, **options):
        bus_name = dbus.service.BusName(DS_SERVICE,
                                        bus=dbus.SessionBus(),
                                        replace_existing=False,
                                        allow_replacement=False)
        dbus.service.Object.__init__(self, bus_name, DS_OBJECT_PATH)

        migrated, initiated = self._open_layout()

        self._metadata_store = MetadataStore()
        self._file_store = FileStore()
        self._optimizer = Optimizer(self._file_store, self._metadata_store)
        self._index_store = IndexStore()
        self._index_updating = False

        root_path = layoutmanager.get_instance().get_root_path()
        self._cleanflag = os.path.join(root_path, 'ds_clean')

        if initiated:
            logging.debug('Initiate datastore')
            self._rebuild_index()
            self._index_store.flush()
            self._mark_clean()
            return

        if migrated:
            self._rebuild_index()
            self._mark_clean()
            return

        rebuild = False
        stat = os.statvfs(root_path)
        da = stat.f_bavail * stat.f_bsize

        if not self._index_store.index_updated:
            logging.warn('Index is not up-to-date')
            rebuild = True
        elif not os.path.exists(self._cleanflag):
            logging.warn('DS state is not clean')
            rebuild = True
        elif da < MIN_INDEX_FREE_BYTES:
            logging.warn('Disk space tight for index')
            rebuild = True

        if rebuild:
            logging.warn('Trigger index rebuild')
            self._rebuild_index()
        else:
            # fast path
            try:
                self._index_store.open_index()
            except:
                logging.exception('Failed to open index')
                # try...
                self._rebuild_index()

        self._mark_clean()
        return
示例#29
0
    def _add_checksum_entry(self, uid, checksum):
        """Create a file in the checksum dir with the uid of the entry

        """
        checksums_dir = layoutmanager.get_instance().get_checksums_dir()
        checksum_path = os.path.join(checksums_dir, checksum)

        logging.debug('touch %r', os.path.join(checksum_path, uid))
        open(os.path.join(checksum_path, uid), 'w').close()
示例#30
0
 def delete(self, uid):
     '''
     Delete a journal entry
     '''
     # Name of the entry
     entity_name = layoutmanager.get_instance().get_entity_name(uid)
     
     # Delete
     self._ers.delete_entity(entity_name)
    def contains(self, uid):
        '''
        Check if there is a journal entry with the given UID
        '''
        # Name of the entry
        entity_name = layoutmanager.get_instance().get_entity_name(uid)

        # Tells if the UID is in the store or not
        return self._ers.contains_entity(entity_name)
示例#32
0
    def _add_checksum_entry(self, uid, checksum):
        """Create a file in the checksum dir with the uid of the entry

        """
        checksums_dir = layoutmanager.get_instance().get_checksums_dir()
        checksum_path = os.path.join(checksums_dir, checksum)

        logging.debug('touch %r', os.path.join(checksum_path, uid))
        open(os.path.join(checksum_path, uid), 'w').close()
示例#33
0
    def __update_index_cb(self, uids):
        if uids:
            uid = uids.pop()

            logging.debug('Updating entry %r in index. %d to go.', uid,
                          len(uids))

            if not self._index_store.contains(uid):
                try:
                    update_metadata = False
                    props = self._metadata_store.retrieve(uid)
                    if 'filesize' not in props:
                        path = self._file_store.get_file_path(uid)
                        if os.path.exists(path):
                            props['filesize'] = os.stat(path).st_size
                            update_metadata = True
                    if 'timestamp' not in props:
                        props['timestamp'] = str(int(time.time()))
                        update_metadata = True
                    if 'creation_time' not in props:
                        if 'ctime' in props:
                            try:
                                props['creation_time'] = time.mktime(
                                    time.strptime(
                                        props['ctime'],
                                        migration.DATE_FORMAT))
                            except (TypeError, ValueError):
                                pass
                        if 'creation_time' not in props:
                            props['creation_time'] = props['timestamp']
                        update_metadata = True
                    if update_metadata:
                        self._metadata_store.store(uid, props)
                    self._index_store.store(uid, props)
                except Exception:
                    logging.exception('Error processing %r', uid)
                    logging.warn('Will attempt to delete corrupt entry %r',
                                 uid)
                    try:
                        # self.delete(uid) only works on well-formed
                        # entries :-/
                        entry_path = \
                            layoutmanager.get_instance().get_entry_path(uid)
                        shutil.rmtree(entry_path)
                    except Exception:
                        logging.exception('Error deleting corrupt entry %r',
                                          uid)

        if not uids:
            self._index_store.flush()
            self._index_updating = False
            logging.debug('Finished updating index.')
            return False
        else:
            return True
示例#34
0
    def __update_index_cb(self, uids):
        if uids:
            uid = uids.pop()

            logging.debug('Updating entry %r in index. %d to go.', uid,
                          len(uids))

            if not self._index_store.contains(uid):
                try:
                    update_metadata = False
                    props = self._metadata_store.retrieve(uid)
                    if 'filesize' not in props:
                        path = self._file_store.get_file_path(uid)
                        if os.path.exists(path):
                            props['filesize'] = os.stat(path).st_size
                            update_metadata = True
                    if 'timestamp' not in props:
                        props['timestamp'] = str(int(time.time()))
                        update_metadata = True
                    if 'creation_time' not in props:
                        if 'ctime' in props:
                            try:
                                props['creation_time'] = time.mktime(
                                    time.strptime(
                                        props['ctime'],
                                        migration.DATE_FORMAT))
                            except (TypeError, ValueError):
                                pass
                        if 'creation_time' not in props:
                            props['creation_time'] = props['timestamp']
                        update_metadata = True
                    if update_metadata:
                        self._metadata_store.store(uid, props)
                    self._index_store.store(uid, props)
                except Exception:
                    logging.exception('Error processing %r', uid)
                    logging.warn('Will attempt to delete corrupt entry %r',
                                 uid)
                    try:
                        # self.delete(uid) only works on well-formed
                        # entries :-/
                        entry_path = \
                            layoutmanager.get_instance().get_entry_path(uid)
                        shutil.rmtree(entry_path)
                    except Exception:
                        logging.exception('Error deleting corrupt entry %r',
                                          uid)

        if not uids:
            self._index_store.flush()
            self._index_updating = False
            logging.debug('Finished updating index.')
            return False
        else:
            return True
示例#35
0
    def store(self, uid, metadata):
        metadata_path = layoutmanager.get_instance().get_metadata_path(uid)
        if not os.path.exists(metadata_path):
            os.makedirs(metadata_path)
        else:
            received_keys = metadata.keys()
            for key in os.listdir(metadata_path):
                if key not in _INTERNAL_KEYS and key not in received_keys:
                    os.remove(os.path.join(metadata_path, key))

        metadata['uid'] = uid
        for key, value in metadata.items():
            self._set_property(uid, key, value, md_path=metadata_path)
示例#36
0
    def store(self, uid, metadata):
        metadata_path = layoutmanager.get_instance().get_metadata_path(uid)
        if not os.path.exists(metadata_path):
            os.makedirs(metadata_path)
        else:
            received_keys = metadata.keys()
            for key in os.listdir(metadata_path):
                if key not in _INTERNAL_KEYS and key not in received_keys:
                    os.remove(os.path.join(metadata_path, key))

        metadata['uid'] = uid
        for key, value in metadata.items():
            self._set_property(uid, key, value, md_path=metadata_path)
示例#37
0
    def get_property(self, uid, key):
        '''
        Get a single property
        '''
        # Name of the entry
        entity_name = layoutmanager.get_instance().get_entity_name(uid)
        
        # Get all the (accessible) documents describing that identifier
        entity = self._ers.get_entity(entity_name)
        
        # Get all the properties
        description = entity.get_properties()

        # Return the result
        return description[key]
示例#38
0
    def optimize(self, uid):
        """Add an entry to a queue of entries to be checked for duplicates.

        """
        if not os.path.exists(self._file_store.get_file_path(uid)):
            return

        queue_path = layoutmanager.get_instance().get_queue_path()
        open(os.path.join(queue_path, uid), 'w').close()
        logging.debug('optimize %r', os.path.join(queue_path, uid))

        if self._enqueue_checksum_id is None:
            self._enqueue_checksum_id = \
                GLib.idle_add(self._process_entry_cb,
                              priority=GLib.PRIORITY_LOW)
示例#39
0
    def optimize(self, uid):
        """Add an entry to a queue of entries to be checked for duplicates.

        """
        if not os.path.exists(self._file_store.get_file_path(uid)):
            return

        queue_path = layoutmanager.get_instance().get_queue_path()
        open(os.path.join(queue_path, uid), 'w').close()
        logging.debug('optimize %r', os.path.join(queue_path, uid))

        if self._enqueue_checksum_id is None:
            self._enqueue_checksum_id = \
                    GObject.idle_add(self._process_entry_cb,
                                     priority=GObject.PRIORITY_LOW)
示例#40
0
 def retrieve(self, uid, properties=None):
     '''
     Retrieve some properties of a journal entry
     '''
     # Name of the entry        
     entity_name = layoutmanager.get_instance().get_entity_name(uid)
     
     # Get all the (accessible) documents describing that identifier
     entity = self._ers.get_entity(entity_name)
     
     # Get all the properties
     description = entity.get_properties()
             
     # TODO if properties != None filter the output
     return description
示例#41
0
    def _find_all(self, query, properties):
        uids = layoutmanager.get_instance().find_all()
        count = len(uids)

        offset = query.get('offset', 0)
        limit = query.get('limit', MAX_QUERY_LIMIT)
        uids = uids[offset:offset + limit]

        entries = []
        for uid in uids:
            metadata = self._metadata_store.retrieve(uid, properties)
            self._fill_internal_props(metadata, uid, properties)
            entries.append(metadata)

        return entries, count
示例#42
0
    def store(self, uid, file_path, transfer_ownership, completion_cb):
        """Store a file for a given entry.

        """
        dir_path = layoutmanager.get_instance().get_entry_path(uid)
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

        destination_path = layoutmanager.get_instance().get_data_path(uid)
        if file_path:
            if not os.path.isfile(file_path):
                raise ValueError('No file at %r' % file_path)

            if os.path.islink(file_path):
                # Can't keep symlinks (especially pointed to removable medias).
                # Later, optimizer will help with saving duplicates
                file_path = os.path.realpath(file_path)
                # We should not move original file
                transfer_ownership = False

            if transfer_ownership:
                try:
                    logging.debug('FileStore moving from %r to %r', file_path,
                        destination_path)
                    os.rename(file_path, destination_path)
                    completion_cb()
                except OSError, e:
                    if e.errno == errno.EXDEV:
                        self._async_copy(file_path, destination_path,
                                         completion_cb, unlink_src=True)
                    else:
                        raise
            else:
                self._async_copy(file_path, destination_path, completion_cb,
                        unlink_src=False)
            """
示例#43
0
    def _find_all(self, query, properties):
        uids = layoutmanager.get_instance().find_all()
        count = len(uids)

        offset = query.get('offset', 0)
        limit = query.get('limit', MAX_QUERY_LIMIT)
        uids = uids[offset:offset + limit]

        entries = []
        for uid in uids:
            metadata = self._metadata_store.retrieve(uid, properties)
            self._fill_internal_props(metadata, uid, properties)
            entries.append(metadata)

        return entries, count
示例#44
0
    def retrieve(self, uid, user_id, extension):
        """Place the file associated to a given entry into a directory
           where the user can read it. The caller is reponsible for
           deleting this file.

        """
        file_path = layoutmanager.get_instance().get_data_path(uid)
        if not os.path.exists(file_path):
            logger.debug('Entry %r doesnt have any file', uid)
            return ''

        use_instance_dir = os.path.exists('/etc/olpc-security') and \
            os.getuid() != user_id
        if use_instance_dir:
            if not user_id:
                raise ValueError('Couldnt determine the current user uid.')
            destination_dir = os.path.join(os.environ['HOME'], 'isolation',
                                           '1', 'uid_to_instance_dir',
                                           str(user_id))
        else:
            destination_dir = env.get_profile_path('data')
            if not os.path.exists(destination_dir):
                os.makedirs(destination_dir)

        if extension is None:
            extension = ''
        elif extension:
            extension = '.' + extension

        fd, destination_path = tempfile.mkstemp(prefix=uid + '_',
                                                suffix=extension,
                                                dir=destination_dir)
        os.close(fd)
        os.unlink(destination_path)

        # Try to hard link from the original file to the targetpath. This can
        # fail if the file is in a different filesystem. Do a symlink instead.
        try:
            os.link(file_path, destination_path)
        except OSError as e:
            if e.errno == errno.EXDEV:
                os.symlink(file_path, destination_path)
            else:
                raise

        return destination_path
示例#45
0
    def retrieve(self, uid, user_id, extension):
        """Place the file associated to a given entry into a directory
           where the user can read it. The caller is reponsible for
           deleting this file.

        """
        file_path = layoutmanager.get_instance().get_data_path(uid)
        if not os.path.exists(file_path):
            logging.debug('Entry %r doesnt have any file', uid)
            return ''

        use_instance_dir = os.path.exists('/etc/olpc-security') and \
                           os.getuid() != user_id
        if use_instance_dir:
            if not user_id:
                raise ValueError('Couldnt determine the current user uid.')
            destination_dir = os.path.join(os.environ['HOME'], 'isolation',
                '1', 'uid_to_instance_dir', str(user_id))
        else:
            profile = os.environ.get('SUGAR_PROFILE', 'default')
            destination_dir = os.path.join(os.path.expanduser('~'), '.sugar',
                    profile, 'data')
            if not os.path.exists(destination_dir):
                os.makedirs(destination_dir)

        if extension is None:
            extension = ''
        elif extension:
            extension = '.' + extension

        fd, destination_path = tempfile.mkstemp(prefix=uid + '_',
                suffix=extension, dir=destination_dir)
        os.close(fd)
        os.unlink(destination_path)

        # Try to hard link from the original file to the targetpath. This can
        # fail if the file is in a different filesystem. Do a symlink instead.
        try:
            os.link(file_path, destination_path)
        except OSError, e:
            if e.errno == errno.EXDEV:
                os.symlink(file_path, destination_path)
            else:
                raise
示例#46
0
    def set_property(self, uid, key, value):
        '''
        Set a single property
        '''        
        # Name of the entry
        entity_name = layoutmanager.get_instance().get_entity_name(uid)

        # Get all the (accessible) documents describing that identifier
        entity = self._ers.get_entity(entity_name)

        # Set new value
        #if isinstance(value, unicode):
        #    value = value.encode('utf-8')
        #elif not isinstance(value, basestring):
        #    value = str(value)
        entity.set_property_value(key, value, private=True)
        
        # Persist
        self._ers.persist_entity(entity)
示例#47
0
    def _rebuild_index(self):
        """Remove and recreate index."""
        self._index_store.close_index()
        self._index_store.remove_index()

        # rebuild the index in tmpfs to better handle ENOSPC
        temp_index_path = tempfile.mkdtemp(prefix='sugar-datastore-index-')
        logger.debug('Rebuilding index in %s' % temp_index_path)
        self._index_store.open_index(temp_path=temp_index_path)
        self._update_index()
        self._index_store.close_index()

        on_disk = False

        # can we fit the index on disk? get disk usage in bytes...
        index_du = subprocess.check_output(['/usr/bin/du', '-bs',
                                            temp_index_path])
        index_du = int(index_du.split(b'\t')[0])
        # disk available, in bytes
        stat = os.statvfs(temp_index_path)
        da = stat.f_bavail * stat.f_bsize
        if da > (index_du * 1.2) and da > MIN_INDEX_FREE_BYTES:
            # 1.2 due to 20% room for growth
            logger.debug('Attempting to move tempfs index to disk')
            # move to internal disk
            try:
                index_path = layoutmanager.get_instance().get_index_path()
                if os.path.exists(index_path):
                    shutil.rmtree(index_path)
                shutil.copytree(temp_index_path, index_path)
                shutil.rmtree(temp_index_path)
                on_disk = True
            except Exception:
                logger.exception('Error copying tempfs index to disk,'
                                 'revert to using tempfs index.')
        else:
            logger.warn("Not enough disk space, using tempfs index")

        if on_disk:
            self._index_store.open_index()
        else:
            self._index_store.open_index(temp_path=temp_index_path)
示例#48
0
    def _rebuild_index(self):
        """Remove and recreate index."""
        self._index_store.close_index()
        self._index_store.remove_index()

        # rebuild the index in tmpfs to better handle ENOSPC
        temp_index_path = tempfile.mkdtemp(prefix='sugar-datastore-index-')
        logger.warn('Rebuilding index in %s' % temp_index_path)
        self._index_store.open_index(temp_path=temp_index_path)
        self._update_index()
        self._index_store.close_index()

        on_disk = False

        # can we fit the index on disk? get disk usage in bytes...
        index_du = subprocess.check_output(['/usr/bin/du', '-bs',
                                            temp_index_path])
        index_du = int(index_du.split('\t')[0])
        # disk available, in bytes
        stat = os.statvfs(temp_index_path)
        da = stat.f_bavail * stat.f_bsize
        if da > (index_du * 1.2) and da > MIN_INDEX_FREE_BYTES:
            # 1.2 due to 20% room for growth
            logger.warn('Attempting to move tempfs index to disk')
            # move to internal disk
            try:
                index_path = layoutmanager.get_instance().get_index_path()
                if os.path.exists(index_path):
                    shutil.rmtree(index_path)
                shutil.copytree(temp_index_path, index_path)
                shutil.rmtree(temp_index_path)
                on_disk = True
            except Exception:
                logger.exception('Error copying tempfs index to disk,'
                                 'revert to using tempfs index.')
        else:
            logger.warn("Not enough disk space, using tempfs index")

        if on_disk:
            self._index_store.open_index()
        else:
            self._index_store.open_index(temp_path=temp_index_path)
示例#49
0
    def delete(self, uid):
        self._mark_dirty()
        try:
            entry_path = layoutmanager.get_instance().get_entry_path(uid)
            self._optimizer.remove(uid)
            self._index_store.delete(uid)
            self._file_store.delete(uid)
            self._metadata_store.delete(uid)
            # remove the dirtree
            shutil.rmtree(entry_path)
            try:
                # will remove the hashed dir if nothing else is there
                os.removedirs(os.path.dirname(entry_path))
            except BaseException:
                pass
        except BaseException:
            logger.exception('Exception deleting entry')
            raise

        self.Deleted(uid)
        logger.debug('deleted %s', uid)
        self._mark_clean()
示例#50
0
    def _set_property(self, uid, key, value, md_path=False):
        """Set a property in metadata store

        Value datatypes are almost entirely dbus.String, with
        exceptions for certain keys as follows;

        * "timestamp", and "creation_time" of dbus.Int32,
        * "preview" of dbus.ByteArray,
        * "filesize" of int, and
        * "checksum" of str.
        """
        if not md_path:
            md_path = layoutmanager.get_instance().get_metadata_path(uid)
        # Hack to support activities that still pass properties named as
            # for example title:text.
        if ':' in key:
            key = key.split(':', 1)[0]

        changed = True
        fpath = os.path.join(md_path, key)
        tpath = os.path.join(md_path, '.' + key)

        if isinstance(value, int):  # int or dbus.Int32
            value = str(value).encode()
        elif isinstance(value, str):  # str or dbus.String
            value = value.encode()

        # avoid pointless writes; replace atomically
        if os.path.exists(fpath):
            f = open(fpath, 'rb')
            stored_val = f.read()
            f.close()
            if stored_val == value:
                changed = False
        if changed:
            f = open(tpath, 'wb')
            f.write(value)
            f.close()
            os.rename(tpath, fpath)
示例#51
0
    def _process_entry_cb(self):
        """Process one item in the checksums queue by calculating its checksum,
           checking if there exist already an identical file, and in that case
           substituting its file with a hard link to that pre-existing file.

        """
        queue_path = layoutmanager.get_instance().get_queue_path()
        queue = os.listdir(queue_path)
        if queue:
            uid = queue[0]
            logging.debug('_process_entry_cb processing %r', uid)

            file_in_entry_path = self._file_store.get_file_path(uid)
            if not os.path.exists(file_in_entry_path):
                logging.info('non-existent entry in queue: %r', uid)
            else:
                checksum = self._calculate_md5sum(file_in_entry_path)
                self._metadata_store.set_property(uid, 'checksum', checksum)

                if self._identical_file_already_exists(checksum):
                    if not self._already_linked(uid, checksum):
                        existing_entry_uid = \
                                self._get_uid_from_checksum(checksum)

                        self._file_store.hard_link_entry(uid,
                                                         existing_entry_uid)

                        self._add_checksum_entry(uid, checksum)
                else:
                    self._create_checksum_dir(checksum)
                    self._add_checksum_entry(uid, checksum)

            os.remove(os.path.join(queue_path, uid))

        if len(queue) <= 1:
            self._enqueue_checksum_id = None
            return False
        else:
            return True
示例#52
0
    def _process_entry_cb(self):
        """Process one item in the checksums queue by calculating its checksum,
           checking if there exist already an identical file, and in that case
           substituting its file with a hard link to that pre-existing file.

        """
        queue_path = layoutmanager.get_instance().get_queue_path()
        queue = os.listdir(queue_path)
        if queue:
            uid = queue[0]
            logging.debug('_process_entry_cb processing %r', uid)

            file_in_entry_path = self._file_store.get_file_path(uid)
            if not os.path.exists(file_in_entry_path):
                logging.info('non-existent entry in queue: %r', uid)
            else:
                checksum = self._calculate_md5sum(file_in_entry_path)
                self._metadata_store.set_property(uid, 'checksum', checksum)

                if self._identical_file_already_exists(checksum):
                    if not self._already_linked(uid, checksum):
                        existing_entry_uid = \
                            self._get_uid_from_checksum(checksum)

                        self._file_store.hard_link_entry(
                            uid, existing_entry_uid)

                        self._add_checksum_entry(uid, checksum)
                else:
                    self._create_checksum_dir(checksum)
                    self._add_checksum_entry(uid, checksum)

            os.remove(os.path.join(queue_path, uid))

        if len(queue) <= 1:
            self._enqueue_checksum_id = None
            return False
        else:
            return True
示例#53
0
    def delete(self, uid):
        self._mark_dirty()
        try:
            entry_path = layoutmanager.get_instance().get_entry_path(uid)
            self._optimizer.remove(uid)
            self._index_store.delete(uid)
            self._file_store.delete(uid)
            self._metadata_store.delete(uid)
            # remove the dirtree
            shutil.rmtree(entry_path)
            try:
                # will remove the hashed dir if nothing else is there
                os.removedirs(os.path.dirname(entry_path))
            except:
                pass
        except:
            logger.exception('Exception deleting entry')
            raise

        self.Deleted(uid)
        logger.debug('deleted %s', uid)
        self._mark_clean()
示例#54
0
    def remove(self, uid):
        """Remove any structures left from space optimization

        """
        checksum = self._metadata_store.get_property(uid, 'checksum')
        if checksum is None:
            return

        checksums_dir = layoutmanager.get_instance().get_checksums_dir()
        checksum_path = os.path.join(checksums_dir, checksum)
        checksum_entry_path = os.path.join(checksum_path, uid)

        if os.path.exists(checksum_entry_path):
            logging.debug('remove %r', checksum_entry_path)
            os.remove(checksum_entry_path)

        if os.path.exists(checksum_path):
            try:
                os.rmdir(checksum_path)
                logging.debug('removed %r', checksum_path)
            except OSError as e:
                if e.errno != errno.ENOTEMPTY:
                    raise
示例#55
0
    def _open_layout(self):
        """Open layout manager, check version of data store on disk and
        migrate if necessary.

        Returns a pair of booleans. For the first, True if migration was done
        and an index rebuild is required. For the second, True if datastore was
        just initiated.
        """
        layout_manager = layoutmanager.get_instance()

        if layout_manager.is_empty():
            layout_manager.set_version(layoutmanager.CURRENT_LAYOUT_VERSION)
            return False, True

        old_version = layout_manager.get_version()
        if old_version == layoutmanager.CURRENT_LAYOUT_VERSION:
            return False, False

        if old_version == 0:
            migration.migrate_from_0()

        layout_manager.set_version(layoutmanager.CURRENT_LAYOUT_VERSION)
        return True, False
示例#56
0
    def retrieve(self, uid, properties=None):
        """Retrieve metadata for an object from the store.

        Values are read as dbus.ByteArray, then converted to expected
        types.
        """
        metadata_path = layoutmanager.get_instance().get_metadata_path(uid)

        if properties is not None:
            properties = [x.encode('utf-8') if isinstance(x, str)
                          else x for x in properties]

        metadata = metadatareader.retrieve(metadata_path, properties)

        # convert from dbus.ByteArray to expected types
        for key, value in metadata.items():
            if key in ['filesize', 'creation_time', 'timestamp']:
                metadata[key] = dbus.Int32(value)
            elif key in ['checksum']:
                metadata[key] = value.decode()
            elif key != 'preview':
                metadata[key] = dbus.String(value.decode())

        return metadata
示例#57
0
def _migrate_file(root_path, old_root_path, uid):
    if os.path.exists(os.path.join(old_root_path, uid)):
        new_data_path = layoutmanager.get_instance().get_data_path(uid)
        os.rename(os.path.join(old_root_path, uid),
                  new_data_path)
示例#58
0
def _migrate_preview(root_path, old_root_path, uid):
    metadata_path = layoutmanager.get_instance().get_metadata_path(uid)
    os.rename(os.path.join(old_root_path, 'preview', uid),
              os.path.join(metadata_path, 'preview'))