Beispiel #1
0
    def container_audit(self, path):
        """
        Audits the given container path

        :param path: the path to a container db
        """
        try:
            if not path.endswith('.db'):
                return
            broker = ContainerBroker(path)
            if not broker.is_deleted():
                info = broker.get_info()
                self.container_passes += 1
                self.logger.debug(_('Audit passed for %s'), broker.db_file)
        except (Exception, Timeout):
            self.container_failures += 1
            self.logger.exception(_('ERROR Could not get container info %s'),
                (broker.db_file))
Beispiel #2
0
    def container_audit(self, path):
        """
        Audits the given container path

        :param path: the path to a container db
        """
        try:
            if not path.endswith('.db'):
                return
            broker = ContainerBroker(path)
            if not broker.is_deleted():
                info = broker.get_info()
                self.container_passes += 1
                self.logger.debug(_('Audit passed for %s'), broker.db_file)
        except (Exception, Timeout):
            self.container_failures += 1
            self.logger.exception(_('ERROR Could not get container info %s'),
                                  (broker.db_file))
Beispiel #3
0
    def container_sync(self, path):
        """
        Checks the given path for a container database, determines if syncing
        is turned on for that database and, if so, sends any updates to the
        other container.

        :param path: the path to a container db
        """
        try:
            if not path.endswith('.db'):
                return
            broker = ContainerBroker(path)
            info = broker.get_info()
            x, nodes = self.container_ring.get_nodes(info['account'],
                                                     info['container'])
            for ordinal, node in enumerate(nodes):
                if node['ip'] in self._myips and node['port'] == self._myport:
                    break
            else:
                return
            if not broker.is_deleted():
                sync_to = None
                sync_key = None
                sync_point1 = info['x_container_sync_point1']
                sync_point2 = info['x_container_sync_point2']
                for key, (value, timestamp) in broker.metadata.iteritems():
                    if key.lower() == 'x-container-sync-to':
                        sync_to = value
                    elif key.lower() == 'x-container-sync-key':
                        sync_key = value
                if not sync_to or not sync_key:
                    self.container_skips += 1
                    return
                sync_to = sync_to.rstrip('/')
                err = validate_sync_to(sync_to, self.allowed_sync_hosts)
                if err:
                    self.logger.info(
                        _('ERROR %(db_file)s: %(validate_sync_to_err)s'), {
                            'db_file': broker.db_file,
                            'validate_sync_to_err': err
                        })
                    self.container_failures += 1
                    return
                stop_at = time() + self.container_time
                while time() < stop_at and sync_point2 < sync_point1:
                    rows = broker.get_items_since(sync_point2, 1)
                    if not rows:
                        break
                    row = rows[0]
                    if row['ROWID'] >= sync_point1:
                        break
                    key = hash_path(info['account'],
                                    info['container'],
                                    row['name'],
                                    raw_digest=True)
                    # This node will only intially sync out one third of the
                    # objects (if 3 replicas, 1/4 if 4, etc.). This section
                    # will attempt to sync previously skipped rows in case the
                    # other nodes didn't succeed.
                    if unpack_from('>I', key)[0] % \
                            self.container_ring.replica_count != ordinal:
                        if not self.container_sync_row(row, sync_to, sync_key,
                                                       broker, info):
                            return
                    sync_point2 = row['ROWID']
                    broker.set_x_container_sync_points(None, sync_point2)
                while time() < stop_at:
                    rows = broker.get_items_since(sync_point1, 1)
                    if not rows:
                        break
                    row = rows[0]
                    key = hash_path(info['account'],
                                    info['container'],
                                    row['name'],
                                    raw_digest=True)
                    # This node will only intially sync out one third of the
                    # objects (if 3 replicas, 1/4 if 4, etc.). It'll come back
                    # around to the section above and attempt to sync
                    # previously skipped rows in case the other nodes didn't
                    # succeed.
                    if unpack_from('>I', key)[0] % \
                            self.container_ring.replica_count == ordinal:
                        if not self.container_sync_row(row, sync_to, sync_key,
                                                       broker, info):
                            return
                    sync_point1 = row['ROWID']
                    broker.set_x_container_sync_points(sync_point1, None)
                self.container_syncs += 1
        except (Exception, Timeout), err:
            self.container_failures += 1
            self.logger.exception(_('ERROR Syncing %s'), (broker.db_file))
Beispiel #4
0
    def container_sync(self, path):
        """
        Checks the given path for a container database, determines if syncing
        is turned on for that database and, if so, sends any updates to the
        other container.

        :param path: the path to a container db
        """
        try:
            if not path.endswith('.db'):
                return
            broker = ContainerBroker(path)
            info = broker.get_info()
            x, nodes = self.container_ring.get_nodes(info['account'],
                                                     info['container'])
            for ordinal, node in enumerate(nodes):
                if node['ip'] in self._myips and node['port'] == self._myport:
                    break
            else:
                return
            if not broker.is_deleted():
                sync_to = None
                sync_key = None
                sync_point1 = info['x_container_sync_point1']
                sync_point2 = info['x_container_sync_point2']
                for key, (value, timestamp) in broker.metadata.iteritems():
                    if key.lower() == 'x-container-sync-to':
                        sync_to = value
                    elif key.lower() == 'x-container-sync-key':
                        sync_key = value
                if not sync_to or not sync_key:
                    self.container_skips += 1
                    return
                sync_to = sync_to.rstrip('/')
                err = validate_sync_to(sync_to, self.allowed_sync_hosts)
                if err:
                    self.logger.info(
                        _('ERROR %(db_file)s: %(validate_sync_to_err)s'),
                        {'db_file': broker.db_file,
                         'validate_sync_to_err': err})
                    self.container_failures += 1
                    return
                stop_at = time() + self.container_time
                while time() < stop_at and sync_point2 < sync_point1:
                    rows = broker.get_items_since(sync_point2, 1)
                    if not rows:
                        break
                    row = rows[0]
                    if row['ROWID'] >= sync_point1:
                        break
                    key = hash_path(info['account'], info['container'],
                                    row['name'], raw_digest=True)
                    # This node will only intially sync out one third of the
                    # objects (if 3 replicas, 1/4 if 4, etc.). This section
                    # will attempt to sync previously skipped rows in case the
                    # other nodes didn't succeed.
                    if unpack_from('>I', key)[0] % \
                            self.container_ring.replica_count != ordinal:
                        if not self.container_sync_row(row, sync_to, sync_key,
                                                       broker, info):
                            return
                    sync_point2 = row['ROWID']
                    broker.set_x_container_sync_points(None, sync_point2)
                while time() < stop_at:
                    rows = broker.get_items_since(sync_point1, 1)
                    if not rows:
                        break
                    row = rows[0]
                    key = hash_path(info['account'], info['container'],
                                    row['name'], raw_digest=True)
                    # This node will only intially sync out one third of the
                    # objects (if 3 replicas, 1/4 if 4, etc.). It'll come back
                    # around to the section above and attempt to sync
                    # previously skipped rows in case the other nodes didn't
                    # succeed.
                    if unpack_from('>I', key)[0] % \
                            self.container_ring.replica_count == ordinal:
                        if not self.container_sync_row(row, sync_to, sync_key,
                                                       broker, info):
                            return
                    sync_point1 = row['ROWID']
                    broker.set_x_container_sync_points(sync_point1, None)
                self.container_syncs += 1
        except (Exception, Timeout), err:
            self.container_failures += 1
            self.logger.exception(_('ERROR Syncing %s'), (broker.db_file))