Exemple #1
0
    def test_converter_old_chunk_with_link_on_same_object(self):
        for c in self.chunks:
            convert_to_old_chunk(
                self._chunk_path(c), self.account, self.container, self.path,
                self.version, self.content_id)

        self.api.object_link(
            self.account, self.container, self.path,
            self.account, self.container, self.path)

        linked_meta, linked_chunks = self.api.object_locate(
            self.account, self.container, self.path)
        self.assertNotEqual(self.content_id, linked_meta['id'])

        linked_chunk = random.choice(linked_chunks)
        linked_chunk_volume = linked_chunk['url'].split('/')[2]
        linked_chunk_id = linked_chunk['url'].split('/')[3]
        linked_chunk_path = self._chunk_path(linked_chunk)

        # old xattr not removed
        _, expected_raw_meta = read_chunk_metadata(linked_chunk_path,
                                                   linked_chunk_id)
        expected_raw_meta[chunk_xattr_keys['oio_version']] = OIO_VERSION

        self._convert_and_check(
            linked_chunk_volume, linked_chunk_path,
            {linked_chunk_id: (self.account, self.container,
                               self.path, linked_meta['version'],
                               linked_meta['id'])},
            expected_raw_meta=expected_raw_meta, expected_errors=1)
Exemple #2
0
    def chunk_audit(self, path):
        with open(path) as f:
            try:
                meta = read_chunk_metadata(f)
            except exc.MissingAttribute as e:
                raise exc.FaultyChunk(
                    'Missing extended attribute %s' % e)
            size = int(meta['chunk_size'])
            md5_checksum = meta['chunk_hash'].lower()
            reader = ChunkReader(f, size, md5_checksum)
            with closing(reader):
                for buf in reader:
                    buf_len = len(buf)
                    self.bytes_running_time = ratelimit(
                        self.bytes_running_time,
                        self.max_bytes_per_second,
                        increment=buf_len)
                    self.bytes_processed += buf_len
                    self.total_bytes_processed += buf_len

            try:
                content_cid = meta['content_cid']
                content_path = meta['content_path']
                content_attr, data = self.container_client.content_show(
                    cid=content_cid, path=content_path)

                # Check chunk data
                chunks_nb = 0
                chunk_data = None
                for c in data:
                    if c['url'].endswith(meta['chunk_id']):
                        chunks_nb += 1  # FIXME: won't work with DUP
                        chunk_data = c
                if not chunk_data:
                    raise exc.OrphanChunk('Not found in content')

                if chunk_data['size'] != int(meta['chunk_size']):
                    raise exc.FaultyChunk('Invalid chunk size found')

                if chunk_data['hash'] != meta['chunk_hash']:
                    raise exc.FaultyChunk('Invalid chunk hash found')

                if chunk_data['pos'] != meta['chunk_pos']:
                    raise exc.FaultyChunk('Invalid chunk position found')

                # Check content data
                if content_attr['length'] != meta['content_size']:
                    raise exc.FaultyChunk('Invalid content size found')

                if chunks_nb != int(meta['content_chunksnb']):
                    self.logger.warn('Invalid number of chunks found')
                    # TODO: really count chunks and enable the exception
                    # raise exc.FaultyChunk('Invalid number of chunks found')

            except exc.NotFound:
                raise exc.OrphanChunk('Chunk not found in container')
Exemple #3
0
 def update_index(self, path):
     with open(path) as f:
         try:
             meta = read_chunk_metadata(f)
         except exc.MissingAttribute as e:
             raise exc.FaultyChunk('Missing extended attribute %s' % e)
         data = {'mtime': int(time.time())}
         self.index_client.chunk_push(self.volume_id, meta['container_id'],
                                      meta['content_id'], meta['chunk_id'],
                                      **data)
Exemple #4
0
    def pass_without_lock(self):
        last_report = now()
        count, success, fail = 0, 0, 0
        if self.namespace != self.volume_ns:
            self.logger.warn("Forcing the NS to [%s] (previously [%s])",
                             self.namespace, self.volume_ns)

        self.logger.info("START %s", self.volume)

        paths = paths_gen(self.volume)
        for path in paths:
            chunk_id = path.rsplit('/', 1)[-1]
            if len(chunk_id) != STRLEN_CHUNKID:
                self.logger.warn('WARN Not a chunk %s' % path)
                return
            for c in chunk_id:
                if c not in hexdigits:
                    self.logger.warn('WARN Not a chunk %s' % path)
                    return
            # Action
            try:
                with open(path) as f:
                    meta = read_chunk_metadata(f, chunk_id)
                    self.action(self, path, f, meta)
                    success = success + 1
            except NotFound as e:
                fail = fail + 1
                self.logger.info("ORPHAN %s/%s in %s/%s %s",
                                 meta['content_id'], meta['chunk_id'],
                                 meta['container_id'], meta['content_path'],
                                 str(e))
            except Conflict as e:
                fail = fail + 1
                self.logger.info("ALREADY %s/%s in %s/%s %s",
                                 meta['content_id'], meta['chunk_id'],
                                 meta['container_id'], meta['content_path'],
                                 str(e))
            except Exception as e:
                fail = fail + 1
                self.logger.warn("ERROR %s/%s in %s/%s %s",
                                 meta['content_id'], meta['chunk_id'],
                                 meta['container_id'], meta['content_path'],
                                 str(e))
            count = count + 1

            # TODO(jfs): do the throttling

            # periodical reporting
            t = now()
            if t - last_report > self.report_interval:
                self.logger.info("STEP %d ok %d ko %d",
                                 count, success, fail)

        self.logger.info("FINAL %s %d ok %d ko %d",
                         self.volume, count, success, fail)
Exemple #5
0
    def _convert_and_check(self,
                           chunk_volume,
                           chunk_path,
                           chunk_id_info,
                           expected_raw_meta=None,
                           expected_errors=0):
        conf = self.conf
        conf['volume'] = self.rawx_volumes[chunk_volume]
        converter = BlobConverter(conf, logger=self.logger)
        converter.safe_convert_chunk(chunk_path)
        self.assertEqual(1, converter.total_chunks_processed)
        self.assertEqual(1, converter.passes)
        self.assertEqual(expected_errors, converter.errors)

        checker = Checker(self.ns)
        for chunk_id, info in chunk_id_info.items():
            account, container, path, version, content_id = info
            fullpath = encode_fullpath(account, container, path, version,
                                       content_id)
            cid = cid_from_name(account, container)
            meta, raw_meta = read_chunk_metadata(chunk_path, chunk_id)

            self.assertEqual(meta.get('chunk_id'), chunk_id)
            self.assertEqual(meta.get('container_id'), cid)
            self.assertEqual(meta.get('content_path'), path)
            self.assertEqual(meta.get('content_version'), version)
            self.assertEqual(meta.get('content_id'), content_id)
            self.assertEqual(meta.get('full_path'), fullpath)

            checker.check(
                Target(account,
                       container=container,
                       obj=path,
                       chunk='http://' + converter.volume_id + '/' + chunk_id))
            for _ in checker.run():
                pass
            self.assertTrue(checker.report())

            if expected_raw_meta:
                self.assertDictEqual(expected_raw_meta, raw_meta)
                continue

            self.assertNotIn(CHUNK_XATTR_KEYS['chunk_id'], raw_meta)
            self.assertNotIn(CHUNK_XATTR_KEYS['container_id'], raw_meta)
            self.assertNotIn(CHUNK_XATTR_KEYS['content_path'], raw_meta)
            self.assertNotIn(CHUNK_XATTR_KEYS['content_version'], raw_meta)
            self.assertNotIn(CHUNK_XATTR_KEYS['content_id'], raw_meta)
            self.assertIn(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + chunk_id,
                          raw_meta)
            for k in raw_meta.keys():
                if k.startswith('oio:'):
                    self.fail('old fullpath always existing')
            self.assertEqual(raw_meta[CHUNK_XATTR_KEYS['oio_version']],
                             OIO_VERSION)
Exemple #6
0
 def update_index(self, path):
     with open(path) as f:
         try:
             meta = read_chunk_metadata(f)
         except exc.MissingAttribute as e:
             raise exc.FaultyChunk(
                 'Missing extended attribute %s' % e)
         data = {'mtime': int(time.time())}
         self.index_client.chunk_push(self.volume_id,
                                      meta['container_id'],
                                      meta['content_id'],
                                      meta['chunk_id'],
                                      **data)
Exemple #7
0
    def chunk_file_audit(self, chunk_file, chunk_id):
        try:
            meta, _ = read_chunk_metadata(chunk_file, chunk_id)
        except exc.MissingAttribute as err:
            raise exc.FaultyChunk(err)
        size = int(meta['chunk_size'])
        md5_checksum = meta['chunk_hash'].lower()
        reader = ChunkReader(chunk_file,
                             size,
                             md5_checksum,
                             compression=meta.get("compression", ""))
        with closing(reader):
            for buf in reader:
                buf_len = len(buf)
                self.bytes_running_time = ratelimit(self.bytes_running_time,
                                                    self.max_bytes_per_second,
                                                    increment=buf_len)
                self.bytes_processed += buf_len
                self.total_bytes_processed += buf_len

        try:
            container_id = meta['container_id']
            content_id = meta['content_id']
            _obj_meta, data = self.container_client.content_locate(
                cid=container_id, content=content_id, properties=False)

            # Check chunk data
            chunk_data = None
            metachunks = set()
            for c in data:
                if c['url'].endswith(meta['chunk_id']):
                    metachunks.add(c['pos'].split('.', 2)[0])
                    chunk_data = c
            if not chunk_data:
                raise exc.OrphanChunk('Not found in content')

            metachunk_size = meta.get('metachunk_size')
            if metachunk_size is not None \
                    and chunk_data['size'] != int(metachunk_size):
                raise exc.FaultyChunk('Invalid metachunk size found')

            metachunk_hash = meta.get('metachunk_hash')
            if metachunk_hash is not None \
                    and chunk_data['hash'] != meta['metachunk_hash']:
                raise exc.FaultyChunk('Invalid metachunk hash found')

            if chunk_data['pos'] != meta['chunk_pos']:
                raise exc.FaultyChunk('Invalid chunk position found')

        except exc.NotFound:
            raise exc.OrphanChunk('Chunk not found in container')
Exemple #8
0
    def chunk_audit(self, path):
        with open(path) as f:
            try:
                meta = read_chunk_metadata(f)
            except exc.MissingAttribute as e:
                raise exc.FaultyChunk('Missing extended attribute %s' % e)
            size = int(meta['chunk_size'])
            md5_checksum = meta['chunk_hash'].lower()
            reader = ChunkReader(f, size, md5_checksum)
            with closing(reader):
                for buf in reader:
                    buf_len = len(buf)
                    self.bytes_running_time = ratelimit(
                        self.bytes_running_time,
                        self.max_bytes_per_second,
                        increment=buf_len)
                    self.bytes_processed += buf_len
                    self.total_bytes_processed += buf_len

            try:
                container_id = meta['container_id']
                content_path = meta['content_path']
                content_attr, data = self.container_client.content_show(
                    cid=container_id, path=content_path)

                # Check chunk data
                chunk_data = None
                metachunks = set()
                for c in data:
                    if c['url'].endswith(meta['chunk_id']):
                        metachunks.add(c['pos'].split('.', 2)[0])
                        chunk_data = c
                if not chunk_data:
                    raise exc.OrphanChunk('Not found in content')

                if chunk_data['size'] != int(meta['chunk_size']):
                    raise exc.FaultyChunk('Invalid chunk size found')

                if chunk_data['hash'] != meta['chunk_hash']:
                    raise exc.FaultyChunk('Invalid chunk hash found')

                if chunk_data['pos'] != meta['chunk_pos']:
                    raise exc.FaultyChunk('Invalid chunk position found')

            except exc.NotFound:
                raise exc.OrphanChunk('Chunk not found in container')
Exemple #9
0
    def update_index(self, path, chunk_id):
        with open(path) as file_:
            try:
                meta = None
                if meta is None:
                    meta, _ = read_chunk_metadata(file_, chunk_id)
            except exc.MissingAttribute as err:
                raise exc.FaultyChunk(err)

            data = {'mtime': int(time.time())}
            headers = {REQID_HEADER: request_id('blob-indexer-')}
            self.index_client.chunk_push(self.volume_id,
                                         meta['container_id'],
                                         meta['content_id'],
                                         meta['chunk_id'],
                                         headers=headers,
                                         **data)
Exemple #10
0
    def chunk_audit(self, path):
        with open(path) as f:
            try:
                meta = read_chunk_metadata(f)
            except exc.MissingAttribute as e:
                raise exc.FaultyChunk("Missing extended attribute %s" % e)
            size = int(meta["chunk_size"])
            md5_checksum = meta["chunk_hash"].lower()
            reader = ChunkReader(f, size, md5_checksum)
            with closing(reader):
                for buf in reader:
                    buf_len = len(buf)
                    self.bytes_running_time = ratelimit(
                        self.bytes_running_time, self.max_bytes_per_second, increment=buf_len
                    )
                    self.bytes_processed += buf_len
                    self.total_bytes_processed += buf_len

            try:
                container_id = meta["container_id"]
                content_path = meta["content_path"]
                content_attr, data = self.container_client.content_show(cid=container_id, path=content_path)

                # Check chunk data
                chunk_data = None
                metachunks = set()
                for c in data:
                    if c["url"].endswith(meta["chunk_id"]):
                        metachunks.add(c["pos"].split(".", 2)[0])
                        chunk_data = c
                if not chunk_data:
                    raise exc.OrphanChunk("Not found in content")

                if chunk_data["size"] != int(meta["chunk_size"]):
                    raise exc.FaultyChunk("Invalid chunk size found")

                if chunk_data["hash"] != meta["chunk_hash"]:
                    raise exc.FaultyChunk("Invalid chunk hash found")

                if chunk_data["pos"] != meta["chunk_pos"]:
                    raise exc.FaultyChunk("Invalid chunk position found")

            except exc.NotFound:
                raise exc.OrphanChunk("Chunk not found in container")
Exemple #11
0
 def update_index(self, path):
     with open(path) as f:
         try:
             meta = read_chunk_metadata(f)
         except exc.MissingAttribute as e:
             raise exc.FaultyChunk('Missing extended attribute %s' % e)
         data = {
             'content_version': meta['content_version'],
             'content_nbchunks': meta['content_chunksnb'],
             'content_path': meta['content_path'],
             'content_size': meta['content_size'],
             'chunk_hash': meta['chunk_hash'],
             'chunk_position': meta['chunk_pos'],
             'chunk_size': meta['chunk_size'],
             'mtime': int(time.time())
         }
         self.index_client.chunk_push(self.volume_id, meta['content_cid'],
                                      meta['content_id'], meta['chunk_id'],
                                      **data)
Exemple #12
0
    def update_index(self, path, chunk_id):
        with open(path) as f:
            try:
                meta = None
                if self.convert_chunks and self.converter:
                    _, meta = self.converter.convert_chunk(f, chunk_id)
                if meta is None:
                    meta, _ = read_chunk_metadata(f, chunk_id)
            except exc.MissingAttribute as e:
                raise exc.FaultyChunk('Missing extended attribute %s' % e)

            data = {'mtime': int(time.time())}
            headers = {'X-oio-req-id': 'blob-indexer-' + request_id()[:-13]}
            self.index_client.chunk_push(self.volume_id,
                                         meta['container_id'],
                                         meta['content_id'],
                                         meta['chunk_id'],
                                         headers=headers,
                                         **data)
Exemple #13
0
    def chunk_audit(self, path):
        with open(path) as f:
            try:
                meta = read_chunk_metadata(f)
            except exc.MissingAttribute as e:
                raise exc.FaultyChunk(
                    'Missing extended attribute %s' % e)
            size = int(meta['chunk_size'])
            md5_checksum = meta['chunk_hash'].lower()
            reader = ChunkReader(f, size, md5_checksum)
            with closing(reader):
                for buf in reader:
                    buf_len = len(buf)
                    self.bytes_running_time = ratelimit(
                        self.bytes_running_time,
                        self.max_bytes_per_second,
                        increment=buf_len)
                    self.bytes_processed += buf_len
                    self.total_bytes_processed += buf_len

            try:
                content_cid = meta['content_cid']
                content_path = meta['content_path']
                _, data = self.container_client.content_show(
                    cid=content_cid, path=content_path)
                chunk_data = None
                for c in data:
                    if c['url'].endswith(meta['chunk_id']):
                        chunk_data = c
                if not chunk_data:
                    raise exc.OrphanChunk('Not found in content')

                if chunk_data['size'] != int(meta['chunk_size']):
                    raise exc.FaultyChunk('Invalid chunk size found')

                if chunk_data['hash'] != meta['chunk_hash']:
                    raise exc.FaultyChunk('Invalid chunk hash found')

                if chunk_data['pos'] != meta['chunk_pos']:
                    raise exc.FaultyChunk('Invalid chunk position found')
            except exc.NotFound:
                raise exc.OrphanChunk('Chunk not found in container')
Exemple #14
0
    def pass_chunk_file(self, path):
        chunk_id = path.rsplit('/', 1)[-1]
        if len(chunk_id) != STRLEN_CHUNKID:
            if chunk_id.endswith(CHUNK_SUFFIX_PENDING):
                self.logger.info('Skipping pending chunk %s', path)
            else:
                self.logger.warn('WARN Not a chunk %s', path)
            return
        for char in chunk_id:
            if char not in hexdigits:
                self.logger.warn('WARN Not a chunk %s', path)
                return

        with open(path) as f:
            meta, _ = read_chunk_metadata(f, chunk_id)
            if self.container_ids \
                    and meta['container_id'] in self.container_ids:
                self.logger.debug(
                    'Skipping chunk file (container_id=%s content_path=%s '
                    'content_version=%s content_id=%s chunk_id=%s '
                    'chunk_pos=%s)', meta['container_id'],
                    meta['content_path'], meta['content_version'],
                    meta['content_id'], meta['chunk_id'], meta['chunk_pos'])
                return

            beans = self._beans_from_meta(meta)
            for bean in beans:
                try:
                    self.pass_bean(meta, bean)
                except Exception as exc:
                    self.logger.error(
                        'Failed to pass chunk file (container_id=%s '
                        'content_path=%s content_version=%s content_id=%s '
                        'chunk_id=%s chunk_pos=%s): %s', meta['container_id'],
                        meta['content_path'], meta['content_version'],
                        meta['content_id'], meta['chunk_id'],
                        meta['chunk_pos'], exc)
                    self.bean_errors[bean['type']] = \
                        self.bean_errors[bean['type']] + 1
Exemple #15
0
 def update_index(self, path):
     with open(path) as f:
         try:
             meta = read_chunk_metadata(f)
         except exc.MissingAttribute as e:
             raise exc.FaultyChunk(
                 'Missing extended attribute %s' % e)
         data = {
             'content_version': meta['content_version'],
             'content_nbchunks': meta['content_chunksnb'],
             'content_path': meta['content_path'],
             'content_size': meta['content_size'],
             'chunk_hash': meta['chunk_hash'],
             'chunk_position': meta['chunk_pos'],
             'chunk_size': meta['chunk_size'],
             'mtime': int(time.time())
         }
         self.index_client.chunk_push(self.volume_id,
                                      meta['content_cid'],
                                      meta['content_id'],
                                      meta['chunk_id'],
                                      **data)
Exemple #16
0
    def convert_chunk(self, fd, chunk_id):
        meta, raw_meta = read_chunk_metadata(fd,
                                             chunk_id,
                                             check_chunk_id=False)

        links = meta.get('links', dict())
        for chunk_id2, fullpath2 in links.iteritems():
            self.decode_fullpath(fullpath2)

        fullpath = meta.get('full_path')
        if fullpath is not None:
            self.decode_fullpath(fullpath)
            if meta.get('oio_version') == OIO_VERSION:
                return True, meta

        chunk_inode = os.fstat(fd.fileno()).st_ino
        raw_chunk_id = None
        chunk_id = chunk_id.upper()
        chunk_pos = meta['chunk_pos']
        container_id = meta['container_id'].upper()
        path = meta['content_path']
        version = meta['content_version']
        content_id = meta['content_id'].upper()

        new_fullpaths = dict()
        xattr_to_remove = list()
        success = True

        for k, v in raw_meta.iteritems():
            # fetch raw chunk ID
            if k == XATTR_CHUNK_ID:
                raw_chunk_id = v.upper()

            # search old fullpaths
            if not k.startswith(XATTR_OLD_FULLPATH) \
                    or not is_hexa(k[4:], size=64):
                continue

            try:
                account2, container2, container_id2, path2, version2, \
                    content_id2 = self.decode_old_fullpath(v)

                if container_id == container_id2 and path == path2 \
                        and version == version2:
                    if content_id2 is None:
                        content_id2 = self.content_id_from_name(container_id2,
                                                                path2,
                                                                version2,
                                                                search=True)

                    chunk_id, new_fullpath = self.encode_fullpath(
                        chunk_inode, chunk_id, account2, container2, path2,
                        version2, content_id2)
                    new_fullpaths[chunk_id] = new_fullpath
                else:
                    chunk_id2, new_fullpath = self.get_chunk_id_and_fullpath(
                        chunk_inode,
                        chunk_pos,
                        container_id2,
                        path2,
                        version2,
                        account=account2,
                        container=container2,
                        content_id=content_id2)
                    new_fullpaths[chunk_id2] = new_fullpath

                xattr_to_remove.append(k)
            except Exception as exc:
                success = False
                self.logger.warn('chunk_id=%s old_fullpath=%s: %s', chunk_id,
                                 k, exc)

        # old xattr
        if raw_chunk_id is not None:
            try:
                if raw_chunk_id != chunk_id and raw_chunk_id not in links:
                    if raw_chunk_id not in new_fullpaths:
                        meta2, _ = read_chunk_metadata(fd, raw_chunk_id)
                        container_id2 = meta2['container_id'].upper()
                        path2 = meta2['content_path']
                        version2 = meta2['content_version']
                        content_id2 = meta2['content_id'].upper()

                        raw_chunk_id, new_fullpath = \
                            self.get_chunk_id_and_fullpath(
                                chunk_inode, chunk_pos, container_id2, path2,
                                version2, chunk_id=raw_chunk_id,
                                content_id=content_id2)
                        new_fullpaths[raw_chunk_id] = new_fullpath
                elif raw_chunk_id == chunk_id and fullpath is None:
                    if raw_chunk_id not in new_fullpaths:
                        raw_chunk_id, new_fullpath = \
                            self.get_chunk_id_and_fullpath(
                                chunk_inode, chunk_pos, container_id, path,
                                version, chunk_id=raw_chunk_id,
                                content_id=content_id)
                        new_fullpaths[raw_chunk_id] = new_fullpath
            except Exception as exc:
                success = False
                self.logger.warn('chunk_id=%s (old xattr): %s', raw_chunk_id,
                                 exc)

        self.save_xattr(chunk_id, raw_meta)

        if self.dry_run:
            self.logger.info(
                "[dryrun] Converting chunk %s: success=%s new_fullpaths=%s "
                "xattr_to_remove=%s", chunk_id, str(success),
                str(new_fullpaths), str(xattr_to_remove))
        else:
            # for security, if there is an error, we don't delete old xattr
            modify_xattr(fd, new_fullpaths, success, xattr_to_remove)
        return success, None
Exemple #17
0
    def _cycle_copy(self, path):
        if path:
            self.path = path
        chunkid = random_chunk_id()
        chunkdata = random_buffer(string.printable, 1).encode('utf-8')
        chunkurl = self._rawx_url(chunkid)
        chunkpath = self._chunk_path(chunkid)
        headers1 = self._chunk_attr(chunkid, chunkdata)
        metachunk_hash = md5().hexdigest()
        trailers = {'x-oio-chunk-meta-metachunk-size': '1',
                    'x-oio-chunk-meta-metachunk-hash': metachunk_hash}

        self._check_not_present(chunkurl)
        resp, _ = self._http_request(chunkurl, 'PUT', chunkdata, headers1,
                                     trailers)
        self.assertEqual(201, resp.status)
        self.assertEqual(headers1['x-oio-chunk-meta-chunk-hash'].upper(),
                         resp.getheader('x-oio-chunk-meta-chunk-hash'))
        self.assertEqual(headers1['x-oio-chunk-meta-chunk-size'],
                         resp.getheader('x-oio-chunk-meta-chunk-size'))

        copyid = random_chunk_id()
        copyid = chunkid[:-60] + copyid[-60:]
        copyurl = self._rawx_url(copyid)
        copypath = self._chunk_path(copyid)

        headers2 = {}
        headers2["Destination"] = copyurl
        copy_account = "account-snapshot"
        copy_container = "container-snapshot"
        copy_container_id = cid_from_name(copy_account, copy_container)
        copy_path = path+"-snapshot"
        copy_version = 1456938361143741
        copy_id = random_id(32)
        copy_fullpath = encode_fullpath(
            copy_account, copy_container, copy_path, copy_version, copy_id)
        headers2['x-oio-chunk-meta-full-path'] = copy_fullpath
        resp, _ = self._http_request(chunkurl, 'COPY', '', headers2)
        self.assertEqual(201, resp.status)

        resp, body = self._http_request(chunkurl, 'GET', '', {})
        self.assertEqual(200, resp.status)
        headers1['x-oio-chunk-meta-chunk-hash'] = \
            headers1['x-oio-chunk-meta-chunk-hash'].upper()
        for k, v in headers1.items():
            if k == 'x-oio-chunk-meta-content-path':
                self.assertEqual(unquote(resp.getheader(k)), unquote(str(v)))
            else:
                self.assertEqual(resp.getheader(k), str(v))

        resp, body = self._http_request(copyurl, 'GET', '', {})
        self.assertEqual(200, resp.status)
        headers2_bis = headers1.copy()
        headers2_bis['x-oio-chunk-meta-full-path'] = \
            headers2['x-oio-chunk-meta-full-path']
        headers2_bis['x-oio-chunk-meta-content-path'] = copy_path
        headers2_bis['x-oio-chunk-meta-content-version'] = copy_version
        headers2_bis['x-oio-chunk-meta-content-id'] = copy_id
        headers2_bis['x-oio-chunk-meta-container-id'] = copy_container_id
        headers2_bis['x-oio-chunk-meta-chunk-id'] = copyid
        for k, v in headers2_bis.items():
            if k == 'x-oio-chunk-meta-content-path':
                self.assertEqual(unquote(resp.getheader(k)), unquote(str(v)))
            else:
                self.assertEqual(resp.getheader(k), str(v))

        with open(chunkpath, 'r') as fd:
            meta, _ = read_chunk_metadata(fd, chunkid)
            self.assertEqual(headers1['x-oio-chunk-meta-full-path'],
                             meta['full_path'])
            self.assertEqual(1, len(meta['links']))
            self.assertEqual(headers2['x-oio-chunk-meta-full-path'],
                             meta['links'][copyid])

        with open(copypath, 'r') as fd:
            meta, _ = read_chunk_metadata(fd, copyid)
            self.assertEqual(headers2['x-oio-chunk-meta-full-path'],
                             meta['full_path'])
            self.assertEqual(1, len(meta['links']))
            self.assertEqual(headers1['x-oio-chunk-meta-full-path'],
                             meta['links'][chunkid])

        resp, body = self._http_request(chunkurl, 'DELETE', '', {})
        self.assertEqual(204, resp.status)
        resp, body = self._http_request(chunkurl, 'GET', '', {})
        self.assertEqual(404, resp.status)

        resp, body = self._http_request(copyurl, 'GET', '', {})
        self.assertEqual(200, resp.status)
        self.assertEqual(headers2['x-oio-chunk-meta-full-path'],
                         resp.getheader('x-oio-chunk-meta-full-path'))

        with open(copypath, 'r') as fd:
            meta, _ = read_chunk_metadata(fd, copyid)
            self.assertEqual(headers2['x-oio-chunk-meta-full-path'],
                             meta['full_path'])
            self.assertEqual(0, len(meta['links']))

        resp, body = self._http_request(copyurl, 'DELETE', '', {})
        self.assertEqual(204, resp.status)
        resp, body = self._http_request(copyurl, 'GET', '', {})
        self.assertEqual(404, resp.status)
Exemple #18
0
    def test_read_old_chunk(self):
        metachunk_hash = md5().hexdigest()
        trailers = {'x-oio-chunk-meta-metachunk-size': '1',
                    'x-oio-chunk-meta-metachunk-hash': metachunk_hash}

        chunkid = random_chunk_id()
        chunkdata = random_buffer(string.printable, 1).encode('utf-8')
        chunkurl = self._rawx_url(chunkid)
        chunkpath = self._chunk_path(chunkid)
        headers = self._chunk_attr(chunkid, chunkdata)
        self._check_not_present(chunkurl)

        resp, _ = self._http_request(chunkurl, 'PUT', chunkdata, headers,
                                     trailers)
        self.assertEqual(201, resp.status)

        resp1, data1 = self._http_request(chunkurl, 'GET', '', {})
        self.assertEqual(200, resp1.status)
        headers1 = HeadersDict(resp1.getheaders())
        with open(chunkpath, 'r') as fd:
            meta1, _ = read_chunk_metadata(fd, chunkid)

        convert_to_old_chunk(
            chunkpath, self.account, self.container, self.content_path,
            self.content_version, self.content_id)

        resp2, data2 = self._http_request(chunkurl, 'GET', '', {})
        self.assertEqual(200, resp2.status)
        headers2 = HeadersDict(resp2.getheaders())
        with open(chunkpath, 'r') as fd:
            meta2, _ = read_chunk_metadata(fd, chunkid)

        self.assertEqual(data1, data2)
        del headers1[CHUNK_HEADERS['full_path']]
        del headers1[CHUNK_HEADERS['oio_version']]
        del headers2[CHUNK_HEADERS['oio_version']]
        del headers1["date"]
        del headers2["date"]
        self.assertDictEqual(headers1, headers2)
        del meta1['full_path']
        del meta1['oio_version']
        del meta2['oio_version']
        self.assertDictEqual(meta1, meta2)

        # Copy old chunk
        copyid = random_chunk_id()
        copyid = chunkid[:-60] + copyid[-60:]
        copyurl = self._rawx_url(copyid)
        copypath = self._chunk_path(copyid)
        copycontentid = random_id(32)
        copyheaders = {}
        copyheaders["Destination"] = copyurl
        copyheaders['x-oio-chunk-meta-full-path'] = encode_fullpath(
            "account-snapshot", "container-snapshot",
            self.content_path+"-snapshot", 1456938361143741, copycontentid)
        resp, _ = self._http_request(chunkurl, 'COPY', '', copyheaders)
        self.assertEqual(201, resp.status)

        resp2, data2 = self._http_request(chunkurl, 'GET', '', {})
        self.assertEqual(200, resp2.status)
        headers2 = HeadersDict(resp2.getheaders())
        with open(chunkpath, 'r') as fd:
            meta2, _ = read_chunk_metadata(fd, chunkid)

        self.assertEqual(1, len(meta2['links']))
        self.assertEqual(copyheaders['x-oio-chunk-meta-full-path'],
                         meta2['links'][copyid])
        meta2['links'] = dict()

        self.assertEqual(data1, data2)
        del headers2[CHUNK_HEADERS['oio_version']]
        del headers2["date"]
        self.assertDictEqual(headers1, headers2)
        del meta2['oio_version']
        self.assertDictEqual(meta1, meta2)

        resp3, data3 = self._http_request(copyurl, 'GET', '', {})
        self.assertEqual(200, resp3.status)
        headers3 = HeadersDict(resp3.getheaders())
        with open(copypath, 'r') as fd:
            meta3, _ = read_chunk_metadata(fd, copyid)

        self.assertEqual(
            copyheaders['x-oio-chunk-meta-full-path'],
            headers3['x-oio-chunk-meta-full-path'])
        del headers3['x-oio-chunk-meta-full-path']
        self.assertEqual(
            cid_from_name("account-snapshot", "container-snapshot"),
            headers3['x-oio-chunk-meta-container-id'])
        del headers1['x-oio-chunk-meta-container-id']
        del headers3['x-oio-chunk-meta-container-id']
        self.assertEqual(
            self.content_path+"-snapshot",
            unquote(headers3['x-oio-chunk-meta-content-path']))
        del headers1['x-oio-chunk-meta-content-path']
        del headers3['x-oio-chunk-meta-content-path']
        self.assertEqual(
            '1456938361143741',
            headers3['x-oio-chunk-meta-content-version'])
        del headers1['x-oio-chunk-meta-content-version']
        del headers3['x-oio-chunk-meta-content-version']
        self.assertEqual(
            copycontentid, headers3['x-oio-chunk-meta-content-id'])
        del headers1['x-oio-chunk-meta-content-id']
        del headers3['x-oio-chunk-meta-content-id']
        self.assertEqual(copyid, headers3['x-oio-chunk-meta-chunk-id'])
        del headers1['x-oio-chunk-meta-chunk-id']
        del headers3['x-oio-chunk-meta-chunk-id']

        self.assertEqual(
            copyheaders['x-oio-chunk-meta-full-path'], meta3['full_path'])
        del meta3['full_path']
        self.assertEqual(
            cid_from_name("account-snapshot", "container-snapshot"),
            meta3['container_id'])
        del meta1['container_id']
        del meta3['container_id']
        self.assertEqual(self.content_path+"-snapshot", meta3['content_path'])
        del meta1['content_path']
        del meta3['content_path']
        self.assertEqual('1456938361143741', meta3['content_version'])
        del meta1['content_version']
        del meta3['content_version']
        self.assertEqual(copycontentid, meta3['content_id'])
        del meta1['content_id']
        del meta3['content_id']
        self.assertEqual(copyid, meta3['chunk_id'])
        del meta1['chunk_id']
        del meta3['chunk_id']
        # FIXME the old chunk is invisible
        self.assertEqual(0, len(meta3['links']))

        self.assertEqual(data1, data3)
        del headers3[CHUNK_HEADERS['oio_version']]
        del headers3["date"]
        self.assertDictEqual(headers1, headers3)
        del meta3['oio_version']
        self.assertDictEqual(meta1, meta3)
Exemple #19
0
 def load_chunk_metadata(self, path, chunk_id):
     with open(path) as file_:
         meta, _ = read_chunk_metadata(file_, chunk_id)
         return meta
Exemple #20
0
 def load_chunk_metadata(self, path):
     with open(path) as f:
         return read_chunk_metadata(f)
Exemple #21
0
 def load_chunk_metadata(self, path):
     with open(path) as f:
         return read_chunk_metadata(f)