def chunk_copy(self, from_url, to_url, chunk_id=None, fullpath=None, cid=None, path=None, version=None, content_id=None, **kwargs): stream = None # Check source headers only when new fullpath is not provided kwargs['check_headers'] = not bool(fullpath) try: meta, stream = self.chunk_get(from_url, **kwargs) meta['oio_version'] = OIO_VERSION meta['chunk_id'] = chunk_id or to_url.split('/')[-1] meta['full_path'] = fullpath or meta['full_path'] meta['container_id'] = cid or meta.get('container_id') meta['content_path'] = path or meta.get('content_path') # FIXME: the original keys are the good ones. # ReplicatedMetachunkWriter should be modified to accept them. meta['version'] = version or meta.get('content_version') meta['id'] = content_id or meta.get('content_id') meta['chunk_method'] = meta['content_chunkmethod'] meta['policy'] = meta['content_policy'] bytes_transferred, chunk_hash = self.chunk_put( to_url, meta, stream, **kwargs) finally: if stream: stream.close() try: expected_chunk_size = meta.get('chunk_size') if expected_chunk_size is not None: expected_chunk_size = int(expected_chunk_size) if bytes_transferred != expected_chunk_size: raise exc.ChunkException( 'Size isn\'t the same for the copied chunk') expected_chunk_hash = meta.get('chunk_hash') if expected_chunk_hash is not None: expected_chunk_hash = expected_chunk_hash.upper() chunk_hash = chunk_hash.upper() if chunk_hash != expected_chunk_hash: # Should never happen, the hash is checked by the rawx raise exc.ChunkException( 'Hash isn\'t the same for the copied chunk') except exc.ChunkException: # rollback self.chunk_delete(to_url, **kwargs) raise
def __iter__(self): parts_iter = self.get_iter() if not parts_iter: raise exc.ChunkException() for part in parts_iter: for data in part['iter']: yield data raise StopIteration
def chunk_link(self, target, link, fullpath, headers=None, **kwargs): hdrs = headers.copy() if link is None: link = self._generate_fullchunk_copy(target, **kwargs) hdrs['Destination'] = link hdrs[CHUNK_HEADERS['full_path']] = fullpath resp = self.http_pool.request('COPY', self.resolve_url(target), headers=hdrs) if resp.status != 201: raise exc.ChunkException(resp.status) return resp, link
def chunk_link(self, target, link, fullpath, headers=None, write_timeout=None, **kwargs): hdrs = headers.copy() if link is None: link = self._generate_fullchunk_copy(target, **kwargs) hdrs['Destination'] = link hdrs[CHUNK_HEADERS['full_path']] = fullpath if write_timeout is not None: kwargs['read_timeout'] = write_timeout resp = self._request('COPY', target, headers=hdrs, **kwargs) if resp.status != 201: raise exc.ChunkException(resp.status) return resp, link
def chunk_link(self, target, link, fullpath, headers=None, write_timeout=None, **kwargs): hdrs = headers.copy() if link is None: link = self._generate_fullchunk_copy(target, **kwargs) elif not link.startswith('http://'): offset = target.rfind('/') maxlen = len(target) - offset - 1 link = target[:offset + 1] + link[:maxlen] hdrs['Destination'] = link hdrs[CHUNK_HEADERS['full_path']] = fullpath if write_timeout is not None: kwargs['read_timeout'] = write_timeout resp = self._request('COPY', target, headers=hdrs, **kwargs) if resp.status != 201: raise exc.ChunkException(resp.status) return resp, link
def move_chunk(self, chunk_id, service_id=None, check_quality=False, dry_run=False, max_attempts=3, **kwargs): """ Move a chunk to another place. Optionally ensure that the new place is an improvement over the current one. """ if isinstance(chunk_id, Chunk): current_chunk = chunk_id chunk_id = current_chunk.id service_id = current_chunk.host else: candidates = self.chunks.filter(id=chunk_id) if len(candidates) > 1: if service_id is None: raise exc.ChunkException( "Several chunks with ID %s and no service ID" % (chunk_id, )) candidates = candidates.filter(host=service_id) current_chunk = candidates.one() if current_chunk is None or current_chunk not in self.chunks: raise exc.OrphanChunk("Chunk not found in content") if service_id: other_chunks = self.chunks.filter( metapos=current_chunk.metapos).exclude(host=service_id).all() else: other_chunks = self.chunks.filter( metapos=current_chunk.metapos).exclude(id=chunk_id).all() spare_urls, qualities = self._get_spare_chunk( other_chunks, [current_chunk], position=current_chunk.pos, check_quality=check_quality, max_attempts=max_attempts, **kwargs) # Sort chunks by score to try to copy with higher score. # When scores are close together (e.g. [95, 94, 94, 93, 50]), # don't always start with the highest element. duplicate_chunks = self.chunks \ .filter(pos=current_chunk.pos) \ .sort(key=lambda chunk: _get_weighted_random_score(chunk.raw()), reverse=True) \ .all() if dry_run: self.logger.info('Dry-run: would copy chunk from %s to %s', duplicate_chunks[0].url, spare_urls[0]) else: # To reduce the load on the rawx to decommission, # use one of the rawx with a copy of the chunk to move. for src in duplicate_chunks: try: self.logger.info('Copying chunk from %s to %s', src.url, spare_urls[0]) # TODO(FVE): retry to copy (max_attempts times) self.blob_client.chunk_copy(src.url, spare_urls[0], chunk_id=chunk_id, fullpath=self.full_path, cid=self.container_id, path=self.path, version=self.version, content_id=self.content_id, **kwargs) break except Exception as err: self.logger.warn('Failed to copy chunk from %s to %s: %s', src.url, spare_urls[0], err) if len(duplicate_chunks) == 1: raise else: raise UnrecoverableContent( 'No copy available of chunk to move') self._update_spare_chunk(current_chunk, spare_urls[0]) try: self.blob_client.chunk_delete(current_chunk.url, **kwargs) except Exception as err: self.logger.warn("Failed to delete chunk %s: %s", current_chunk.url, err) current_chunk.url = spare_urls[0] current_chunk.quality = qualities[current_chunk.url] return current_chunk.raw()