def _connect_put(chunk): raw_url = chunk["url"] parsed = urlparse(raw_url) try: chunk_path = parsed.path.split('/')[-1] hdrs = {} hdrs["transfer-encoding"] = "chunked" hdrs[chunk_headers["content_id"]] = sysmeta['id'] hdrs[chunk_headers["content_version"]] = sysmeta['version'] hdrs[chunk_headers["content_path"]] = utils.quote(obj_name) hdrs[chunk_headers["content_size"]] = sysmeta['content_length'] hdrs[chunk_headers["content_chunkmethod"]] = \ sysmeta['chunk_method'] hdrs[chunk_headers["content_mimetype"]] = sysmeta['mime_type'] hdrs[chunk_headers["content_policy"]] = sysmeta['policy'] hdrs[chunk_headers["content_chunksnb"]] = len(chunks) hdrs[chunk_headers["container_id"]] = \ utils.name2cid(account, container) hdrs[chunk_headers["chunk_pos"]] = chunk["pos"] hdrs[chunk_headers["chunk_id"]] = chunk_path with ConnectionTimeout(CONNECTION_TIMEOUT): conn = http_connect( parsed.netloc, 'PUT', parsed.path, hdrs) conn.chunk = chunk return conn except (Exception, Timeout): pass
def _object_create(self, account, container, obj_name, source, sysmeta, metadata=None, policy=None, headers=None): meta, raw_chunks = self._content_prepare( account, container, obj_name, sysmeta['content_length'], policy=policy, headers=headers) sysmeta['chunk_size'] = int(meta['X-oio-ns-chunk-size']) sysmeta['id'] = meta[object_headers['id']] sysmeta['version'] = meta[object_headers['version']] sysmeta['policy'] = meta[object_headers['policy']] sysmeta['mime_type'] = meta[object_headers['mime_type']] sysmeta['chunk_method'] = meta[object_headers['chunk_method']] storage_method = STORAGE_METHODS.load(sysmeta['chunk_method']) chunks = _sort_chunks(raw_chunks, storage_method.ec) sysmeta['content_path'] = obj_name sysmeta['container_id'] = utils.name2cid(account, container) if storage_method.ec: handler = ECWriteHandler(source, sysmeta, chunks, storage_method, headers=headers) else: handler = ReplicatedWriteHandler(source, sysmeta, chunks, storage_method, headers=headers) final_chunks, bytes_transferred, content_checksum = handler.stream() etag = sysmeta['etag'] if etag and etag.lower() != content_checksum.lower(): raise exc.EtagMismatch( "given etag %s != computed %s" % (etag, content_checksum)) sysmeta['etag'] = content_checksum h = {} h[object_headers['size']] = bytes_transferred h[object_headers['hash']] = sysmeta['etag'] h[object_headers['version']] = sysmeta['version'] h[object_headers['id']] = sysmeta['id'] h[object_headers['policy']] = sysmeta['policy'] h[object_headers['mime_type']] = sysmeta['mime_type'] h[object_headers['chunk_method']] = sysmeta['chunk_method'] if metadata: for k, v in metadata.iteritems(): h['%sx-%s' % (constants.OBJECT_METADATA_PREFIX, k)] = v m, body = self._content_create(account, container, obj_name, final_chunks, headers=h) return final_chunks, bytes_transferred, content_checksum
def _put_stream_rain(self, account, container, obj_name, src, sysmeta, chunks, headers=None): global_checksum = hashlib.md5() total_bytes_transferred = 0 content_chunks = [] content_length = sysmeta['content_length'] def _encode_rawxlist(chunks_at_pos): res_chunks = [] for subpos, c in chunks_at_pos.iteritems(): host = c['url'].split('/')[2] chunk_id = c['url'].split('/')[-1] res_chunks.append("%s/%s" % (host, chunk_id)) return '|'.join(res_chunks) def _limit_stream(stream, size): read_size = 0 while read_size < size: to_read = size - read_size if to_read > WRITE_CHUNK_SIZE: to_read = WRITE_CHUNK_SIZE data = stream.read(to_read) global_checksum.update(data) read_size += to_read yield data def _decode_chunklist(chunklist): res = [] for c in chunklist.split(';'): pos, url, size, hash = c.split('|') res.append({ "url": "http://%s" % url, "pos": pos, "size": int(size), "hash": hash }) return res for pos in xrange(len(chunks)): rainx_url = self._get_service_url("rainx") chunk_size = chunks[pos][str(0)]['size'] remaining_bytes = content_length - total_bytes_transferred if chunk_size > remaining_bytes: chunk_size = remaining_bytes headers = {} headers["X-oio-chunk-meta-content-storage-policy"] = \ sysmeta['policy'] headers["X-oio-chunk-meta-rawxlist"] = \ _encode_rawxlist(chunks[pos]) headers[chunk_headers["content_id"]] = sysmeta['id'] headers[chunk_headers["content_version"]] = sysmeta['version'] headers[chunk_headers["content_path"]] = utils.quote(obj_name) headers[chunk_headers["content_size"]] = sysmeta['content_length'] headers[chunk_headers["content_chunksnb"]] = len(chunks) headers[chunk_headers["container_id"]] = \ utils.name2cid(account, container) headers[chunk_headers["chunk_pos"]] = pos headers[chunk_headers["chunk_size"]] = chunk_size headers[chunk_headers["content_chunkmethod"]] = \ sysmeta['chunk_method'] headers[chunk_headers["content_mimetype"]] = sysmeta['mime_type'] headers[chunk_headers["content_policy"]] = sysmeta['policy'] resp = self.session.put(rainx_url, data=_limit_stream(src, chunk_size), headers=headers) resp.raise_for_status() content_chunks.extend(_decode_chunklist(resp.headers['chunklist'])) total_bytes_transferred += chunk_size content_checksum = global_checksum.hexdigest() return content_chunks, total_bytes_transferred, content_checksum