def _connect_put(chunk):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
hdrs = {}
hdrs["transfer-encoding"] = "chunked"
hdrs[chunk_headers["content_id"]] = sysmeta['id']
hdrs[chunk_headers["content_version"]] = sysmeta['version']
hdrs[chunk_headers["content_path"]] = utils.quote(obj_name)
hdrs[chunk_headers["content_size"]] = sysmeta['content_length']
hdrs[chunk_headers["content_chunkmethod"]] = \
sysmeta['chunk_method']
hdrs[chunk_headers["content_mimetype"]] = sysmeta['mime_type']
hdrs[chunk_headers["content_policy"]] = sysmeta['policy']
hdrs[chunk_headers["content_chunksnb"]] = len(chunks)
hdrs[chunk_headers["container_id"]] = \
utils.name2cid(account, container)
hdrs[chunk_headers["chunk_pos"]] = chunk["pos"]
hdrs[chunk_headers["chunk_id"]] = chunk_path
with ConnectionTimeout(CONNECTION_TIMEOUT):
conn = http_connect(
parsed.netloc, 'PUT', parsed.path, hdrs)
conn.chunk = chunk
return conn
except (Exception, Timeout):
pass
def _object_create(self, account, container, obj_name, source,
sysmeta, metadata=None, policy=None, headers=None):
meta, raw_chunks = self._content_prepare(
account, container, obj_name, sysmeta['content_length'],
policy=policy, headers=headers)
sysmeta['chunk_size'] = int(meta['X-oio-ns-chunk-size'])
sysmeta['id'] = meta[object_headers['id']]
sysmeta['version'] = meta[object_headers['version']]
sysmeta['policy'] = meta[object_headers['policy']]
sysmeta['mime_type'] = meta[object_headers['mime_type']]
sysmeta['chunk_method'] = meta[object_headers['chunk_method']]
storage_method = STORAGE_METHODS.load(sysmeta['chunk_method'])
chunks = _sort_chunks(raw_chunks, storage_method.ec)
sysmeta['content_path'] = obj_name
sysmeta['container_id'] = utils.name2cid(account, container)
if storage_method.ec:
handler = ECWriteHandler(source, sysmeta, chunks, storage_method,
headers=headers)
else:
handler = ReplicatedWriteHandler(source, sysmeta, chunks,
storage_method, headers=headers)
final_chunks, bytes_transferred, content_checksum = handler.stream()
etag = sysmeta['etag']
if etag and etag.lower() != content_checksum.lower():
raise exc.EtagMismatch(
"given etag %s != computed %s" % (etag, content_checksum))
sysmeta['etag'] = content_checksum
h = {}
h[object_headers['size']] = bytes_transferred
h[object_headers['hash']] = sysmeta['etag']
h[object_headers['version']] = sysmeta['version']
h[object_headers['id']] = sysmeta['id']
h[object_headers['policy']] = sysmeta['policy']
h[object_headers['mime_type']] = sysmeta['mime_type']
h[object_headers['chunk_method']] = sysmeta['chunk_method']
if metadata:
for k, v in metadata.iteritems():
h['%sx-%s' % (constants.OBJECT_METADATA_PREFIX, k)] = v
m, body = self._content_create(account, container, obj_name,
final_chunks, headers=h)
return final_chunks, bytes_transferred, content_checksum
def _put_stream_rain(self, account, container, obj_name, src, sysmeta,
chunks, headers=None):
global_checksum = hashlib.md5()
total_bytes_transferred = 0
content_chunks = []
content_length = sysmeta['content_length']
def _encode_rawxlist(chunks_at_pos):
res_chunks = []
for subpos, c in chunks_at_pos.iteritems():
host = c['url'].split('/')[2]
chunk_id = c['url'].split('/')[-1]
res_chunks.append("%s/%s" % (host, chunk_id))
return '|'.join(res_chunks)
def _limit_stream(stream, size):
read_size = 0
while read_size < size:
to_read = size - read_size
if to_read > WRITE_CHUNK_SIZE:
to_read = WRITE_CHUNK_SIZE
data = stream.read(to_read)
global_checksum.update(data)
read_size += to_read
yield data
def _decode_chunklist(chunklist):
res = []
for c in chunklist.split(';'):
pos, url, size, hash = c.split('|')
res.append({
"url": "http://%s" % url,
"pos": pos,
"size": int(size),
"hash": hash
})
return res
for pos in xrange(len(chunks)):
rainx_url = self._get_service_url("rainx")
chunk_size = chunks[pos][str(0)]['size']
remaining_bytes = content_length - total_bytes_transferred
if chunk_size > remaining_bytes:
chunk_size = remaining_bytes
headers = {}
headers["X-oio-chunk-meta-content-storage-policy"] = \
sysmeta['policy']
headers["X-oio-chunk-meta-rawxlist"] = \
_encode_rawxlist(chunks[pos])
headers[chunk_headers["content_id"]] = sysmeta['id']
headers[chunk_headers["content_version"]] = sysmeta['version']
headers[chunk_headers["content_path"]] = utils.quote(obj_name)
headers[chunk_headers["content_size"]] = sysmeta['content_length']
headers[chunk_headers["content_chunksnb"]] = len(chunks)
headers[chunk_headers["container_id"]] = \
utils.name2cid(account, container)
headers[chunk_headers["chunk_pos"]] = pos
headers[chunk_headers["chunk_size"]] = chunk_size
headers[chunk_headers["content_chunkmethod"]] = \
sysmeta['chunk_method']
headers[chunk_headers["content_mimetype"]] = sysmeta['mime_type']
headers[chunk_headers["content_policy"]] = sysmeta['policy']
resp = self.session.put(rainx_url,
data=_limit_stream(src, chunk_size),
headers=headers)
resp.raise_for_status()
content_chunks.extend(_decode_chunklist(resp.headers['chunklist']))
total_bytes_transferred += chunk_size
content_checksum = global_checksum.hexdigest()
return content_chunks, total_bytes_transferred, content_checksum
