def account_update(self, account, metadata, to_delete=None, headers=None):
uri = "/v1.0/account/update"
account_id = utils.quote(account, '')
params = {'id': account_id}
data = json.dumps({"metadata": metadata, "to_delete": to_delete})
resp, resp_body = self._account_request('POST', uri, params=params,
data=data, headers=headers)
def _connect_put(chunk):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
hdrs = {}
hdrs["transfer-encoding"] = "chunked"
hdrs[chunk_headers["content_id"]] = sysmeta['id']
hdrs[chunk_headers["content_version"]] = sysmeta['version']
hdrs[chunk_headers["content_path"]] = utils.quote(obj_name)
hdrs[chunk_headers["content_size"]] = sysmeta['content_length']
hdrs[chunk_headers["content_chunkmethod"]] = \
sysmeta['chunk_method']
hdrs[chunk_headers["content_mimetype"]] = sysmeta['mime_type']
hdrs[chunk_headers["content_policy"]] = sysmeta['policy']
hdrs[chunk_headers["content_chunksnb"]] = len(chunks)
hdrs[chunk_headers["container_id"]] = \
utils.name2cid(account, container)
hdrs[chunk_headers["chunk_pos"]] = chunk["pos"]
hdrs[chunk_headers["chunk_id"]] = chunk_path
with ConnectionTimeout(CONNECTION_TIMEOUT):
conn = http_connect(
parsed.netloc, 'PUT', parsed.path, hdrs)
conn.chunk = chunk
return conn
except (Exception, Timeout):
pass
def account_show(self, account, headers=None):
uri = "/v1.0/account/show"
account_id = utils.quote(account, '')
params = {'id': account_id}
resp, resp_body = self._account_request('GET', uri, params=params,
headers=headers)
return resp_body
def _connect_put(chunk):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
h = {}
h["transfer-encoding"] = "chunked"
h[chunk_headers["content_id"]] = self.sysmeta['id']
h[chunk_headers["content_version"]] = self.sysmeta['version']
h[chunk_headers["content_path"]] = \
utils.quote(self.sysmeta['content_path'])
h[chunk_headers["content_chunkmethod"]] = \
self.sysmeta['chunk_method']
h[chunk_headers["content_policy"]] = self.sysmeta['policy']
h[chunk_headers["container_id"]] = self.sysmeta['container_id']
h[chunk_headers["chunk_pos"]] = chunk["pos"]
h[chunk_headers["chunk_id"]] = chunk_path
with ConnectionTimeout(io.CONNECTION_TIMEOUT):
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, h)
conn.chunk = chunk
return conn, chunk
except (Exception, Timeout) as e:
msg = str(e)
logger.error("Failed to connect to %s (%s)", chunk, msg)
chunk['error'] = msg
return None, chunk
def account_create(self, account, headers=None):
uri = '/v1.0/account/create'
account_id = utils.quote(account, '')
params = {'id': account_id}
resp, resp_body = self._account_request('PUT', uri, params=params,
headers=headers)
created = (resp.status_code == 201)
return created
def container_list(self, account, limit=None, marker=None,
end_marker=None, prefix=None, delimiter=None,
headers=None):
uri = "v1.0/account/containers"
account_id = utils.quote(account, '')
params = {"id": account_id, "limit": limit, "marker": marker,
"delimiter": delimiter, "prefix": prefix,
"end_marker": end_marker}
resp, resp_body = self._account_request(
'GET', uri, params=params, headers=headers)
listing = resp_body['listing']
del resp_body['listing']
return listing, resp_body
def _put_stream_rain(self, account, container, obj_name, src, sysmeta,
chunks, headers=None):
global_checksum = hashlib.md5()
total_bytes_transferred = 0
content_chunks = []
content_length = sysmeta['content_length']
def _encode_rawxlist(chunks_at_pos):
res_chunks = []
for subpos, c in chunks_at_pos.iteritems():
host = c['url'].split('/')[2]
chunk_id = c['url'].split('/')[-1]
res_chunks.append("%s/%s" % (host, chunk_id))
return '|'.join(res_chunks)
def _limit_stream(stream, size):
read_size = 0
while read_size < size:
to_read = size - read_size
if to_read > WRITE_CHUNK_SIZE:
to_read = WRITE_CHUNK_SIZE
data = stream.read(to_read)
global_checksum.update(data)
read_size += to_read
yield data
def _decode_chunklist(chunklist):
res = []
for c in chunklist.split(';'):
pos, url, size, hash = c.split('|')
res.append({
"url": "http://%s" % url,
"pos": pos,
"size": int(size),
"hash": hash
})
return res
for pos in xrange(len(chunks)):
rainx_url = self._get_service_url("rainx")
chunk_size = chunks[pos][str(0)]['size']
remaining_bytes = content_length - total_bytes_transferred
if chunk_size > remaining_bytes:
chunk_size = remaining_bytes
headers = {}
headers["X-oio-chunk-meta-content-storage-policy"] = \
sysmeta['policy']
headers["X-oio-chunk-meta-rawxlist"] = \
_encode_rawxlist(chunks[pos])
headers[chunk_headers["content_id"]] = sysmeta['id']
headers[chunk_headers["content_version"]] = sysmeta['version']
headers[chunk_headers["content_path"]] = utils.quote(obj_name)
headers[chunk_headers["content_size"]] = sysmeta['content_length']
headers[chunk_headers["content_chunksnb"]] = len(chunks)
headers[chunk_headers["container_id"]] = \
utils.name2cid(account, container)
headers[chunk_headers["chunk_pos"]] = pos
headers[chunk_headers["chunk_size"]] = chunk_size
headers[chunk_headers["content_chunkmethod"]] = \
sysmeta['chunk_method']
headers[chunk_headers["content_mimetype"]] = sysmeta['mime_type']
headers[chunk_headers["content_policy"]] = sysmeta['policy']
resp = self.session.put(rainx_url,
data=_limit_stream(src, chunk_size),
headers=headers)
resp.raise_for_status()
content_chunks.extend(_decode_chunklist(resp.headers['chunklist']))
total_bytes_transferred += chunk_size
content_checksum = global_checksum.hexdigest()
return content_chunks, total_bytes_transferred, content_checksum
def account_delete(self, account, headers=None):
uri = '/v1.0/account/delete'
account_id = utils.quote(account, '')
params = {'id': account_id}
resp, resp_body = self._account_request('POST', uri, params=params,
headers=headers)