def connect(cls, chunk, sysmeta, reqid=None,
connection_timeout=None, write_timeout=None, **kwargs):
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(sysmeta)
if reqid:
hdrs['X-oio-req-id'] = reqid
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
# in the trailer
# metachunk_size & metachunk_hash
trailers = (CHUNK_HEADERS["metachunk_size"],
CHUNK_HEADERS["metachunk_hash"])
if kwargs.get('chunk_checksum_algo'):
trailers = trailers + (CHUNK_HEADERS["chunk_hash"], )
hdrs["Trailer"] = ', '.join(trailers)
with green.ConnectionTimeout(
connection_timeout or io.CONNECTION_TIMEOUT):
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, hdrs)
conn.chunk = chunk
return cls(chunk, conn, write_timeout=write_timeout, **kwargs)
def connect(cls,
chunk,
sysmeta,
reqid=None,
connection_timeout=None,
write_timeout=None,
**_kwargs):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(sysmeta)
if reqid:
hdrs['X-oio-req-id'] = reqid
hdrs[chunk_headers["chunk_pos"]] = chunk["pos"]
hdrs[chunk_headers["chunk_id"]] = chunk_path
# in the trailer
# metachunk_size & metachunk_hash
hdrs["Trailer"] = ', '.join(
(chunk_headers["metachunk_size"], chunk_headers["metachunk_hash"],
chunk_headers["chunk_hash"]))
with green.ConnectionTimeout(connection_timeout
or io.CONNECTION_TIMEOUT):
conn = io.http_connect(parsed.netloc, 'PUT', parsed.path, hdrs)
conn.chunk = chunk
return cls(chunk, conn, write_timeout=write_timeout)
def _connect_put(self, chunk):
"""
Create a connection in order to PUT `chunk`.
:returns: a tuple with the connection object and `chunk`
"""
raw_url = chunk["url"]
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(self.sysmeta)
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
hdrs.update(self.headers)
with green.ConnectionTimeout(self.connection_timeout):
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, hdrs)
conn.chunk = chunk
return conn, chunk
except (Exception, Timeout) as err:
msg = str(err)
logger.exception("Failed to connect to %s (%s)", chunk, msg)
chunk['error'] = msg
return None, chunk
def connect(cls, chunk, sysmeta, reqid=None,
connection_timeout=None, write_timeout=None, **kwargs):
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(sysmeta)
if reqid:
hdrs[REQID_HEADER] = reqid
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
# in the trailer
# metachunk_size & metachunk_hash
trailers = (CHUNK_HEADERS["metachunk_size"],
CHUNK_HEADERS["metachunk_hash"])
if kwargs.get('chunk_checksum_algo'):
trailers = trailers + (CHUNK_HEADERS["chunk_hash"], )
hdrs["Trailer"] = ', '.join(trailers)
with ConnectionTimeout(
connection_timeout or io.CONNECTION_TIMEOUT):
perfdata = kwargs.get('perfdata', None)
if perfdata is not None:
connect_start = monotonic_time()
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, hdrs, scheme=parsed.scheme)
conn.set_cork(True)
if perfdata is not None:
connect_end = monotonic_time()
perfdata_rawx = perfdata.setdefault('rawx', dict())
perfdata_rawx['connect.' + chunk['url']] = \
connect_end - connect_start
conn.chunk = chunk
return cls(chunk, conn, write_timeout=write_timeout,
reqid=reqid, **kwargs)
def _chunk_attr(self, chunk_id, data, path=None):
if path is not None:
self.content_path = path
headers = headers_from_object_metadata({
'id':
self.content_id,
'version':
self.content_version,
'content_path':
self.content_path,
'chunk_method':
'ec/algo=liberasurecode_rs_vand,k=6,m=3',
'policy':
'TESTPOLICY',
'container_id':
self.cid,
'chunk_hash':
md5(data).hexdigest(),
'full_path':
self.fullpath,
'oio_version':
OIO_VERSION
})
headers[CHUNK_HEADERS['chunk_pos']] = 0
headers[CHUNK_HEADERS['chunk_id']] = chunk_id
headers[CHUNK_HEADERS['chunk_size']] = str(len(data))
return headers
def _connect_put(self, chunk):
"""
Create a connection in order to PUT `chunk`.
:returns: a tuple with the connection object and `chunk`
"""
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(self.sysmeta)
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
hdrs.update(self.headers)
hdrs = encode(hdrs)
with green.ConnectionTimeout(self.connection_timeout):
if self.perfdata is not None:
connect_start = monotonic_time()
conn = io.http_connect(parsed.netloc,
'PUT',
parsed.path,
hdrs,
scheme=parsed.scheme)
conn.set_cork(True)
if self.perfdata is not None:
connect_end = monotonic_time()
perfdata_rawx = self.perfdata.setdefault('rawx', dict())
perfdata_rawx['connect.' + chunk['url']] = \
connect_end - connect_start
conn.chunk = chunk
return conn, chunk
except (SocketError, Timeout) as err:
msg = str(err)
self.logger.warn("Failed to connect to %s (reqid=%s): %s", chunk,
self.reqid, err)
except Exception as err:
msg = str(err)
self.logger.exception("Failed to connect to %s (reqid=%s)", chunk,
self.reqid)
chunk['error'] = msg
return None, chunk
