def connect(cls,
chunk,
sysmeta,
reqid=None,
connection_timeout=None,
write_timeout=None,
**_kwargs):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
h = {}
h["transfer-encoding"] = "chunked"
h[chunk_headers["content_id"]] = sysmeta['id']
h[chunk_headers["content_path"]] = sysmeta['content_path']
h[chunk_headers["content_chunkmethod"]] = sysmeta['chunk_method']
h[chunk_headers["container_id"]] = sysmeta['container_id']
h[chunk_headers["chunk_pos"]] = chunk["pos"]
h[chunk_headers["chunk_id"]] = chunk_path
h[chunk_headers["content_policy"]] = sysmeta['policy']
h[chunk_headers["content_version"]] = sysmeta['version']
if reqid:
h['X-oio-req-id'] = reqid
# in the trailer
# metachunk_size & metachunk_hash
h["Trailer"] = (chunk_headers["metachunk_size"],
chunk_headers["metachunk_hash"])
with green.ConnectionTimeout(connection_timeout
or io.CONNECTION_TIMEOUT):
conn = io.http_connect(parsed.netloc, 'PUT', parsed.path, h)
conn.chunk = chunk
return cls(chunk, conn, write_timeout=write_timeout)
def connect(cls, chunk, sysmeta, reqid=None,
connection_timeout=None, write_timeout=None, **kwargs):
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(sysmeta)
if reqid:
hdrs['X-oio-req-id'] = reqid
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
# in the trailer
# metachunk_size & metachunk_hash
trailers = (CHUNK_HEADERS["metachunk_size"],
CHUNK_HEADERS["metachunk_hash"])
if kwargs.get('chunk_checksum_algo'):
trailers = trailers + (CHUNK_HEADERS["chunk_hash"], )
hdrs["Trailer"] = ', '.join(trailers)
with green.ConnectionTimeout(
connection_timeout or io.CONNECTION_TIMEOUT):
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, hdrs)
conn.chunk = chunk
return cls(chunk, conn, write_timeout=write_timeout, **kwargs)
def connect(cls, chunk, sysmeta, reqid=None):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
h = {}
h["transfer-encoding"] = "chunked"
h[chunk_headers["content_id"]] = sysmeta['id']
h[chunk_headers["content_path"]] = sysmeta['content_path']
h[chunk_headers["content_chunkmethod"]] = sysmeta['chunk_method']
h[chunk_headers["container_id"]] = sysmeta['container_id']
h[chunk_headers["chunk_pos"]] = chunk["pos"]
h[chunk_headers["chunk_id"]] = chunk_path
h[chunk_headers["content_policy"]] = sysmeta['policy']
h[chunk_headers["content_version"]] = sysmeta['version']
if reqid:
h['X-oio-req-id'] = reqid
# in the trailer
# metachunk_size & metachunk_hash
h["Trailer"] = (chunk_headers["metachunk_size"],
chunk_headers["metachunk_hash"])
with ConnectionTimeout(io.CONNECTION_TIMEOUT):
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, h)
conn.chunk = chunk
return cls(chunk, conn)
def connect(cls,
chunk,
sysmeta,
reqid=None,
connection_timeout=None,
write_timeout=None,
**_kwargs):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(sysmeta)
if reqid:
hdrs['X-oio-req-id'] = reqid
hdrs[chunk_headers["chunk_pos"]] = chunk["pos"]
hdrs[chunk_headers["chunk_id"]] = chunk_path
# in the trailer
# metachunk_size & metachunk_hash
hdrs["Trailer"] = ', '.join(
(chunk_headers["metachunk_size"], chunk_headers["metachunk_hash"],
chunk_headers["chunk_hash"]))
with green.ConnectionTimeout(connection_timeout
or io.CONNECTION_TIMEOUT):
conn = io.http_connect(parsed.netloc, 'PUT', parsed.path, hdrs)
conn.chunk = chunk
return cls(chunk, conn, write_timeout=write_timeout)
def _connect_put(self, chunk):
"""
Create a connection in order to PUT `chunk`.
:returns: a tuple with the connection object and `chunk`
"""
raw_url = chunk["url"]
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(self.sysmeta)
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
hdrs.update(self.headers)
with green.ConnectionTimeout(self.connection_timeout):
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, hdrs)
conn.chunk = chunk
return conn, chunk
except (Exception, Timeout) as err:
msg = str(err)
logger.exception("Failed to connect to %s (%s)", chunk, msg)
chunk['error'] = msg
return None, chunk
def connect(cls, chunk, sysmeta, reqid=None,
connection_timeout=None, write_timeout=None, **kwargs):
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(sysmeta)
if reqid:
hdrs[REQID_HEADER] = reqid
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
# in the trailer
# metachunk_size & metachunk_hash
trailers = (CHUNK_HEADERS["metachunk_size"],
CHUNK_HEADERS["metachunk_hash"])
if kwargs.get('chunk_checksum_algo'):
trailers = trailers + (CHUNK_HEADERS["chunk_hash"], )
hdrs["Trailer"] = ', '.join(trailers)
with ConnectionTimeout(
connection_timeout or io.CONNECTION_TIMEOUT):
perfdata = kwargs.get('perfdata', None)
if perfdata is not None:
connect_start = monotonic_time()
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, hdrs, scheme=parsed.scheme)
conn.set_cork(True)
if perfdata is not None:
connect_end = monotonic_time()
perfdata_rawx = perfdata.setdefault('rawx', dict())
perfdata_rawx['connect.' + chunk['url']] = \
connect_end - connect_start
conn.chunk = chunk
return cls(chunk, conn, write_timeout=write_timeout,
reqid=reqid, **kwargs)
def _connect_put(chunk):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
h = {}
h["transfer-encoding"] = "chunked"
h[chunk_headers["content_id"]] = self.sysmeta['id']
h[chunk_headers["content_version"]] = self.sysmeta['version']
h[chunk_headers["content_path"]] = \
utils.quote(self.sysmeta['content_path'])
h[chunk_headers["content_chunkmethod"]] = \
self.sysmeta['chunk_method']
h[chunk_headers["content_policy"]] = self.sysmeta['policy']
h[chunk_headers["container_id"]] = self.sysmeta['container_id']
h[chunk_headers["chunk_pos"]] = chunk["pos"]
h[chunk_headers["chunk_id"]] = chunk_path
with ConnectionTimeout(io.CONNECTION_TIMEOUT):
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, h)
conn.chunk = chunk
return conn, chunk
except (Exception, Timeout) as e:
msg = str(e)
logger.error("Failed to connect to %s (%s)", chunk, msg)
chunk['error'] = msg
return None, chunk
def _connect_put(chunk):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
h = {}
h["transfer-encoding"] = "chunked"
h[chunk_headers["content_id"]] = self.sysmeta['id']
h[chunk_headers["content_version"]] = self.sysmeta['version']
h[chunk_headers["content_path"]] = \
utils.quote(self.sysmeta['content_path'])
h[chunk_headers["content_chunkmethod"]] = \
self.sysmeta['chunk_method']
h[chunk_headers["content_policy"]] = self.sysmeta['policy']
h[chunk_headers["container_id"]] = self.sysmeta['container_id']
h[chunk_headers["chunk_pos"]] = chunk["pos"]
h[chunk_headers["chunk_id"]] = chunk_path
with ConnectionTimeout(io.CONNECTION_TIMEOUT):
conn = io.http_connect(parsed.netloc, 'PUT', parsed.path,
h)
conn.chunk = chunk
return conn, chunk
except (Exception, Timeout) as e:
msg = str(e)
logger.error("Failed to connect to %s (%s)", chunk, msg)
chunk['error'] = msg
return None, chunk
def _get_response(self, chunk, headers):
resp = None
parsed = urlparse(chunk['url'])
try:
with ConnectionTimeout(self.connection_timeout):
conn = io.http_connect(
parsed.netloc, 'GET', parsed.path, headers)
with Timeout(self.response_timeout):
resp = conn.getresponse()
if resp.status != 200:
logger.warning('Invalid GET response from %s', chunk)
resp = None
except (Exception, Timeout):
logger.exception('ERROR fetching %s', chunk)
return resp
def _get_response(self, chunk, headers):
resp = None
parsed = urlparse(chunk['url'])
try:
with green.ConnectionTimeout(self.connection_timeout):
conn = io.http_connect(parsed.netloc, 'GET', parsed.path,
headers)
with Timeout(self.read_timeout):
resp = conn.getresponse()
if resp.status != 200:
logger.warning('Invalid GET response from %s', chunk)
resp = None
except (Exception, Timeout):
logger.exception('ERROR fetching %s', chunk)
return resp
def _get_response(self, chunk, headers):
resp = None
parsed = urlparse(chunk.get('real_url', chunk['url']))
try:
with green.ConnectionTimeout(self.connection_timeout):
conn = io.http_connect(
parsed.netloc, 'GET', parsed.path, headers)
with Timeout(self.read_timeout):
resp = conn.getresponse()
if resp.status != 200:
logger.warning('Invalid GET response from %s: %s %s',
chunk, resp.status, resp.reason)
resp = None
except (SocketError, Timeout) as err:
logger.error('ERROR fetching %s: %s', chunk, err)
except Exception:
logger.exception('ERROR fetching %s', chunk)
return resp
def _connect_put(self, chunk):
"""
Create a connection in order to PUT `chunk`.
:returns: a tuple with the connection object and `chunk`
"""
raw_url = chunk.get("real_url", chunk["url"])
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
hdrs = headers_from_object_metadata(self.sysmeta)
hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"]
hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path
hdrs.update(self.headers)
hdrs = encode(hdrs)
with green.ConnectionTimeout(self.connection_timeout):
if self.perfdata is not None:
connect_start = monotonic_time()
conn = io.http_connect(parsed.netloc,
'PUT',
parsed.path,
hdrs,
scheme=parsed.scheme)
conn.set_cork(True)
if self.perfdata is not None:
connect_end = monotonic_time()
perfdata_rawx = self.perfdata.setdefault('rawx', dict())
perfdata_rawx['connect.' + chunk['url']] = \
connect_end - connect_start
conn.chunk = chunk
return conn, chunk
except (SocketError, Timeout) as err:
msg = str(err)
self.logger.warn("Failed to connect to %s (reqid=%s): %s", chunk,
self.reqid, err)
except Exception as err:
msg = str(err)
self.logger.exception("Failed to connect to %s (reqid=%s)", chunk,
self.reqid)
chunk['error'] = msg
return None, chunk
def connect(cls, chunk, sysmeta):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
h = {}
h["transfer-encoding"] = "chunked"
h[chunk_headers["content_id"]] = sysmeta['id']
h[chunk_headers["content_path"]] = sysmeta['content_path']
h[chunk_headers["content_chunkmethod"]] = sysmeta['chunk_method']
h[chunk_headers["container_id"]] = sysmeta['container_id']
h[chunk_headers["chunk_pos"]] = chunk["pos"]
h[chunk_headers["chunk_id"]] = chunk_path
h[chunk_headers["content_policy"]] = sysmeta['policy']
h[chunk_headers["content_version"]] = sysmeta['version']
# in the trailer
# metachunk_size & metachunk_hash
h["Trailer"] = (chunk_headers["metachunk_size"],
chunk_headers["metachunk_hash"])
with ConnectionTimeout(io.CONNECTION_TIMEOUT):
conn = io.http_connect(parsed.netloc, 'PUT', parsed.path, h)
conn.chunk = chunk
return cls(chunk, conn)
def _connect_put(chunk):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
try:
chunk_path = parsed.path.split('/')[-1]
h = {}
h["transfer-encoding"] = "chunked"
# FIXME: remove key incoherencies
# TODO: automatize key conversions
h[chunk_headers["content_id"]] = self.sysmeta['id']
h[chunk_headers["content_version"]] = self.sysmeta['version']
h[chunk_headers["content_path"]] = \
utils.quote(self.sysmeta['content_path'])
h[chunk_headers["content_chunkmethod"]] = \
self.sysmeta['chunk_method']
h[chunk_headers["content_policy"]] = self.sysmeta['policy']
h[chunk_headers["container_id"]] = self.sysmeta['container_id']
h[chunk_headers["chunk_pos"]] = chunk["pos"]
h[chunk_headers["chunk_id"]] = chunk_path
# Used during reconstruction of EC chunks
if self.sysmeta['chunk_method'].startswith('ec'):
h[chunk_headers["metachunk_size"]] = \
self.sysmeta["metachunk_size"]
h[chunk_headers["metachunk_hash"]] = \
self.sysmeta["metachunk_hash"]
with ConnectionTimeout(io.CONNECTION_TIMEOUT):
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, h)
conn.chunk = chunk
return conn, chunk
except (Exception, Timeout) as e:
msg = str(e)
logger.exception("Failed to connect to %s (%s)", chunk, msg)
chunk['error'] = msg
return None, chunk