def connect(cls, chunk, sysmeta, reqid=None, connection_timeout=None, write_timeout=None, **kwargs): raw_url = chunk.get("real_url", chunk["url"]) parsed = urlparse(raw_url) chunk_path = parsed.path.split('/')[-1] hdrs = headers_from_object_metadata(sysmeta) if reqid: hdrs['X-oio-req-id'] = reqid hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"] hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path # in the trailer # metachunk_size & metachunk_hash trailers = (CHUNK_HEADERS["metachunk_size"], CHUNK_HEADERS["metachunk_hash"]) if kwargs.get('chunk_checksum_algo'): trailers = trailers + (CHUNK_HEADERS["chunk_hash"], ) hdrs["Trailer"] = ', '.join(trailers) with green.ConnectionTimeout( connection_timeout or io.CONNECTION_TIMEOUT): conn = io.http_connect( parsed.netloc, 'PUT', parsed.path, hdrs) conn.chunk = chunk return cls(chunk, conn, write_timeout=write_timeout, **kwargs)
def connect(cls, chunk, sysmeta, reqid=None, connection_timeout=None, write_timeout=None, **_kwargs): raw_url = chunk["url"] parsed = urlparse(raw_url) chunk_path = parsed.path.split('/')[-1] hdrs = headers_from_object_metadata(sysmeta) if reqid: hdrs['X-oio-req-id'] = reqid hdrs[chunk_headers["chunk_pos"]] = chunk["pos"] hdrs[chunk_headers["chunk_id"]] = chunk_path # in the trailer # metachunk_size & metachunk_hash hdrs["Trailer"] = ', '.join( (chunk_headers["metachunk_size"], chunk_headers["metachunk_hash"], chunk_headers["chunk_hash"])) with green.ConnectionTimeout(connection_timeout or io.CONNECTION_TIMEOUT): conn = io.http_connect(parsed.netloc, 'PUT', parsed.path, hdrs) conn.chunk = chunk return cls(chunk, conn, write_timeout=write_timeout)
def _connect_put(self, chunk): """ Create a connection in order to PUT `chunk`. :returns: a tuple with the connection object and `chunk` """ raw_url = chunk["url"] parsed = urlparse(raw_url) try: chunk_path = parsed.path.split('/')[-1] hdrs = headers_from_object_metadata(self.sysmeta) hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"] hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path hdrs.update(self.headers) with green.ConnectionTimeout(self.connection_timeout): conn = io.http_connect( parsed.netloc, 'PUT', parsed.path, hdrs) conn.chunk = chunk return conn, chunk except (Exception, Timeout) as err: msg = str(err) logger.exception("Failed to connect to %s (%s)", chunk, msg) chunk['error'] = msg return None, chunk
def connect(cls, chunk, sysmeta, reqid=None, connection_timeout=None, write_timeout=None, **kwargs): raw_url = chunk.get("real_url", chunk["url"]) parsed = urlparse(raw_url) chunk_path = parsed.path.split('/')[-1] hdrs = headers_from_object_metadata(sysmeta) if reqid: hdrs[REQID_HEADER] = reqid hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"] hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path # in the trailer # metachunk_size & metachunk_hash trailers = (CHUNK_HEADERS["metachunk_size"], CHUNK_HEADERS["metachunk_hash"]) if kwargs.get('chunk_checksum_algo'): trailers = trailers + (CHUNK_HEADERS["chunk_hash"], ) hdrs["Trailer"] = ', '.join(trailers) with ConnectionTimeout( connection_timeout or io.CONNECTION_TIMEOUT): perfdata = kwargs.get('perfdata', None) if perfdata is not None: connect_start = monotonic_time() conn = io.http_connect( parsed.netloc, 'PUT', parsed.path, hdrs, scheme=parsed.scheme) conn.set_cork(True) if perfdata is not None: connect_end = monotonic_time() perfdata_rawx = perfdata.setdefault('rawx', dict()) perfdata_rawx['connect.' + chunk['url']] = \ connect_end - connect_start conn.chunk = chunk return cls(chunk, conn, write_timeout=write_timeout, reqid=reqid, **kwargs)
def _chunk_attr(self, chunk_id, data, path=None): if path is not None: self.content_path = path headers = headers_from_object_metadata({ 'id': self.content_id, 'version': self.content_version, 'content_path': self.content_path, 'chunk_method': 'ec/algo=liberasurecode_rs_vand,k=6,m=3', 'policy': 'TESTPOLICY', 'container_id': self.cid, 'chunk_hash': md5(data).hexdigest(), 'full_path': self.fullpath, 'oio_version': OIO_VERSION }) headers[CHUNK_HEADERS['chunk_pos']] = 0 headers[CHUNK_HEADERS['chunk_id']] = chunk_id headers[CHUNK_HEADERS['chunk_size']] = str(len(data)) return headers
def _connect_put(self, chunk): """ Create a connection in order to PUT `chunk`. :returns: a tuple with the connection object and `chunk` """ raw_url = chunk.get("real_url", chunk["url"]) parsed = urlparse(raw_url) try: chunk_path = parsed.path.split('/')[-1] hdrs = headers_from_object_metadata(self.sysmeta) hdrs[CHUNK_HEADERS["chunk_pos"]] = chunk["pos"] hdrs[CHUNK_HEADERS["chunk_id"]] = chunk_path hdrs.update(self.headers) hdrs = encode(hdrs) with green.ConnectionTimeout(self.connection_timeout): if self.perfdata is not None: connect_start = monotonic_time() conn = io.http_connect(parsed.netloc, 'PUT', parsed.path, hdrs, scheme=parsed.scheme) conn.set_cork(True) if self.perfdata is not None: connect_end = monotonic_time() perfdata_rawx = self.perfdata.setdefault('rawx', dict()) perfdata_rawx['connect.' + chunk['url']] = \ connect_end - connect_start conn.chunk = chunk return conn, chunk except (SocketError, Timeout) as err: msg = str(err) self.logger.warn("Failed to connect to %s (reqid=%s): %s", chunk, self.reqid, err) except Exception as err: msg = str(err) self.logger.exception("Failed to connect to %s (reqid=%s)", chunk, self.reqid) chunk['error'] = msg return None, chunk