def _get_request(self, chunk):
"""
Connect to a chunk, fetch headers but don't read data.
Save the response object in `self.sources` list.
"""
try:
with green.ConnectionTimeout(self.connection_timeout):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
conn = http_connect(parsed.netloc, 'GET', parsed.path,
self.request_headers)
with green.OioTimeout(self.read_timeout):
source = conn.getresponse()
source.conn = conn
except (Exception, Timeout) as error:
logger.exception('Connection failed to %s', chunk)
self._resp_by_chunk[chunk["url"]] = (0, str(error))
return False
if source.status in (200, 206):
self.status = source.status
self._headers = source.getheaders()
self.sources.append((source, chunk))
return True
else:
logger.warn("Invalid response from %s: %d %s", chunk,
source.status, source.reason)
self._resp_by_chunk[chunk["url"]] = (source.status,
str(source.reason))
return False
def _http_request(self, chunkurl, method, body, headers, trailers=None):
parsed = urlparse(chunkurl)
if method == 'PUT':
headers['transfer-encoding'] = 'chunked'
if trailers:
headers['Trailer'] = list()
for k, v in trailers.iteritems():
headers['Trailer'].append(k)
conn = http_connect(parsed.netloc, method, parsed.path,
headers)
if method == 'PUT':
if body:
conn.send('%x\r\n%s\r\n' % (len(body), body))
conn.send('0\r\n')
if trailers:
for k, v in trailers.iteritems():
conn.send('%s: %s\r\n' % (k, v))
conn.send('\r\n')
if method == 'PUT':
del headers['transfer-encoding']
if trailers:
del headers['Trailer']
resp = conn.getresponse()
body = resp.read()
conn.close()
return resp, body
def _http_request(self, chunkurl, method, body, headers, trailers=None):
parsed = urlparse(chunkurl)
if method == 'PUT':
headers['transfer-encoding'] = 'chunked'
if trailers:
headers['Trailer'] = list()
for k, v in trailers.iteritems():
headers['Trailer'].append(k)
conn = http_connect(parsed.netloc, method, parsed.path, headers)
if method == 'PUT':
if body:
conn.send('%x\r\n%s\r\n' % (len(body), body))
conn.send('0\r\n')
if trailers:
for k, v in trailers.iteritems():
conn.send('%s: %s\r\n' % (k, v))
conn.send('\r\n')
if method == 'PUT':
del headers['transfer-encoding']
if trailers:
del headers['Trailer']
resp = conn.getresponse()
body = resp.read()
conn.close()
return resp, body
def delete_chunk(chunk):
resp = None
p = urlparse(chunk['id'])
try:
with Timeout(CHUNK_TIMEOUT):
conn = http_connect(p.netloc, 'DELETE', p.path)
resp = conn.getresponse()
resp.chunk = chunk
except (Exception, Timeout) as e:
self.logger.warn('error while deleting chunk %s "%s"',
chunk['id'], str(e.message))
return resp
def delete_chunk(chunk):
resp = None
p = urlparse(chunk['id'])
try:
with Timeout(CHUNK_TIMEOUT):
conn = http_connect(p.netloc, 'DELETE', p.path)
resp = conn.getresponse()
resp.chunk = chunk
except (Exception, Timeout) as e:
self.logger.warn(
'error while deleting chunk %s "%s"',
chunk['id'], str(e.message))
return resp
def delete_chunk(self, chunk, cid, reqid):
resp = None
parsed = urlparse(chunk['id'])
headers = {'X-oio-req-id': reqid, 'X-oio-chunk-meta-container-id': cid}
try:
with Timeout(CHUNK_TIMEOUT):
conn = http_connect(parsed.netloc,
'DELETE',
parsed.path,
headers=headers)
resp = conn.getresponse()
resp.chunk = chunk
except (Exception, Timeout) as exc:
self.logger.warn('error while deleting chunk %s "%s"', chunk['id'],
str(exc.message))
return resp
def delete_chunk(self, chunk, cid, reqid):
resp = None
parsed = urlparse(chunk['id'])
headers = {'X-oio-req-id': reqid,
'X-oio-chunk-meta-container-id': cid}
try:
with Timeout(CHUNK_TIMEOUT):
conn = http_connect(parsed.netloc, 'DELETE', parsed.path,
headers=headers)
resp = conn.getresponse()
resp.chunk = chunk
except (Exception, Timeout) as exc:
self.logger.warn(
'error while deleting chunk %s "%s"',
chunk['id'], str(exc.message))
return resp
def get_stats(self):
result = {}
resp = None
try:
conn = http_connect(self.host, self.port, 'GET', self.path)
resp = conn.getresponse()
if resp.status == 200:
result = self._parse_func(resp.read())
else:
raise Exception("status code != 200: %s" % resp.status)
except Exception as e:
self.logger.debug("get_stats error: %s", e)
finally:
if resp:
try:
resp.force_close()
except Exception:
pass
return result
def get_stats(self):
result = {}
resp = None
try:
conn = http_connect(self.netloc, 'GET', self.path)
resp = conn.getresponse()
if resp.status == 200:
result = self._parse_func(resp.read())
else:
raise Exception("status code != 200: %s" % resp.status)
except Exception as e:
self.logger.debug("get_stats error: %s", e)
finally:
if resp:
try:
resp.force_close()
except Exception:
pass
return result
def _get_request(self, chunk):
# connect to chunk
try:
with ConnectionTimeout(self.connection_timeout):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
conn = http_connect(parsed.netloc, "GET", parsed.path, self.request_headers)
with Timeout(self.response_timeout):
source = conn.getresponse()
source.conn = conn
except (Exception, Timeout):
logger.exception("Connection failed to %s", chunk)
return False
if source.status in (200, 206):
self.status = source.status
self._headers = source.getheaders()
self.sources.append((source, chunk))
return True
else:
logger.warn("Invalid GET response from %s", chunk)
return False
def check(self):
success = False
resp = None
try:
conn = http_connect(self.netloc, 'GET', self.path)
resp = conn.getresponse()
if resp.status == 200:
success = True
else:
raise Exception("status code != 200: %s" % resp.status)
except Exception as e:
self.logger.warn('ERROR performing http check: %s', e)
finally:
if resp:
try:
resp.force_close()
except Exception:
pass
if not success:
self.logger.warn('%s check failed', self.name)
return success
def check(self):
success = False
resp = None
try:
conn = http_connect(self.host, self.port, 'GET', self.path)
resp = conn.getresponse()
if resp.status == 200:
success = True
else:
raise Exception("status code != 200: %s" % resp.status)
except Exception as e:
self.logger.warn('ERROR performing http check: %s', e)
finally:
if resp:
try:
resp.force_close()
except Exception:
pass
if not success:
self.logger.warn('%s check failed', self.name)
return success
def _get_request(self, chunk):
# connect to chunk
try:
with ConnectionTimeout(self.connection_timeout):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
conn = http_connect(parsed.netloc, 'GET', parsed.path,
self.request_headers)
with Timeout(self.response_timeout):
source = conn.getresponse()
source.conn = conn
except (Exception, Timeout):
logger.exception('Connection failed to %s', chunk)
return False
if source.status in (200, 206):
self.status = source.status
self._headers = source.getheaders()
self.sources.append((source, chunk))
return True
else:
logger.warn("Invalid GET response from %s", chunk)
return False