def _get_request(self, chunk): """ Connect to a chunk, fetch headers but don't read data. Save the response object in `self.sources` list. """ try: with green.ConnectionTimeout(self.connection_timeout): raw_url = chunk["url"] parsed = urlparse(raw_url) conn = http_connect(parsed.netloc, 'GET', parsed.path, self.request_headers) with green.OioTimeout(self.read_timeout): source = conn.getresponse() source.conn = conn except (Exception, Timeout) as error: logger.exception('Connection failed to %s', chunk) self._resp_by_chunk[chunk["url"]] = (0, str(error)) return False if source.status in (200, 206): self.status = source.status self._headers = source.getheaders() self.sources.append((source, chunk)) return True else: logger.warn("Invalid response from %s: %d %s", chunk, source.status, source.reason) self._resp_by_chunk[chunk["url"]] = (source.status, str(source.reason)) return False
def _http_request(self, chunkurl, method, body, headers, trailers=None): parsed = urlparse(chunkurl) if method == 'PUT': headers['transfer-encoding'] = 'chunked' if trailers: headers['Trailer'] = list() for k, v in trailers.iteritems(): headers['Trailer'].append(k) conn = http_connect(parsed.netloc, method, parsed.path, headers) if method == 'PUT': if body: conn.send('%x\r\n%s\r\n' % (len(body), body)) conn.send('0\r\n') if trailers: for k, v in trailers.iteritems(): conn.send('%s: %s\r\n' % (k, v)) conn.send('\r\n') if method == 'PUT': del headers['transfer-encoding'] if trailers: del headers['Trailer'] resp = conn.getresponse() body = resp.read() conn.close() return resp, body
def delete_chunk(chunk): resp = None p = urlparse(chunk['id']) try: with Timeout(CHUNK_TIMEOUT): conn = http_connect(p.netloc, 'DELETE', p.path) resp = conn.getresponse() resp.chunk = chunk except (Exception, Timeout) as e: self.logger.warn('error while deleting chunk %s "%s"', chunk['id'], str(e.message)) return resp
def delete_chunk(chunk): resp = None p = urlparse(chunk['id']) try: with Timeout(CHUNK_TIMEOUT): conn = http_connect(p.netloc, 'DELETE', p.path) resp = conn.getresponse() resp.chunk = chunk except (Exception, Timeout) as e: self.logger.warn( 'error while deleting chunk %s "%s"', chunk['id'], str(e.message)) return resp
def delete_chunk(self, chunk, cid, reqid): resp = None parsed = urlparse(chunk['id']) headers = {'X-oio-req-id': reqid, 'X-oio-chunk-meta-container-id': cid} try: with Timeout(CHUNK_TIMEOUT): conn = http_connect(parsed.netloc, 'DELETE', parsed.path, headers=headers) resp = conn.getresponse() resp.chunk = chunk except (Exception, Timeout) as exc: self.logger.warn('error while deleting chunk %s "%s"', chunk['id'], str(exc.message)) return resp
def delete_chunk(self, chunk, cid, reqid): resp = None parsed = urlparse(chunk['id']) headers = {'X-oio-req-id': reqid, 'X-oio-chunk-meta-container-id': cid} try: with Timeout(CHUNK_TIMEOUT): conn = http_connect(parsed.netloc, 'DELETE', parsed.path, headers=headers) resp = conn.getresponse() resp.chunk = chunk except (Exception, Timeout) as exc: self.logger.warn( 'error while deleting chunk %s "%s"', chunk['id'], str(exc.message)) return resp
def get_stats(self): result = {} resp = None try: conn = http_connect(self.host, self.port, 'GET', self.path) resp = conn.getresponse() if resp.status == 200: result = self._parse_func(resp.read()) else: raise Exception("status code != 200: %s" % resp.status) except Exception as e: self.logger.debug("get_stats error: %s", e) finally: if resp: try: resp.force_close() except Exception: pass return result
def get_stats(self): result = {} resp = None try: conn = http_connect(self.netloc, 'GET', self.path) resp = conn.getresponse() if resp.status == 200: result = self._parse_func(resp.read()) else: raise Exception("status code != 200: %s" % resp.status) except Exception as e: self.logger.debug("get_stats error: %s", e) finally: if resp: try: resp.force_close() except Exception: pass return result
def _get_request(self, chunk): # connect to chunk try: with ConnectionTimeout(self.connection_timeout): raw_url = chunk["url"] parsed = urlparse(raw_url) conn = http_connect(parsed.netloc, "GET", parsed.path, self.request_headers) with Timeout(self.response_timeout): source = conn.getresponse() source.conn = conn except (Exception, Timeout): logger.exception("Connection failed to %s", chunk) return False if source.status in (200, 206): self.status = source.status self._headers = source.getheaders() self.sources.append((source, chunk)) return True else: logger.warn("Invalid GET response from %s", chunk) return False
def check(self): success = False resp = None try: conn = http_connect(self.netloc, 'GET', self.path) resp = conn.getresponse() if resp.status == 200: success = True else: raise Exception("status code != 200: %s" % resp.status) except Exception as e: self.logger.warn('ERROR performing http check: %s', e) finally: if resp: try: resp.force_close() except Exception: pass if not success: self.logger.warn('%s check failed', self.name) return success
def check(self): success = False resp = None try: conn = http_connect(self.host, self.port, 'GET', self.path) resp = conn.getresponse() if resp.status == 200: success = True else: raise Exception("status code != 200: %s" % resp.status) except Exception as e: self.logger.warn('ERROR performing http check: %s', e) finally: if resp: try: resp.force_close() except Exception: pass if not success: self.logger.warn('%s check failed', self.name) return success
def _get_request(self, chunk): # connect to chunk try: with ConnectionTimeout(self.connection_timeout): raw_url = chunk["url"] parsed = urlparse(raw_url) conn = http_connect(parsed.netloc, 'GET', parsed.path, self.request_headers) with Timeout(self.response_timeout): source = conn.getresponse() source.conn = conn except (Exception, Timeout): logger.exception('Connection failed to %s', chunk) return False if source.status in (200, 206): self.status = source.status self._headers = source.getheaders() self.sources.append((source, chunk)) return True else: logger.warn("Invalid GET response from %s", chunk) return False