def urlopen(self, method, url, body=None, headers=None, retries=3, redirect=True, assert_same_host=True,
timeout=None, pool_timeout=None, release_conn=None, **response_kw):
"""
Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
with custom cross-host redirect logic and only sends the request-uri
portion of the ``url``.
The given ``url`` parameter must be absolute, such that an appropriate
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
if headers is None:
headers = {k.lower(): v for (k, v) in self.headers.items()}
headers.setdefault("cookie", "")
for key, val in self.headers.items():
headers.setdefault(key, val)
# Now the updated Cookie string will be stored into the HTTP request.
# The cookie header may contain duplicate entries (e.g. k=a; k=b;)
headers["cookie"] = self.headers["cookie"] + headers["cookie"]
# This will be resolved by putting the header in the SimpleCookie
self.cookie_session.feed(self.headers)
self.cookie_session.feed(headers)
headers["cookie"] = self.cookie_session.extract()
response = HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, False, assert_same_host, timeout, pool_timeout,
release_conn, **response_kw)
self.cookie_session.feed(self.headers)
self.cookie_session.feed(response.headers)
self.cookie_session.feed(headers)
headers["cookie"] = self.cookie_session.extract()
redirect_location = redirect and response.get_redirect_location()
if not redirect_location:
return response
if response.status == 303:
method = "GET"
return self.urlopen(method, redirect_location, body, headers, retries - 1, redirect, assert_same_host, timeout, pool_timeout,
release_conn, **response_kw)
def _test(exception):
pool = HTTPConnectionPool(host='localhost', maxsize=1, block=True)
# Verify that the request succeeds after two attempts, and that the
# connection is left on the response object, instead of being
# released back into the pool.
pool._make_request = _raise_once_make_request_function(exception)
response = pool.urlopen('GET', '/', retries=1,
release_conn=False, preload_content=False,
chunked=True)
self.assertEqual(pool.pool.qsize(), 0)
self.assertEqual(pool.num_connections, 2)
self.assertTrue(response.connection is not None)
response.release_conn()
self.assertEqual(pool.pool.qsize(), 1)
self.assertTrue(response.connection is None)
class ConnectionPoolManager(LoggerMixin):
def __init__(self,
host,
port,
certfile=None,
keyfile=None,
cacertfile=None,
force_ssl=False,
*args,
**kw):
super(ConnectionPoolManager, self).__init__(*args, **kw)
self.logger.debug("Creating ConnectionPoolManager for %s:%s", host,
port)
if certfile or keyfile or force_ssl:
#https://docs.python.org/2/library/ssl.html#ssl.SSLContext
from ssl import SSLContext, PROTOCOL_SSLv23
ssl_context = SSLContext(PROTOCOL_SSLv23)
ssl_context.load_cert_chain(certfile=certfile, keyfile=keyfile)
ssl_context.load_verify_locations(cafile=cacertfile)
#https://docs.python.org/2/library/httplib.html
self.__pool = HTTPSConnectionPool(host,
port,
maxsize=16,
context=ssl_context)
else:
self.__pool = HTTPConnectionPool(host, port, maxsize=16)
def request(self, method, path, body, headers, timeout):
return Urllib3ResponseWrapper(
self.__pool.urlopen(method,
path,
body,
headers,
timeout=timeout,
pool_timeout=30,
preload_content=False,
assert_same_host=False))
class HTTPBatchWriter(BatchWriter):
def __init__(
self,
schema: TableSchema,
host,
port,
encoder: Callable[[WriterTableRow], bytes],
options=None,
table_name=None,
chunk_size: int = 1,
):
"""
Builds a writer to send a batch to Clickhouse.
:param schema: The dataset schema to take the table name from
:param host: Clickhosue host
:param port: Clickhosue port
:param encoder: A function that will be applied to each row to turn it into bytes
:param options: options passed to Clickhouse
:param table_name: Overrides the table coming from the schema (generally used for uplaoding
on temporary tables)
:param chunk_size: The chunk size (in rows).
We send data to the server with Transfer-Encoding: chunked. If 0 we send the entire
content in one chunk.
"""
self.__pool = HTTPConnectionPool(host, port)
self.__options = options if options is not None else {}
self.__table_name = table_name or schema.get_table_name()
self.__chunk_size = chunk_size
self.__encoder = encoder
def _prepare_chunks(self,
rows: Iterable[WriterTableRow]) -> Iterable[bytes]:
chunk = []
for row in rows:
chunk.append(self.__encoder(row))
if self.__chunk_size and len(chunk) == self.__chunk_size:
yield b"".join(chunk)
chunk = []
if chunk:
yield b"".join(chunk)
def write(self, rows: Iterable[WriterTableRow]):
response = self.__pool.urlopen(
"POST",
"/?" + urlencode({
**self.__options,
"query":
f"INSERT INTO {self.__table_name} FORMAT JSONEachRow",
}),
headers={
"Connection": "keep-alive",
"Accept-Encoding": "gzip,deflate"
},
body=self._prepare_chunks(rows),
chunked=True,
)
if response.status != 200:
# XXX: This should be switched to just parse the JSON body after
# https://github.com/yandex/ClickHouse/issues/6272 is available.
content = response.data.decode("utf8")
details = CLICKHOUSE_ERROR_RE.match(content)
if details is not None:
code, type, message = details.groups()
raise ClickhouseError(int(code), message)
else:
raise HTTPError(
f"Received unexpected {response.status} response: {content}"
)
class DataBoxConnector(object):
def __init__(self, host, port, user="", passwd=""):
self.c = HTTPConnectionPool(host, port)
url = url_prefix + "/metadata/authenticate"
headers = {
"content-type": "application/x-www-form-urlencoded",
}
params = {"user": user, "passwd": passwd}
resp = self.c.urlopen("POST",
url,
body=urlencode(params),
headers=headers)
jdata = json.loads(resp.data.decode("utf-8"))
# jdata = json.loads(resp.data)
self.auth_code = jdata["auth_code"]
def _load_resp(self, resp):
ctype = resp.headers["Content-Type"]
# print( resp.headers )
code = resp.headers.get("Error-Code", None)
reason = resp.headers.get("Error-Message", None)
if ctype.startswith("application/json") == True:
jdata = json.loads(resp.data.decode("utf-8"))
elif ctype.startswith("application/pickle-bytes") == True:
jdata = pickle.loads(resp.data)
else:
jdata = None
return jdata, resp.status, code, reason
def _info_query(self, product, ctype, params):
url = url_prefix + "/info_query/{product}/{ctype}".format(
product=product, ctype=ctype)
# params["crs"] = "" if crs is None else crs
params["format"] = "bytes"
headers = {
"content-type": "application/x-www-form-urlencoded",
"auth_code": self.auth_code
}
# print(params)
resp = self.c.urlopen("POST",
url,
body=urlencode(params),
headers=headers)
return self._load_resp(resp)
def query_by_point(self, product, ctype, x, y, crs=None, timeslice=None):
'''
返回 x,y 坐标点所在 cube 信息，返回：bands, crs, bbox, res, size，xy, nctimes
ctype：数据产品名称
x, y：坐标
crs：坐标对应投影信息，如为：None，默认：EPSG:4326
'''
return self._info_query(
product, ctype, crs, {
"x": x,
"y": y,
"start_time": "19600101" if start_time is None else start_time,
"end_time": "20501231" if end_time is None else end_time
})
# info_by_bbox(self, minx, miny, maxx, maxy, start_time, end_time, fmt="json")
def query_by_bbox(self,
product,
ctype,
bbox,
start_time=None,
end_time=None):
return self._info_query(
product,
ctype,
{
"bbox": ",".join(map(lambda a: str(a), bbox)),
# "times": "" if timeslice is None else timeslice
"start_time": "19600101" if start_time is None else start_time,
"end_time": "20501231" if end_time is None else end_time
})
def query_by_geom(self, product, ctype, geom, start_time, end_time):
return self._info_with(
product, ctype, {
"geom": geom,
"start_time": "19600101" if start_time is None else start_time,
"end_time": "20501231" if end_time is None else end_time
})
def read_by_point(self, product, ctype, x, y, tif_file):
url = url_prefix + "/read_point/{product}/{ctype}".format(
product=product, ctype=ctype)
params = {"x": x, "y": y}
params["tif_file"] = tif_file
headers = {
"content-type": "application/x-www-form-urlencoded",
"auth_code": self.auth_code
}
# print(params)
resp = self.c.urlopen("POST",
url,
body=urlencode(params),
headers=headers)
return self._load_resp(resp)
def read_by_geom(self, product, ctype, geom, tif_file):
url = url_prefix + "/read_geom/{product}/{ctype}".format(
product=product, ctype=ctype)
params = {
"geometry": geom,
}
# params[ "geom_info" ] = json.dumps(geom_info, ensure_ascii=False)
params["tif_file"] = tif_file
headers = {
"content-type": "application/x-www-form-urlencoded",
"auth_code": self.auth_code
}
# print(params)
resp = self.c.urlopen("POST",
url,
body=urlencode(params),
headers=headers)
return self._load_resp(resp)
class HtClient(object):
def __init__(self, hostname, port=8060):
self.c = HTTPConnectionPool(hostname, port)
def erase_data(self, namespace, key):
try:
url = "/ht/del?ns=%s&key=%s" % (
encode_key(namespace),
encode_key(key),
)
resp = self.c.urlopen("DELETE", url)
jdata = json.loads(resp.data.decode("utf-8"), encoding="utf-8")
return jdata.get("code", None) == 1
except Exception as e:
print(e)
return False
def erase_nss(
self,
namespace,
):
try:
url = "/ht/del?ns=%s" % (encode_key(namespace), )
resp = self.c.urlopen("DELETE", url)
jdata = json.loads(resp.data.decode("utf-8"), encoding="utf-8")
return jdata.get("code", None) == 1
except Exception as e:
print(e)
return False
def put_data(self,
namespace,
key,
content,
ctype=None,
ttl=DEFAULT_TTL,
overwrite="yes",
**kwargs):
try:
ctype = "" if ctype is None else ctype
url = "/ht/put?ns=%s&key=%s&ctype=%s&ttl=%s&overwrite=%s" % (
encode_key(namespace), encode_key(key), ctype, ttl, overwrite)
cdata = kwargs
headers = {"cdata": json.dumps(cdata, ensure_ascii=False)}
resp = self.c.urlopen("PUT",
str(url),
body=content,
headers=headers)
jdata = json.loads(resp.data.decode("utf-8"), encoding="utf-8")
return jdata["code"] == 1
except Exception as e:
print_error(e)
return False
put = put_data
def has_data(self, namespace, key):
try:
url = "/ht/get?ns=%s&key=%s&action=has" % (encode_key(namespace),
encode_key(key))
resp = self.c.urlopen("GET", url)
jdata = json.loads(resp.data.decode("utf-8"), encoding="utf-8")
return jdata.get("code", None) == 1
except Exception as e:
print(e)
return False
has = has_data
def get_stat(self, node=None):
try:
url = "/ht/stat?node=%s" % (encode_key(node), )
resp = self.c.urlopen("GET", url)
jdata = json.loads(resp.data.decode("utf-8"), encoding='utf-8')
return jdata
except Exception as e:
print(e)
return None
stat = get_stat
def get_nss(self, node=None):
try:
url = "/ht/stat/nss?node=%s" % (encode_key(node), )
resp = self.c.urlopen("GET", url)
jdata = json.loads(resp.data.decode("utf-8"), encoding='utf-8')
return jdata
except Exception as e:
print(e)
return None
nss = get_nss
def get_nodes(self):
try:
url = "/ht/stat/nodes"
resp = self.c.urlopen("GET", url)
jdata = json.loads(resp.data.decode("utf-8"), encoding='utf-8')
return jdata
except Exception as e:
print(e)
return None
nodes = get_nodes
def get_keys(self, namespace, node=None):
return KeysIter(self.c, encode_key(namespace), node)
keys = get_keys
def get_data(self, namespace, key):
try:
url = "/ht/get?ns=%s&key=%s" % (encode_key(namespace),
encode_key(key))
resp = self.c.urlopen("GET", url)
try:
cdata = json.loads(resp.headers.get("cdata", "{}"),
encoding="utf-8")
except:
cdata = {}
options = {"cdata": cdata, "ctype": resp.headers.get("ctype", "")}
return resp.data, cdata, options, resp.status
except Exception as e:
print(e)
return None, {}, {}, 500
get = get_data