def __init__(
self,
executor: ThreadPoolExecutor,
pool: HTTPConnectionPool,
user: str,
password: str,
statement: InsertStatement,
encoding: Optional[str],
buffer_size: int, # 0 means unbounded
options: Mapping[str, Any], # should be ``Mapping[str, str]``?
chunk_size: Optional[int] = None,
) -> None:
if chunk_size is None:
chunk_size = settings.CLICKHOUSE_HTTP_CHUNK_SIZE
self.__queue: Union[Queue[Union[bytes, None]],
SimpleQueue[Union[bytes,
None]]] = (Queue(buffer_size)
if buffer_size else
SimpleQueue())
body = self.__read_until_eof()
if chunk_size > 1:
body = (b"".join(chunk) for chunk in chunked(body, chunk_size))
elif not chunk_size > 0:
raise ValueError("chunk size must be greater than zero")
headers = {
"X-ClickHouse-User": user,
"Connection": "keep-alive",
"Accept-Encoding": "gzip,deflate",
}
if password != "":
headers["X-ClickHouse-Key"] = password
if encoding:
headers["Content-Encoding"] = encoding
self.__result = executor.submit(
pool.urlopen,
"POST",
"/?" + urlencode({
**options, "query": statement.build_statement()
}),
headers=headers,
body=body,
)
self.__rows = 0
self.__size = 0
self.__closed = False
def __init__(
self,
executor: ThreadPoolExecutor,
pool: HTTPConnectionPool,
database: str,
table_name: str,
user: str,
password: str,
options: Mapping[str, Any], # should be ``Mapping[str, str]``?
chunk_size: Optional[int] = None,
) -> None:
if chunk_size is None:
chunk_size = settings.CLICKHOUSE_HTTP_CHUNK_SIZE
self.__queue: SimpleQueue[Union[JSONRow, None]] = SimpleQueue()
body = self.__read_until_eof()
if chunk_size > 1:
body = (b"".join(chunk) for chunk in chunked(body, chunk_size))
elif not chunk_size > 0:
raise ValueError("chunk size must be greater than zero")
self.__result = executor.submit(
pool.urlopen,
"POST",
"/?"
+ urlencode(
{
**options,
"query": f"INSERT INTO {database}.{table_name} FORMAT JSONEachRow",
}
),
headers={
"X-ClickHouse-User": user,
"X-ClickHouse-Key": password,
"Connection": "keep-alive",
"Accept-Encoding": "gzip,deflate",
},
body=body,
)
self.__rows = 0
self.__size = 0
self.__closed = False
def test_chunked() -> None:
assert [*chunked([], 3)] == []
assert [*chunked(range(3), 3)] == [[0, 1, 2]]
assert [*chunked(range(10), 3)] == [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]