class ClickhouseTestCase(unittest.TestCase):
def setUp(self):
self.client = Client('localhost')
self.client.execute('DROP TABLE IF EXISTS test')
self.client.execute(
'CREATE TABLE test (id String, x Int32, dict String) ENGINE = ReplacingMergeTree() ORDER BY id'
)
self.new_client = CustomClickhouse()
def _add_records(self):
documents = [{
'x': 1,
"id": "1"
}, {
'x': 2,
"id": "2"
}, {
'x': 3,
"id": "3"
}, {
'x': 100,
"id": "100"
}]
formatted_documents = [{
"_id": doc["id"],
"_source": {
'x': doc["x"]
}
} for doc in documents]
self.client.execute('INSERT INTO test (id, x) VALUES', documents)
return formatted_documents
def test_search(self):
formatted_documents = self._add_records()
result = self.new_client.search(index="test", fields=["x"])
self.assertCountEqual(formatted_documents, result)
def test_search_with_query(self):
formatted_documents = self._add_records()
formatted_documents = [
doc for doc in formatted_documents if doc["_source"]['x'] < 3
]
result = self.new_client.search(index="test",
query="WHERE x < 3",
fields=["x"])
self.assertSequenceEqual(formatted_documents, result)
def test_count(self):
formatted_documents = self._add_records()
formatted_documents = [
doc for doc in formatted_documents if doc["_source"]['x'] < 3
]
result = self.new_client.count(index="test", query="WHERE x < 3")
assert result == len(formatted_documents)
def test_iterate(self):
test_per = 2
formatted_documents = self._add_records()
formatted_documents = [
doc for doc in formatted_documents if doc["_source"]['x'] < 4
]
result = self.new_client.iterate(index="test",
fields=["x"],
query="WHERE x < 4",
per=test_per)
self.assertSequenceEqual(formatted_documents[0:test_per], next(result))
self.assertSequenceEqual(formatted_documents[test_per:2 * test_per],
next(result))
def test_multiple_iterate(self):
test_per = 2
self._add_records()
first_result = self.new_client.iterate(index="test",
fields=["x"],
per=test_per)
next(first_result)
second_result = self.new_client.iterate(index="test",
fields=["x"],
per=test_per)
next(second_result)
def test_iterate_without_id(self):
self._add_records()
result = self.new_client.iterate(index="test",
fields=["distinct(ceiling(x / 10))"],
return_id=False)
result = next(result)
assert len(result) == 2
def test_iterate_with_and_without_final(self):
self._add_records()
self._add_records()
result_with_final = self.new_client.iterate(index="test", fields=[])
result_without_final = self.new_client.iterate(index="test",
fields=[],
final=False)
assert len(next(result_without_final)) > len(next(result_with_final))
def test_iterate_with_derived_fields(self):
self._add_records()
result = self.new_client.iterate(index="test", fields=["x - 1 AS y"])
result_record = next(result)[0]
assert "y" in result_record["_source"]
def test_bulk_index(self):
documents = [{"x": i} for i in range(10)]
self.new_client.bulk_index(index="test",
docs=[d.copy() for d in documents],
id_field="x")
result = self.client.execute('SELECT id FROM test')
self.assertCountEqual(result, [(str(doc["x"]), ) for doc in documents])
def test_bulk_index_check_schema(self):
self.new_client.bulk_index(index="test", docs=[{"y": 1, "id": 1}])
result = self.client.execute('SELECT id FROM test')
self.assertCountEqual(result, [('1', )])
def test_bulk_index_empty_fields(self):
documents = [{"id": 1, "x": 1}]
self.new_client.bulk_index(index="test", docs=[d for d in documents])
def test_bulk_index_dict_values(self):
documents = [{"x": i, "dict": {"test": i}} for i in range(10)]
self.new_client.bulk_index(index="test",
docs=[d.copy() for d in documents],
id_field="x")
result = self.client.execute('SELECT dict FROM test')
self.assertCountEqual(result, [(json.dumps(doc["dict"]), )
for doc in documents])
def test_bulk_index_split_records(self):
test_docs = [{"docs": True}]
test_chunks = ["records1", "records2"]
self.new_client._split_records = MagicMock(return_value=test_chunks)
self.new_client.client.execute = MagicMock()
self.new_client._set_id = MagicMock()
self.new_client._filter_schema = MagicMock()
self.new_client.bulk_index(index="test_index", docs=test_docs)
self.new_client._split_records.assert_called_with(test_docs)
calls = [call(ANY, records) for records in test_chunks]
self.new_client.client.execute.assert_has_calls(calls)
def test_send_sql_request(self):
formatted_documents = self._add_records()
result = self.new_client.send_sql_request("SELECT max(x) FROM test")
assert result == max(doc["_source"]["x"]
for doc in formatted_documents)
def test_split_records(self):
test_record = {"test": "123"}
test_record_size = sys.getsizeof(test_record)
test_records = [test_record] * 5
chunks = list(
self.new_client._split_records(test_records,
max_bytes=test_record_size * 2 + 1))
self.assertSequenceEqual(
chunks, [[test_record] * 2, [test_record] * 2, [test_record]])
def test_split_records_one_record(self):
test_record = {"test": "123"}
test_record_size = sys.getsizeof(test_record)
test_records = [test_record]
chunks = list(
self.new_client._split_records(test_records,
max_bytes=test_record_size))
self.assertSequenceEqual(chunks, [[test_record]])
def test_split_records_same_chunk(self):
test_record = {"test": "123"}
test_record_size = sys.getsizeof(test_record)
test_records = [test_record] * 6
chunks = list(
self.new_client._split_records(test_records,
max_bytes=test_record_size * 2))
self.assertSequenceEqual(
chunks, [[test_record] * 2, [test_record] * 2, [test_record] * 2])
class ClickhouseEvents:
def __init__(self, indices=INDICES, parity_hosts=PARITY_HOSTS):
self.client = CustomClickhouse()
self.indices = indices
self.web3 = Web3(
HTTPProvider(parity_hosts[0][-1], request_kwargs={'timeout': 100}))
def _iterate_block_ranges(self, range_size=EVENTS_RANGE_SIZE):
"""
Iterate over unprocessed block ranges with given size
Parameters
----------
range_size : list
Size of each block range
Returns
-------
generator
Generator that iterates through unprocessed block ranges
"""
range_query = "distinct(toInt32(floor(number / {}))) AS range".format(
range_size)
flags_query = "ANY LEFT JOIN (SELECT id, value FROM {} FINAL WHERE name = 'events_extracted') USING id WHERE value IS NULL".format(
self.indices["block_flag"])
for ranges_chunk in self.client.iterate(index=self.indices["block"],
fields=[range_query],
query=flags_query,
return_id=False):
for range in ranges_chunk:
range_bounds = (range["_source"]["range"] * range_size,
(range["_source"]["range"] + 1) * range_size)
yield range_bounds
def _get_events(self, block_range):
"""
Get events from parity for given block range
Parameters
----------
block_range : tuple
Start and end of block range
Returns
-------
list
Events inside given block range (not including end block)
"""
event_filter = self.web3.eth.filter({
"fromBlock": block_range[0],
"toBlock": block_range[1] - 1
})
events = event_filter.get_all_entries()
return events
def _save_events(self, events):
"""
Prepare and save each event to a database
Parameters
----------
events : list
Events extracted from parity
"""
events = [self._process_event(event) for event in events]
if events:
self.client.bulk_index(index=self.indices["event"], docs=events)
def _process_event(self, event):
"""
Prepare event - parse hexadecimal numbers, assign id, lowercase each string
Parameters
----------
event : dict
Event extracted from parity
Returns
-------
dict
Prepared event
"""
processed_event = event.copy()
processed_event["transactionLogIndex"] = int(
event["transactionLogIndex"], 0)
processed_event["id"] = "{}.{}".format(
event['transactionHash'].hex(),
processed_event["transactionLogIndex"])
processed_event["address"] = event["address"].lower()
processed_event["blockHash"] = event["blockHash"].hex()
processed_event["transactionHash"] = event["transactionHash"].hex()
processed_event["topics"] = [topic.hex() for topic in event["topics"]]
return processed_event
def _save_processed_blocks(self, block_range):
"""
Save events_extracted flag for processed blocks
Parameters
----------
block_range : tuple
Start and end of processed block range
"""
block_flags = [{
"id": block,
"name": "events_extracted",
"value": 1
} for block in range(*block_range)]
self.client.bulk_index(index=self.indices["block_flag"],
docs=block_flags)
def extract_events(self):
"""
Extract parity events to a database
This function is an entry point for extract-events operation
"""
for block_range in self._iterate_block_ranges():
events = self._get_events(block_range)
self._save_events(events)
self._save_processed_blocks(block_range)
class ClickhouseContractMethods:
"""
Check if contract is token, is it compliant with token standards and get variables from it such as name or symbol
Parameters
----------
indices: dict
Dictionary containing exisiting database indices
parity_hosts: list
List of tuples that includes 3 elements: start block, end_block and Parity URL
"""
_external_links = {}
_constants_types = [
('name', {
"string": lambda x: str(x).replace("\\x00", ""),
"bytes32": lambda x: str(x).replace("\\x00", "")[2:-1].strip()
}, ''),
('symbol', {
"string": lambda x: str(x).replace("\\x00", ""),
"bytes32": lambda x: str(x).replace("\\x00", "")[2:-1].strip()
}, ''), ('decimals', {
"uint8": None
}, 18), ('totalSupply', {
"uint256": None
}, 0), ('owner', {
"address": lambda x: x.lower()
}, None)
]
def __init__(self, indices=INDICES, parity_hosts=PARITY_HOSTS):
self.indices = indices
self.client = CustomClickhouse()
self.w3 = Web3(HTTPProvider(parity_hosts[0][2]))
self.standard_token_abi = standard_token_abi
self._set_external_links()
def _set_external_links(self):
"""
Sets website slug and cmc_id for this object
"""
with open('{}/tokens.json'.format(CURRENT_DIR)) as json_file:
tokens = json.load(json_file)
for token in tokens:
self._external_links[token["address"]] = {
"website_slug": token["website_slug"],
"cmc_id": token["cmc_id"],
}
def _iterate_unprocessed_contracts(self):
"""
Iterate over ERC20 contracts that were not processed yet
Returns
-------
generator
Generator that iterates over contracts in database
"""
return self.client.iterate(index=self.indices["contract"],
fields=["address"],
query="""
WHERE standard_erc20 = 1
AND id not in(
SELECT id
FROM {}
)
""".format(self.indices["contract_description"]))
def _round_supply(self, supply, decimals):
"""
Divide supply by 10 ** decimals, and round it
Parameters
----------
supply: int
Contract total supply
decimals: int
Contract decimals
Returns
-------
str
Contract total supply without decimals
"""
if decimals > 0:
supply = supply / math.pow(10, decimals)
supply = Decimal(supply)
supply = round(supply)
return min(supply, MAX_TOTAL_SUPPLY)
def _get_constant(self, address, constant, types, placeholder=None):
"""
Get value through contract function marked as constant
Tries every type from types dict and returns first value that are not empty
If it fails, returns placeholder
Parameters
----------
address: str
Contract address
constant: str
Name of constant
types: dict
Dict with all possible types and converter functions for target value
placeholder
Default value for target value
Returns
-------
Value returned by a contract and converted with the function
Placeholder, if there are no non-empty values
"""
contract_checksum_addr = self.w3.toChecksumAddress(address)
contract_abi = [{
"constant": True,
"inputs": [],
"name": constant,
"outputs": [{
"name": "",
"type": None
}],
"payable": False,
"type": "function"
}]
response = None
for constant_type, convert in types.items():
try:
contract_abi[0]["outputs"][0]["type"] = constant_type
contract_instance = self.w3.eth.contract(
address=contract_checksum_addr, abi=contract_abi)
response = getattr(contract_instance.functions,
constant)().call()
if convert:
response = convert(response)
if response:
return response
except Exception as e:
pass
if type(response) != int:
return placeholder
else:
return response
def _get_constants(self, address):
"""
Return contract ERC20 info
Parameters
----------
address: str
Contract address
Returns
-------
list
Name, symbol, decimals, total supply, owner address
"""
contract_constants = []
for constant, types, placeholder in self._constants_types:
response = self._get_constant(address, constant, types,
placeholder)
contract_constants.append(response)
contract_constants[3] = self._round_supply(contract_constants[3],
contract_constants[2])
return contract_constants
def _update_contract_descr(self, doc_id, body):
"""
Store contract description in database
Parameters
----------
doc_id: str
id of contract
body: dict
Dictionary with new values
"""
body["id"] = doc_id
self.client.bulk_index(self.indices['contract_description'],
docs=[body])
def _get_external_links(self, address):
"""
Add Cryptocompare and Coinmarketcap info as a field of this object
"""
external_links = self._external_links.get(address, {
"website_slug": None,
"cmc_id": None
})
return external_links.get("website_slug"), external_links.get("cmc_id")
def _classify_contract(self, contract):
"""
Extract contract ERC20 info and stores it into the database
Extracts ERC20 token description from parity and from token.json file
Parameters
----------
contract: dict
Dictionary with contract info
"""
name, symbol, decimals, total_supply, owner = self._get_constants(
contract['_source']['address'])
website_slug, cmc_id = self._get_external_links(
contract["_source"]["address"])
update_body = {
'token_name': name,
'token_symbol': symbol,
'decimals': decimals,
'total_supply': total_supply,
'token_owner': owner,
"website_slug": website_slug,
"cmc_id": cmc_id
}
self._update_contract_descr(contract['_id'], update_body)
def search_methods(self):
"""
Extract public values for ERC20 contracts
This function is an entry point for extract-tokens operation
"""
for contracts_chunk in self._iterate_unprocessed_contracts():
for contract in contracts_chunk:
self._classify_contract(contract)