def get_logs(addresses: List[str], topics: List,
start_block: int) -> Dict[str, List]:
@retry(stop_max_attempt_number=3, wait_fixed=1)
def get_chunk(start):
resp = requests.post(GRAPHQL_ENDPOINT,
json={
'query':
GRAPHQL_LOGS_QUERY.format(
fromBlock=start,
toBlock=min(start + LOGS_BLOCKS_CHUNK - 1,
CURRENT_BLOCK),
addresses=json.dumps(addresses),
topics=json.dumps(topics))
})
return postprocess_graphql_response(resp.json()['data']['logs'])
_addresses = addresses
len_addresses = len(addresses)
len_part = len_addresses // 10
log_chunks = []
for i in range(0, len_addresses, len_part):
addresses = _addresses[i:i + len_part]
log_chunks += pool.map(
get_chunk, range(start_block, CURRENT_BLOCK + 1,
LOGS_BLOCKS_CHUNK))
logs = [log for chunk in log_chunks for log in chunk]
logs.sort(key=lambda l: (l['address'], l['blockNumber']))
return dict((k, list(g)) for k, g in groupby(logs, itemgetter('address')))
def populate_liquidity_history(
infos: List[ExchangeInfo]) -> List[ExchangeInfo]:
for info in infos:
history_len = len(info.history)
new_history = pool.map(
lambda block_number: web3_infura.eth.getBalance(
info.exchange_address, block_number) / ETH,
get_chart_range(HISTORY_BEGIN_BLOCK +
history_len * HISTORY_CHUNK_SIZE))
info.history += new_history
print('Loaded history of balances of {} exchanges'.format(len(infos)))
return infos
def get_logs(address: str, topics: Iterable[Iterable[str]],
start_block: int) -> Iterable:
@retry(stop_max_attempt_number=3, wait_fixed=1)
def get_chunk(start: int):
return web3.eth.getLogs({
'fromBlock':
start,
'toBlock':
min(start + LOGS_BLOCKS_CHUNK - 1, CURRENT_BLOCK),
'address':
address,
'topics':
topics
})
log_chunks = pool.map(get_chunk,
range(start_block, CURRENT_BLOCK, LOGS_BLOCKS_CHUNK))
return [log for chunk in log_chunks for log in chunk]
def get_logs(address: str, topics: List, start_block: int) -> List:
@retry(stop_max_attempt_number=3, wait_fixed=1)
def get_chunk(start):
resp = requests.post(GRAPHQL_ENDPOINT,
json={
'query':
GRAPHQL_LOGS_QUERY.format(
fromBlock=start,
toBlock=min(start + LOGS_BLOCKS_CHUNK,
CURRENT_BLOCK),
addresses=json.dumps([address]),
topics=json.dumps(topics))
})
return postprocess_graphql_response(resp.json()['data']['logs'])
log_chunks = pool.map(get_chunk,
range(start_block, CURRENT_BLOCK, LOGS_BLOCKS_CHUNK))
logging.info('Loaded logs for {} successfully'.format(address))
return [log for chunk in log_chunks for log in chunk]
def load_block_timestamps(blocks: Iterable[int]) -> Dict[int, int]:
return {
d['number']: d['timestamp']
for d in pool.map(web3.eth.getBlock, blocks)
}
def load_timestamps() -> List[int]:
return [
d['timestamp'] for d in pool.map(web3.eth.getBlock, get_chart_range())
]