def get_data_filtered(query_input, entity, mappings_file, endpoint, filters):
"""
Gets all the existing data from the subgraph applying the given filters
"""
entity_name = mappings_file['entities'][entity]['query']['name']
filters_str = get_filters(filters)
attributes = get_attributes(entity, mappings_file)
query = query_input.format(
entity_name=entity_name,
filters=filters_str,
attributes=attributes
)
json_data = call_api(endpoint=endpoint, query=query)
json_records = []
response_lenght = len(json_data['data'][entity_name])
if (response_lenght > 0):
list_data = json_data['data'][entity_name]
json_records = [*list_data]
return json_records
def get_data_parameter(query_input, entity, mappings_file, endpoint):
"""
Gets all the existing data for the given entity.
If this entity has some filter, in the config file, the query will apply
this filter
"""
are_data = True
json_records = []
entity_name = mappings_file['entities'][entity]['query']['name']
order_by = mappings_file['entities'][entity]['query']['params']['orderBy']
filter_value = mappings_file['entities'][entity]['query']['params'][
'initial_value']
attributes = get_attributes(entity, mappings_file)
while are_data:
query = query_input.format(entity_name=entity_name,
order_by=order_by,
filter_value=filter_value,
attributes=attributes)
json_data = call_api(endpoint=endpoint, query=query)
response_lenght = len(json_data['data'][entity_name])
if (response_lenght > 0):
list_data = json_data['data'][entity_name]
filter_value = json_data['data'][entity_name][response_lenght -
1][order_by]
json_records = [*json_records, *list_data]
else:
are_data = False
return json_records
def get_first_element(query_input, entity, mappings_file, endpoint, timestamp, aditional_filters, block=None):
"""
Gets first existing data from the subgraph applying the given filters
"""
entity_name = mappings_file['entities'][entity]['query']['name']
filters_str = get_filters(aditional_filters)
order_by = mappings_file['entities'][entity]['query']['params']['orderBy']
attributes = get_attributes(entity, mappings_file)
params = filter_method(block=block,
order_by_filter=order_by,
lte='_lte:',
timestamp=timestamp)
query = query_input.format(
entity_name=entity_name,
order_by=order_by,
order_by_filter=params['order_by_filter'],
aditional_filters=filters_str,
attributes=attributes,
timestamp=params['timestamp'],
lte=params['lte'],
block=params['block']
)
json_data = call_api(endpoint=endpoint, query=query)
json_records = []
response_lenght = len(json_data['data'][entity_name])
if (response_lenght > 0):
list_data = json_data['data'][entity_name]
json_records = [*list_data]
return json_records
def test_get_attributes():
config_file = pkgutil.get_data('deficrawler.config',
'aave' + "-" + str('2') + ".json")
map_file = json.loads(config_file.decode())
borrow_attr = get_attributes('borrow', map_file)
print(borrow_attr ==
"reserve{decimals} id user{id} reserve{symbol} amount timestamp ")
config_file = pkgutil.get_data('deficrawler.config',
'uniswap' + "-" + str('2') + ".json")
map_file = json.loads(config_file.decode())
swap_attr = get_attributes('swap', map_file)
assert (
swap_attr ==
"sender transaction{id} pair{token0{symbol}} pair{token1{symbol}} pair{token0{symbol}} pair{token1{symbol}} amount0In amount1In amount0Out amount1Out pair{id} timestamp "
)
def get_data_from(query_input,
entity,
from_timestamp,
to_timestamp,
mappings_file,
endpoint,
aditional_filters=""):
"""
Gets all the existing data from the subgraph at the given time range.
One or mor filters can be passed as parameters and will be applied in the where clause
"""
are_data = True
json_records = []
iteration_timestamp = from_timestamp
entity_name = mappings_file['entities'][entity]['query']['name']
order_by = mappings_file['entities'][entity]['query']['params']['orderBy']
attributes = get_attributes(entity, mappings_file)
filters_str = get_filters(aditional_filters)
while are_data:
query = query_input.format(entity_name=entity_name,
order_by=order_by,
from_timestamp=iteration_timestamp,
to_timestamp=to_timestamp,
attributes=attributes,
aditional_filters=filters_str)
json_data = call_api(endpoint=endpoint, query=query)
response_lenght = len(json_data['data'][entity_name])
if (response_lenght > 0):
list_data = json_data['data'][entity_name]
iteration_timestamp = json_data['data'][entity_name][
response_lenght - 1][order_by]
json_records = [*json_records, *list_data]
else:
are_data = False
return json_records
