def derive_secondary_tick(timepoint):
data = None
if timepoint < SECONDARY_SURGE_START:
data = "phase1"
elif timepoint < SECONDARY_SURGE_END:
data = "phase2"
else:
data = "phase3"
values = SECONDARY_DATA[data]
return TickSchema(price=values[0], timestamp=timepoint, volume=values[1])
def to_trades(self, in_data):
trades = []
for result in in_data:
trade = TickSchema(
price=result["price"],
volume=result["size"],
funds=float(result["price"]) * float(result["size"]),
timestamp=result["time"],
)
trades.append(trade)
return self.deduplicate(trades)
def to_trades(self, in_data):
trades = []
for result in in_data:
trade = TickSchema(
price=result["price"],
volume=result["volume"],
funds=result["funds"],
timestamp=result["created_at"],
)
trades.append(trade)
return self.deduplicate(trades)
def to_trades(self, in_data):
trades = []
for result in in_data:
trade = TickSchema(
price=result['price'],
volume=result['volume'],
funds=result['funds'],
timestamp=result['created_at'],
)
trades.append(trade)
return self.deduplicate(trades)
def to_trades(self, in_data):
trades = []
data = in_data["result"]
for result in data:
trade = TickSchema(
price=result[2],
volume=result[3],
funds=result[2] * result[3],
timestamp=result[1],
)
trades.append(trade)
return self.deduplicate(trades)
def get_all_data_gen(self, start_datetime: datetime, end_datetime: datetime):
job_config = bigquery.QueryJobConfig(
query_parameters=[
bigquery.ScalarQueryParameter('start', 'TIMESTAMP', start_datetime),
bigquery.ScalarQueryParameter('end', 'TIMESTAMP', end_datetime),
]
)
sql_query = load_ticks_query.format(
table_fullname=self.config['table_name'],
)
bigquery_result = exec_query(sql_query, bigquery_client, job_config=job_config)
chunk = []
print('generating')
for result in bigquery_result:
chunk.append(TickSchema(**dict(result.items())))
if len(chunk) >= 5000:
yield chunk
chunk = []
if len(chunk) > 0:
yield chunk