def test_put_tuple_record_success(self):
project_name = 'put'
topic_name = 'success'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
records = []
record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000])
record0.shard_id = '0'
record0.shard_id = '0'
records.append(record0)
record1 = TupleRecord(schema=record_schema)
record1.values = [-9223372036854775808, 'yc1', 10.01, True, -62135798400000000]
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records.append(record1)
record2 = TupleRecord(schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000])
record2.set_value(0, 9223372036854775807)
record2.set_value('string_field', 'yc1')
record2.partition_key = 'TestPartitionKey'
records.append(record2)
with HTTMock(datahub_api_mock):
put_result = dh.put_records(project_name, topic_name, records)
assert put_result.failed_record_count == 0
assert put_result.failed_records == []
def pub_topic(self):
#logger.debug(self.to_string() + "pub_topic() self.symbols_analyze={0}".format(self.symbols_analyze))
if self.userid != 0:
return
topic_name = "t_symbols_analyze"
topic, shards = g_datahub.get_topic(topic_name)
record = TupleRecord(schema=topic.record_schema)
record.values = [
self.symbols_analyze.f_ex_id, self.symbols_analyze.f_symbol,
self.symbols_analyze.f_timeframe, self.symbols_analyze.f_bid,
self.symbols_analyze.f_ask, self.symbols_analyze.f_spread,
self.symbols_analyze.f_bar_trend,
self.symbols_analyze.f_volume_mean, self.symbols_analyze.f_volume,
self.symbols_analyze.f_ma_period, self.symbols_analyze.f_ma_up,
self.symbols_analyze.f_ma_low, self.symbols_analyze.f_ma_trend,
self.symbols_analyze.f_channel_period,
self.symbols_analyze.f_channel_up,
self.symbols_analyze.f_channel_low,
self.symbols_analyze.f_breakout_trend,
self.symbols_analyze.f_breakout_ts,
self.symbols_analyze.f_breakout_price,
self.symbols_analyze.f_breakout_volume,
self.symbols_analyze.f_breakout_volume_rate,
self.symbols_analyze.f_breakout_price_highest,
self.symbols_analyze.f_breakout_price_highest_ts,
self.symbols_analyze.f_breakout_rate,
self.symbols_analyze.f_breakout_rate_max,
arrow.utcnow().timestamp * 1000
]
record.shard_id = shards[randint(1, 1000) % len(shards)].shard_id
g_datahub.pub_topic(topic_name, [record])
def fetch_ticker_2_datahub(self, msg):
#logger.debug(self.to_string() + 'callback_ticker() msg={0}'.format(msg))
topic, shards = self.topic['t_ticker']
records = []
for t in msg:
ticker = self.ticker_2_ccxt(t)
symbol = ticker['symbol']
f_ts = ticker['timestamp'] and ticker['timestamp'] or int(
arrow.utcnow().timestamp * 1000)
f_bid = ticker['bid'] and ticker['bid'] or 0
f_bid_volume = ticker['bidVolume'] and ticker['bidVolume'] or 0
f_ask = ticker['ask'] and ticker['ask'] or 0
f_ask_volume = ticker['askVolume'] and ticker['askVolume'] or 0
f_vwap = ticker['vwap'] and ticker['vwap'] or 0
f_open = ticker['open'] and ticker['open'] or 0
f_high = ticker['high'] and ticker['high'] or 0
f_low = ticker['low'] and ticker['low'] or 0
f_close = ticker['close'] and ticker['close'] or 0
f_last = ticker['last'] and ticker['last'] or 0
f_previous_close = ticker['previousClose'] and ticker[
'previousClose'] or 0
f_change = ticker['change'] and ticker['change'] or 0
f_percentage = ticker['percentage'] and ticker['percentage'] or 0
f_average = ticker['average'] and ticker['average'] or 0
f_base_volume = ticker['baseVolume'] and ticker['baseVolume'] or 0
f_quote_volume = ticker['quoteVolume'] and ticker[
'quoteVolume'] or 0
f_ts_update = arrow.utcnow().timestamp
v = [
ws_binance.ex_id, symbol, f_ts, f_bid, f_bid_volume, f_ask,
f_ask_volume, f_vwap, f_open, f_high, f_low, f_close, f_last,
f_previous_close, f_change, f_percentage, f_average,
f_base_volume, f_quote_volume, f_ts_update
]
record = TupleRecord(schema=topic.record_schema)
record.values = v
i = random.randint(1, 100) % len(shards)
record.shard_id = shards[i].shard_id
#logger.debug(self.to_string() + 'callback_ticker() record={}'.format(record))
records.append(record)
logger.debug(self.to_string() +
'callback_ticker() len(records)={0}'.format(len(records)))
try:
g_datahub.pub_topic('t_ticker', records)
except Exception as e:
logger.info(self.to_string() + 'callback_ticker() e={0}'.format(e))
async def fetch_ticker(self, ex_id, topic, shards, symbol):
self.init_exchange(ex_id)
#logger.debug(self.to_string() + "fetch_ticker({0})".format(ex_id))
records = []
ticker = await self.exchanges[ex_id].ex.fetch_ticker(symbol)
f_ts = ticker['timestamp'] and ticker['timestamp'] or int(
arrow.utcnow().timestamp * 1000)
f_bid = ticker['bid'] and ticker['bid'] or 0
f_bid_volume = ticker['bidVolume'] and ticker['bidVolume'] or 0
f_ask = ticker['ask'] and ticker['ask'] or 0
f_ask_volume = ticker['askVolume'] and ticker['askVolume'] or 0
f_vwap = ticker['vwap'] and ticker['vwap'] or 0
f_open = ticker['open'] and ticker['open'] or 0
f_high = ticker['high'] and ticker['high'] or 0
f_low = ticker['low'] and ticker['low'] or 0
f_close = ticker['close'] and ticker['close'] or 0
f_last = ticker['last'] and ticker['last'] or 0
f_previous_close = ticker['previousClose'] and ticker[
'previousClose'] or 0
f_change = ticker['change'] and ticker['change'] or 0
f_percentage = ticker['percentage'] and ticker['percentage'] or 0
f_average = ticker['average'] and ticker['average'] or 0
f_base_volume = ticker['baseVolume'] and ticker['baseVolume'] or 0
f_quote_volume = ticker['quoteVolume'] and ticker['quoteVolume'] or 0
f_ts_update = arrow.utcnow().timestamp
v = [
ex_id, symbol, f_ts, f_bid, f_bid_volume, f_ask, f_ask_volume,
f_vwap, f_open, f_high, f_low, f_close, f_last, f_previous_close,
f_change, f_percentage, f_average, f_base_volume, f_quote_volume,
f_ts_update
]
record = TupleRecord(schema=topic.record_schema)
record.values = v
i = random.randint(1, 100) % len(shards)
record.shard_id = shards[i].shard_id
fetch_base.__symbol_ex_ticker[symbol][ex_id] = {
"f_ts": f_ts,
"f_bid": f_bid,
"f_ask": f_ask,
}
#await fetch_base.__queue_task_spread.put(v)
records.append(record)
return records
shards = shards_result.shards
for shard in shards:
print(shard)
print("=======================================\n\n")
while True:
records = []
record0 = TupleRecord(schema=topic.record_schema,
values=[1, 'yc1', 10.01, True, 1455869335000000])
record0.shard_id = shards[0].shard_id
record0.put_attribute('AK', '47')
records.append(record0)
record1 = TupleRecord(schema=topic.record_schema)
record1.values = [1, 'yc1', 10.01, True, 1455869335000000]
record1.shard_id = shards[1].shard_id
records.append(record1)
record2 = TupleRecord(schema=topic.record_schema)
record2.set_value(0, 3)
record2.set_value(1, 'yc3')
record2.set_value('double_field', 10.03)
record2.set_value('bool_field', False)
record2.set_value('time_field', 1455869335000013)
record2.shard_id = shards[2].shard_id
records.append(record2)
failed_indexs = dh.put_records(project_name, topic_name, records)
print("put tuple %d records, failed list: %s" %
(len(records), failed_indexs))
async def run_fetch_ohlcv(self, ex_id, topic_name, symbols, timeframe_str,
since_ms, split_i, max_split_count):
self.init_exchange(ex_id)
if not self.exchanges[ex_id].has_api('fetchOHLCV'):
logger.warn(self.to_string() +
"run_fetch_ohlcv({0}) NOT has interface".format(ex_id))
return
#logger.debug(self.to_string() + "run_fetch_ohlcv({0})".format(ex_id))
'''
if not self.exchanges[ex_id].ex.timeframes or timeframe_str not in self.exchanges[ex_id].ex.timeframes:
logger.info(self.to_string() + "run_fetch_ohlcv({0}) NOT has timeframe={1}".format(ex_id, timeframe_str))
return
'''
if not symbols or len(symbols) <= 0:
symbols = [k for k in fetch_base.__ex_symbol_fee[ex_id].keys()]
#logger.debug(self.to_string() + "run_fetch_ohlcv({0},{1},{2}) len(symbols)={3}".format(ex_id, topic_name, timeframe_str, len(symbols)))
symbols_todu = []
s = 0
for symbol in symbols:
if s % max_split_count == split_i:
symbols_todu.append(symbol)
s = s + 1
#logger.debug(self.to_string() + "run_fetch_ohlcv({0},{1},{2}) len(symbols_todu)={3}".format(ex_id, topic_name, timeframe_str, len(symbols_todu)))
if len(symbols_todu) <= 0:
return
topic, shards = g_datahub.get_topic(topic_name)
f_timeframe = util.TimeFrame_Minutes[timeframe_str]
while True:
ts_start = arrow.utcnow().shift(
minutes=-f_timeframe).timestamp * 1000
i = 0
for symbol in symbols_todu:
try:
data = await self.exchanges[ex_id].fetch_ohlcv(
symbol, timeframe_str, since_ms)
except ccxt.RequestTimeout:
logger.debug(
self.to_string() +
"run_fetch_ohlcv({0},{1},{2}) len(symbols_todu)={3} symbol={4} RequestTimeout "
.format(ex_id, topic_name, timeframe_str,
len(symbols_todu), symbol))
await asyncio.sleep(10)
continue
except ccxt.DDoSProtection:
logger.debug(
self.to_string() +
"run_fetch_ohlcv({0},{1},{2}) len(symbols_todu)={3} symbol={4} RequestTimeout "
.format(ex_id, topic_name, timeframe_str,
len(symbols_todu), symbol))
await asyncio.sleep(10)
continue
except Exception as e:
#logger.error(traceback.format_exc())
logger.debug(
self.to_string() +
"run_fetch_ohlcv({0},{1},{2}) len(symbols_todu)={3} Exception={4}"
.format(ex_id, topic_name, timeframe_str,
len(symbols_todu), e))
await asyncio.sleep(10)
continue
f_ts_update = arrow.utcnow().timestamp
#logger.debug(self.to_string() + "run_fetch_ohlcv() f_ts_update={0}".format(f_ts_update))
records = []
for d in data:
f_ts = d[0]
f_o = d[1]
f_h = d[2]
f_l = d[3]
f_c = d[4]
f_v = d[5]
record = TupleRecord(schema=topic.record_schema)
record.values = [
ex_id, symbol, f_timeframe, f_ts, f_o, f_h, f_l, f_c,
f_v, f_ts_update
]
record.shard_id = shards[i % len(shards)].shard_id
records.append(record)
i = i + 1
#logger.debug(self.to_string() + "run_fetch_ohlcv({0},{1},{2},{3})len(records) = {4}".format(ex_id, topic_name, symbol, timeframe_str, len(records)))
g_datahub.pub_topic(topic_name, records)
await asyncio.sleep(3)
since_ms = ts_start
await asyncio.sleep(3)
async def run_calc_spread(self, topic_name="t_spread"):
#logger.debug(self.to_string() + "run_calc_spread()")
topic, shards = g_datahub.get_topic(topic_name)
shard_count = len(shards)
while True:
try:
# 数据太多，处理不完
qsize = fetch_base.__queue_task_spread.qsize()
if qsize >= 100:
logger.warn(self.to_string() +
"run_calc_spread() qsize={0}".format(qsize))
# for test
await fetch_base.__queue_task_spread.get()
continue
# [f_ex_id, f_symbol, f_ts, f_bid, f_bid_volume, f_ask, f_ask_volume, f_vwap, f_open, f_high, f_low, f_close, f_last, f_previous_close, f_change, f_percentage, f_average, f_base_volume, f_quote_volume]
task_record = await fetch_base.__queue_task_spread.get()
symbol = task_record[1]
ex1 = task_record[0]
ex1_name = self.exchanges[ex1].ex.name
ex1_bid = task_record[3]
ex1_ask = task_record[5]
ex1_ts = task_record[2]
ex1_fee = fetch_base.__ex_symbol_fee[ex1][
symbol] if fetch_base.__ex_symbol_fee[ex1][symbol] else 0
record2s = fetch_base.__symbol_ex_ticker[
symbol] if fetch_base.__symbol_ex_ticker[symbol] else {}
records = []
for ex2, v in record2s.items():
if ex2 == ex1:
continue
ex2_name = self.exchanges[ex2].ex.name
ex2_bid = v["f_bid"]
ex2_ask = v["f_ask"]
ex2_ts = v["f_ts"]
ex2_fee = fetch_base.__ex_symbol_fee[ex2][
symbol] if fetch_base.__ex_symbol_fee[ex2][
symbol] else 0
if abs(ex1_ts - ex2_ts) > 30000:
logger.info(
self.to_string() +
"run_calc_spread() abs(ex1_ts - ex2_ts)={0}".
format(abs(ex1_ts - ex2_ts)))
continue
spread_ts = ex1_ts if ex1_ts > ex2_ts else ex2_ts
f_fee = (ex1_bid * ex1_fee + ex2_ask * ex2_fee)
f_spread = ex1_bid - ex2_ask
f_profit = (f_spread - f_fee)
f_profit_p = (f_profit / ex1_bid) if ex1_bid > 0.0 else 0.0
f_ts_update = arrow.utcnow().timestamp
record1 = TupleRecord(schema=topic.record_schema)
record1.values = [
symbol, ex1, ex1_name, ex1_bid, ex1_ts, ex1_fee, ex2,
ex2_name, ex2_ask, ex2_ts, ex2_fee, spread_ts,
f_spread, f_fee, f_profit, f_profit_p, f_ts_update
]
i = random.randint(1, 100) % shard_count
record1.shard_id = shards[i].shard_id
records.append(record1)
f_spread = ex2_bid - ex1_ask
f_profit = (f_spread - f_fee)
f_profit_p = (f_profit / ex2_bid) if ex2_bid > 0.0 else 0.0
record2 = TupleRecord(schema=topic.record_schema)
record2.values = [
symbol, ex2, ex2_name, ex2_bid, ex2_ts, ex2_fee, ex1,
ex1_name, ex1_ask, ex1_ts, ex1_fee, spread_ts,
f_spread, f_fee, f_profit, f_profit_p, f_ts_update
]
i = random.randint(1, 100) % shard_count
record2.shard_id = shards[i].shard_id
records.append(record2)
g_datahub.pub_topic(topic_name, records)
except DatahubException as e:
logger.error(traceback.format_exc(e))
except Exception as e:
logger.error(traceback.format_exc(e))
except:
logger.error(traceback.format_exc())
async def fetch_tickers(self, ex_id, topic, shards):
# 降低 CPU ，暂时性
await asyncio.sleep(30)
self.init_exchange(ex_id)
#logger.debug(self.to_string() + "fetch_tickers({0})".format(ex_id))
records = []
if not self.exchanges[ex_id].has_api('fetchTickers'):
for symbol in fetch_base.__ex_symbol_fee[ex_id].keys():
# 降低 CPU ，暂时性
await asyncio.sleep(3)
try:
rs = await self.fetch_ticker(ex_id, topic, shards, symbol)
records.extend(rs)
except ccxt.RequestTimeout:
#logger.error(traceback.format_exc())
logger.info(
self.to_string() +
"fetch_tickers() fetch_ticker({0},{1}) RequestTimeout "
.format(ex_id, symbol))
await asyncio.sleep(10)
except ccxt.DDoSProtection:
#logger.error(traceback.format_exc())
logger.info(
self.to_string() +
"fetch_tickers() fetch_ticker({0},{1}) DDoSProtection "
.format(ex_id, symbol))
await asyncio.sleep(10)
except Exception as e:
#logger.error(traceback.format_exc())
logger.error(
self.to_string() +
"fetch_tickers() fetch_ticker({0},{1}) Exception={2}".
format(ex_id, symbol, e))
await asyncio.sleep(10)
except:
logger.error(traceback.format_exc())
logger.error(self.to_string() +
"fetch_tickers() fetch_ticker({0},{1})".
format(ex_id, symbol))
await asyncio.sleep(10)
logger.debug(self.to_string() +
"fetch_tickers({0}) len(records)={1}".format(
ex_id, len(records)))
return records
tickers = await self.exchanges[ex_id].ex.fetch_tickers()
i = 0
for symbol, ticker in tickers.items():
f_ts = ticker['timestamp'] and ticker['timestamp'] or int(
arrow.utcnow().timestamp * 1000)
f_bid = ticker['bid'] and ticker['bid'] or 0
f_bid_volume = ticker['bidVolume'] and ticker['bidVolume'] or 0
f_ask = ticker['ask'] and ticker['ask'] or 0
f_ask_volume = ticker['askVolume'] and ticker['askVolume'] or 0
f_vwap = ticker['vwap'] and ticker['vwap'] or 0
f_open = ticker['open'] and ticker['open'] or 0
f_high = ticker['high'] and ticker['high'] or 0
f_low = ticker['low'] and ticker['low'] or 0
f_close = ticker['close'] and ticker['close'] or 0
f_last = ticker['last'] and ticker['last'] or 0
f_previous_close = ticker['previousClose'] and ticker[
'previousClose'] or 0
f_change = ticker['change'] and ticker['change'] or 0
f_percentage = ticker['percentage'] and ticker['percentage'] or 0
f_average = ticker['average'] and ticker['average'] or 0
f_base_volume = ticker['baseVolume'] and ticker['baseVolume'] or 0
f_quote_volume = ticker['quoteVolume'] and ticker[
'quoteVolume'] or 0
f_ts_update = arrow.utcnow().timestamp
v = [
ex_id, symbol, f_ts, f_bid, f_bid_volume, f_ask, f_ask_volume,
f_vwap, f_open, f_high, f_low, f_close, f_last,
f_previous_close, f_change, f_percentage, f_average,
f_base_volume, f_quote_volume, f_ts_update
]
record = TupleRecord(schema=topic.record_schema)
record.values = v
record.shard_id = shards[i % len(shards)].shard_id
records.append(record)
i = i + 1
fetch_base.__symbol_ex_ticker[symbol][ex_id] = {
"f_ts": f_ts,
"f_bid": f_bid,
"f_ask": f_ask,
}
#await fetch_base.__queue_task_spread.put(v)
logger.debug(
self.to_string() +
"fetch_tickers({0}) len(records)={1}".format(ex_id, len(records)))
return records
def handler(event, context):
logger = logging.getLogger()
evt = json.loads(event)
#print("[print1]IoT trigger and send data to FunctionCompute test output, The content of event is : %s" % (evt))
timestamp = evt['timestamp']
values = evt['values']
count_of_value = len(values)
ACCESS_ID = 'XXXXX'
ACCESS_KEY = 'XXXXX'
ENDPOINT = 'http://dh-cn-XXXXX.aliyun-inc.com'
dh = DataHub(ACCESS_ID, ACCESS_KEY, ENDPOINT)
PROJECT_NAME = 'veolia_d4b_poc'
TOPIC_NAME = 'extract_result_table'
# ===================== put tuple records =====================
# block等待所有shard状态ready
dh.wait_shards_ready(PROJECT_NAME, TOPIC_NAME)
topic = dh.get_topic(PROJECT_NAME, TOPIC_NAME)
record_schema = topic.record_schema
shards_result = dh.list_shard(PROJECT_NAME, TOPIC_NAME)
shards = shards_result.shards
shard_count = len(shards)
# for shard in shards:
# print("[print8]IoT trigger and send data to FunctionCompute test output, The Shard is : (%s)" % (shard))
records = []
for value in values:
# id sample: SE433_OPC.S01.AISA0101
id = value['id']
id_list = id.split('.')
id_company_code = (id_list[0].split('_'))[0]
id_protocol_name = (id_list[0].split('_'))[1]
id_system_code = id_list[1]
id_tagname = id_list[2]
v = value['v']
q = 'true' if value['q'] else 'false'
t = value['t']
#print("[print7]IoT trigger and send data to FunctionCompute test output, The value is : (%s, %s, %s, %s)" % (id,v,q,t))
rec = TupleRecord(schema=topic.record_schema)
rec.values = [
timestamp, id_company_code, id_protocol_name, id_system_code,
id_tagname, v, q, t
]
rec.shard_id = shards[random.randint(0, shard_count - 1)].shard_id
records.append(rec)
failed_indexs = dh.put_records(PROJECT_NAME, TOPIC_NAME, records)
print("[print9] put tuple %d records, shard_id = %s, failed list: %s" %
(len(records), rec.shard_id, failed_indexs))
# failed_indexs如果非空最好对failed record再进行重试
return 'success'
# event样例：
# {
# "timestamp":1521698375065,
# "values":[
# {
# "id":"SE433_OPC.S01.IW1440",
# "v":206,
# "q":true,
# "t":1521698358299
# },
# {
# "id":"SESE433_OPC433.S01.LCV1414_ACT",
# "v":42,
# "q":true,
# "t":1521698358222
# },
# {
# "id":"SE433_OPC.S01.LT1430A",
# "v":22,
# "q":true,
# "t":1521698358235
# },
# …
# ]
# }
def handler(event, context):
logger.debug(event)
str = event.decode()
if str == "":
logger.debug("str == \"\"")
return
evt = json.loads(str)
if not evt:
logger.debug("evt is None")
return
records = evt.get("records")
if not records:
logger.debug("records is None")
return
logger.debug("len(records)={0}".format(len(records)))
logger.debug(records)
for record in records:
data = record["data"]
ex1 = data[0]
symbol = data[1]
ex1_ts = data[2]
ex1_bid = data[3]
ex1_ask = data[5]
sql_select = "SELECT f_ex_id,f_bid,f_ask,f_ts FROM t_ticker_crrent where f_symbol=\"{0}\" and f_ex_id!=\"{1}\" and f_ts > (UNIX_TIMESTAMP() - 30) * 1000;".format(
symbol, ex1)
logger.debug("sql_select={0}".format(sql_select))
cursor_reaed = conn_read.cursor()
cursor_reaed.execute(sql_select)
rows = cursor_reaed.fetchall()
logger.debug("rows={0}".format(len(rows)))
if len(rows) <= 0:
logger.debug("len(rows) <= 0")
continue
records = []
sql_insert = "replace into t_spread_current(f_symbol,f_ex1,f_ex2,f_spread,f_ts) values"
c = 0
cursor_write = conn_write.cursor()
for row in rows:
ex2 = row[0]
ex2_bid = row[1]
ex2_ask = row[2]
ex2_ts = row[3]
spread_ts = ex1_ts if ex1_ts > ex2_ts else ex2_ts
sql_value1 = "('{0}','{1}','{2}',{3},{4})".format(
symbol, ex1, ex2, ex1_bid - ex2_ask, spread_ts)
sql_value2 = ",('{0}','{1}','{2}',{3},{4})".format(
symbol, ex2, ex1, ex2_bid - ex1_ask, spread_ts)
if c == 0:
sql_insert = sql_insert + sql_value1 + sql_value2
else:
sql_insert = sql_insert + "," + sql_value1 + sql_value2
c = c + 1
record1 = TupleRecord(schema=topic.record_schema)
record1.values = [symbol, ex1, ex2, ex1_bid - ex2_ask, spread_ts]
record1.shard_id = shards[c % len(shards)].shard_id
records.append(record1)
c = c + 1
record2 = TupleRecord(schema=topic.record_schema)
record2.values = [symbol, ex2, ex1, ex2_bid - ex1_ask, spread_ts]
record2.shard_id = shards[c % len(shards)].shard_id
records.append(record2)
sql_insert = sql_insert + ";"
logger.debug("sql_insert={0}".format(sql_insert))
count_write = cursor_write.execute(sql_insert)
logger.debug(count_write)
count_write.commit()
logger.debug("datahub.put_records={0}".format(len(records)))
datahub.put_records(project_name, topic_name, records)