def generate_new_thread(exchange, number):
if exchange == "Bitfinex":
client = Web_Client.Bitfinex()
client.start_book("tETHBTC", partial(process_message_2,exchange = "Bitfinex", pair = "ethbtc" + str(number)))
elif exchange == "Coinbase":
client = Web_Client.Coinbase()
client.start_ticker('ETH-BTC', partial(process_message_2,exchange = "Coinbase", pair = "ethbtc" + str(number)))
elif exchange == "Kraken":
client = Web_Client.Kraken()
client.start_book('ETH/XBT', partial(process_message_2,exchange = "Kraken", pair = "ethbtc" + str(number)))
elif exchange == "Bithumb":
client = Web_Client.Bithumb()
client.start_order_book('ETH-BTC', partial(process_message_2,exchange = "Bithumb", pair = "ethbtc" + str(number)))
return client
print(a.currencies())
print("We get the first pair!11\n")
print(a.time())
a.close()
def process_message(msg):
try:
print("message type: {}".format(msg['e']))
print(msg)
except:
print("message type: Other")
print(msg)
b = Web_Client.Huobi()
# start any sockets here, i.e a trade socket
#b.start_candle('btcusdt', process_message)
#b.start_by_price('btcusdt', process_message)
#b.start_depth('btcusdt', process_message)
#b.start_bid_offer('btcusdt', process_message)
#b.start_24h('btcusdt', process_message)
b.start_trade('btcusdt', process_message)
b.start()
#print(b.is_alive())
time.sleep(20)
b.close()
print("LIQUIDATION\n")
print(cl.liquidation())
print("LEADERBOARD\n")
print(cl.leaderboards("tBTCUSD"))
def process_message(msg):
try:
print("message type: {}".format(msg['e']))
print(msg)
except:
print("message type: Other")
print(msg)
# do something
b = Web_Client.Bitfinex()
# start any sockets here, i.e a trade socket
b.start_ticker('tBTCUSD', process_message)
b.start_trades('tBTCUSD', process_message)
b.start_book('tBTCUSD', process_message)
b.start_raw_book('tBTCUSD', process_message)
b.start_candles('tBTCUSD', process_message)
b.start_status(process_message)
b.start()
#print(b.is_alive())
time.sleep(20)
b.close()
#print("We get the first pair!\n")
#print(a.order_book("BTC", "KRW"))
#print("We get the first pair!\n")
#print(a.History("BTC", "KRW"))
#print("We get the first pair!\n")
#print(a.index())
a.close()
def process_message(msg):
try:
print("message type: {}".format(msg['e']))
print(msg)
except:
print("message type: Other")
print(msg)
b = Web_Client.Bithumb()
# start any sockets here, i.e a trade socket
b.start_ticker('BTC-USDT', process_message)
b.start_order_book('BTC-USDT', process_message)
b.start_trade('BTC-USDT', process_message)
b.start()
#print(b.is_alive())
time.sleep(20)
b.close()
print("We get the first pair!\n")
print(a.transactions("BTC", "USD"))
print("We get the first pair!\n")
print(a.eur_usd())
a.close()
def process_message(msg):
try:
print("message type: {}".format(msg['e']))
print(msg)
except:
print("message type: Other")
print(msg)
b = Web_Client.Bitstamp()
# start any sockets here, i.e a trade socket
b.start_ticker('btcusd', process_message)
b.start_orderBook('btcusd', process_message)
b.start_liveOrder('btcusd', process_message)
b.start_detailOrder('btcusd', process_message)
b.start_liveFull('btcusd', process_message)
b.start()
time.sleep(4)
b.close()
#print("We get the first pair!\n")
#print(a.Trades("ADAEUR"))
#print("We get the first pair!\n")
#print(a.Spread("ADAEUR"))
#a.close()
def process_message(msg):
try:
print("message type: {}".format(msg['e']))
print(msg)
except:
print("message type: Other")
print(msg)
b = Web_Client.Kraken()
## start any sockets here, i.e a trade socket
b.start_ticker('XBT/EUR', process_message)
b.start_OHLC('XBT/EUR', process_message)
b.start_trade('XBT/EUR', process_message)
b.start_book('XBT/EUR', process_message)
b.start_spread('XBT/EUR', process_message)
b.start()
#print(b.is_alive())
time.sleep(20)
b.close()
def loadPair():
groups = ["Binance_ETHBTC","Bitfinex_ETHBTC","Bitfinex_ETHBTC2","Bitfinex_ETHBTC3","Bitstamp_ETHBTC",
"Bithumb_ETHBTC","Bithumb_ETHBTC2", "Coinbase_ETHBTC","Coinbase_ETHBTC2","Coinbase_ETHBTC3","Huobi_ETHBTC", "Kraken_ETHBTC", "Kraken_ETHBTC2","Kraken_ETHBTC3"]
text = "global client, file_out "
for element in groups:
text = text + "," + element + "_df_dif"
text = text + "," + element + "_df"
text = text + "," + element + "_df_Book"
exec(text, globals())
for element in groups:
exec(element + "_df_dif = pd.DataFrame(columns = ['t', 'Host', 'Pair', 'Seg', 'Bids', 'Asks'])", globals())
exec(element + "_df = pd.DataFrame(columns = ['t', 'Host', 'Pair', 'Price', 'Q'])", globals())
exec(element + "_df_Book = pd.DataFrame(columns = ['t', 'Host', 'Pair', 'LastUpdateID','Bids','Asks'])" , globals())
client = DataFrameClient('localhost', 8086, 'root', 'root')
script_dir = os.path.dirname(__file__)
date = datetime.now()
rel_path = "Logs\\log_" + str(date.date()) + "_" + str(date.hour) + str(date.minute) + str(date.second) + ".txt"
abs_file_path = os.path.join(script_dir, rel_path)
globals()["file_out"] = open(abs_file_path, "a")
#if {"name":"SecondM"} not in client.get_list_database():
# client.create_database("SecondM")
#else:
# client.drop_database("SecondM")
# client.create_database("SecondM")
Binance = Web_Client.Binance()
Binance_REST = Client.Binance()
Bitfinex = Web_Client.Bitfinex()
Bitfinex2 = generate_new_thread("Bitfinex", "")
Bitfinex_REST = Client.Bitfinex()
Bithumb = Web_Client.Bithumb()
Bithumb_REST = Client.Bithumb()
Bitstamp = Web_Client.Bitstamp()
Coinbase = Web_Client.Coinbase()
Coinbase2 = generate_new_thread("Coinbase", "")
Kraken = Web_Client.Kraken()
Kraken2 = generate_new_thread("Kraken", "")
Huobi = Web_Client.Huobi()
Binance.start_trade_socket('ethbtc', partial(process_message,exchange = "Binance", pair = "ethbtc"))
Bitfinex.start_trades("tETHBTC", partial(process_message,exchange = "Bitfinex", pair = "ethbtc"))
Bithumb.start_trade('ETH-BTC', partial(process_message,exchange = "Bithumb", pair = "ethbtc"))
Bitstamp.start_ticker('ethbtc', partial(process_message,exchange = "Bitstamp", pair = "ethbtc"))
Coinbase.start_matches('ETH-BTC', partial(process_message,exchange = "Coinbase", pair = "ethbtc"))
Huobi.start_trade('ethbtc', partial(process_message,exchange = "Huobi", pair = "ethbtc"))
Kraken.start_trade('ETH/XBT', partial(process_message,exchange = "Kraken", pair = "ethbtc"))
#############################################
#############################################
#############################################
#############################################
Binance.start_depth_socket("ethbtc",partial(process_message_2,exchange = "Binance", pair = "ethbtc"))
Bithumb.start_order_book('ETH-BTC', partial(process_message_2,exchange = "Bithumb", pair = "ethbtc"))
Bitstamp.start_detailOrder('ethbtc', partial(process_message_2,exchange = "Bitstamp", pair = "ethbtc"))
Huobi.start_depth('ethbtc', partial(process_message_2,exchange = "Huobi", pair = "ethbtc"))
Binance.start()
Bithumb.start()
Bitstamp.start()
Coinbase.start()
Bitfinex.start()
Huobi.start()
Kraken.start()
Kraken2.start()
Coinbase2.start()
Bitfinex2.start()
for i in range(6):
try:
_msg_ = Binance_REST.depth("ETHBTC", limit = 1000)
t_temp = BinanceToTime(int(round(time.time() * 1000)))
d_2 = pd.DataFrame({ "t": [t_temp],
"Host": ["Binance"],
"Pair": ["ethbtc"],
"LastUpdateID": [_msg_["lastUpdateId"]],
"Bids":[_msg_["bids"]],
"Asks":[_msg_["asks"]]
})
var = globals()
var["d_2"] = d_2
exec("Binance_ETHBTC_df_Book = Binance_ETHBTC_df_Book.append(d_2)", var)
except Exception as e:
print("EXCEPTION!", file = file_out)
print("EXCEPTION!")
print(e, file = file_out)
print(e)
time.sleep(4)
status = 3
for i in range(2100):
try:
Bithumb2.close()
except:
pass
if status == 3:
try:
Bitfinex3.close()
Kraken3.close()
Coinbase3.close()
except:
pass
time.sleep(1)
Bitfinex3 = generate_new_thread("Bitfinex", "2")
Kraken3 = generate_new_thread("Kraken", "2")
Coinbase3 = generate_new_thread("Coinbase", "2")
time.sleep(3)
Bitfinex3.start()
Kraken3.start()
Coinbase3.start()
status = 4
elif status == 4:
try:
Bitfinex4.close()
Kraken4.close()
Coinbase4.close()
except:
pass
time.sleep(1)
Bitfinex4 = generate_new_thread("Bitfinex", "3")
Kraken4 = generate_new_thread("Kraken", "3")
Coinbase4 = generate_new_thread("Coinbase", "3")
time.sleep(3)
Kraken4.start()
Bitfinex4.start()
Coinbase4.start()
status = 2
elif status == 2:
try:
Bitfinex2.close()
Kraken2.close()
Coinbase2.close()
except:
pass
time.sleep(1)
Bitfinex2 = generate_new_thread("Bitfinex", "")
Kraken2 = generate_new_thread("Kraken", "")
Coinbase2 = generate_new_thread("Coinbase", "")
time.sleep(1)
Bitfinex2.start()
Kraken2.start()
Coinbase2.start()
status = 3
print("\n----------------")
print("\n----------------", file = globals()["file_out"])
print("CHECKING DATAFRAMES - " + str(i) + ": " + str(BinanceToTime(int(round(time.time() * 1000)))), file = globals()["file_out"])
print("CHECKING DATAFRAMES - " + str(i) + ": " + str(BinanceToTime(int(round(time.time() * 1000)))))
print("---------------- \n")
print("---------------- \n", file = globals()["file_out"])
if i % 5 == 0:
for element in groups:
ldict = {}
var = globals()
exec("df_temp = " + element + "_df",var, ldict)
df_temp = ldict["df_temp"]
exec("df_Book_temp = " + element + "_df_Book",var, ldict)
df_Book_temp = ldict["df_Book_temp"]
exec("df_dif_temp = " + element + "_df_dif",var, ldict)
df_dif_temp = ldict["df_dif_temp"]
var.update(ldict)
print(element + " : " + str(len(df_temp)) + "; " + str(len(df_Book_temp)) + "; " + str(len(df_dif_temp)))
print(element + " : " + str(len(df_temp)) + "; " + str(len(df_Book_temp)) + "; " + str(len(df_dif_temp)), file = globals()["file_out"])
if len(df_Book_temp)>0:
print("-------------------------------", file = globals()["file_out"])
print("-------------------------------")
df_Book_temp = df_Book_temp.set_index("t")
print("WRITING " + str(len(df_Book_temp)) + " lines in Book FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))), file = globals()["file_out"])
print("WRITING " + str(len(df_Book_temp)) + " lines in Book FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))))
try:
client.write_points(df_Book_temp.assign(Batch_ID = [i for i in range(len(df_Book_temp))]),"Book",time_precision = "n", database = "SecondM", tag_columns = ['Host', 'Pair', 'Batch_ID'], batch_size = 1000)
exec(element + "_df_Book = " + element + "_df_Book.iloc[len(df_Book_temp):]", var)
except Exception as e:
print("DATABASE PROBLEM!")
print(e)
print("-------------------------------", file = globals()["file_out"])
print("-------------------------------")
if len(df_dif_temp)>0:
print("-------------------------------", file = globals()["file_out"])
print("-------------------------------")
df_dif_temp = df_dif_temp.set_index("t")
print("WRITING " + str(len(df_dif_temp)) + " lines in difBook FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))), file = globals()["file_out"])
print("WRITING " + str(len(df_dif_temp)) + " lines in difBook FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))))
try:
client.write_points(df_dif_temp.assign(Batch_ID = [i for i in range(len(df_dif_temp))]),"difBook",time_precision = "n", database = "SecondM", tag_columns = ['Host', 'Pair', 'Batch_ID'], batch_size = 1000)
exec(element + "_df_dif = " + element + "_df_dif.iloc[len(df_dif_temp):]", var)
except Exception as e:
print("DATABASE PROBLEM!")
print(e)
print("-------------------------------", file = globals()["file_out"])
print("-------------------------------")
if len(df_temp)>0:
print("-------------------------------", file = globals()["file_out"])
print("-------------------------------")
df_temp = df_temp.set_index("t")
print("WRITING " + str(len(df_temp)) + " lines in Price FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))), file = globals()["file_out"])
print("WRITING " + str(len(df_temp)) + " lines in Price FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))))
try:
client.write_points(df_temp.assign(Batch_ID = [i for i in range(len(df_temp))]),"Price", time_precision = "n", database = "SecondM", tag_columns = ['Host', 'Pair', 'Batch_ID'], batch_size = 1000)
exec(element + "_df = " + element + "_df.iloc[len(df_temp):]", var)
except Exception as e:
print("DATABASE PROBLEM!")
print(e)
print("-------------------------------", file = globals()["file_out"])
print("-------------------------------")
try:
_msg_ = Binance_REST.depth("ETHBTC", limit = 1000)
d_2 = pd.DataFrame({ "t": [BinanceToTime(int(round(time.time() * 1000)))],
"Host": ["Binance"],
"Pair": ["ethbtc"],
"LastUpdateID": [_msg_["lastUpdateId"]],
"Bids":[_msg_["bids"]],
"Asks":[_msg_["asks"]]
})
var = globals()
var["d_2"] = d_2
exec("Binance_ETHBTC_df_Book = Binance_ETHBTC_df_Book.append(d_2)", var)
except Exception as e:
print("EXCEPTION!", file = file_out)
print("EXCEPTION!")
print(e, file = file_out)
print(e)
if i % 10 == 0:
Bithumb2 = generate_new_thread("Bithumb", "2")
time.sleep(2)
Bithumb2.start()
time.sleep(24)
Binance.close()
Bitfinex.close()
Bitfinex2.close()
Bitfinex3.close()
Bitfinex3.close()
Bithumb.close()
Bitstamp.close()
Coinbase.close()
Coinbase2.close()
Coinbase3.close()
Coinbase4.close()
Huobi.close()
Kraken.close()
Kraken2.close()
Kraken3.close()
Kraken4.close()
for element in groups:
ldict = {}
var = globals()
exec("df_temp = " + element + "_df",var, ldict)
df_temp = ldict["df_temp"]
exec("df_Book_temp = " + element + "_df_Book",var, ldict)
df_Book_temp = ldict["df_Book_temp"]
exec("df_dif_temp = " + element + "_df_dif",var, ldict)
df_dif_temp = ldict["df_dif_temp"]
var.update(ldict)
print("\n-------------------------------", file = globals()["file_out"])
print("\n-------------------------------")
df_Book_temp = df_Book_temp.set_index("t")
print("WRITING " + str(len(df_Book_temp)) + " lines in Book FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))), file = globals()["file_out"])
print("WRITING " + str(len(df_Book_temp)) + " lines in Book FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))))
try:
client.write_points(df_Book_temp.assign(Batch_ID = [i for i in range(len(df_Book_temp))]),"Book",time_precision = "n", database = "SecondM", tag_columns = ['Host', 'Pair', 'Batch_ID'], batch_size = 100)
except:
df_Book_temp.to_csv(element + "_df_Book")
df_dif_temp = df_dif_temp.set_index("t")
print("WRITING " + str(len(df_dif_temp)) + " lines in difBook FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))), file = globals()["file_out"])
print("WRITING " + str(len(df_dif_temp)) + " lines in difBook FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))))
try:
client.write_points(df_dif_temp.assign(Batch_ID = [i for i in range(len(df_dif_temp))]),"difBook",time_precision = "n", database = "SecondM", tag_columns = ['Host', 'Pair', 'Batch_ID'], batch_size = 100)
except:
df_dif_temp.to_csv(element + "_df_dif")
df_temp = df_temp.set_index("t")
print("WRITING " + str(len(df_temp)) + " lines in Price FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))), file = globals()["file_out"])
print("WRITING " + str(len(df_temp)) + " lines in Price FROM " + element + ": " + str(BinanceToTime(int(round(time.time() * 1000)))))
try:
client.write_points(df_temp.assign(Batch_ID = [i for i in range(len(df_temp))]),"Price", time_precision = "n", database = "SecondM", tag_columns = ['Host', 'Pair', 'Batch_ID'], batch_size = 100)
except:
df_temp.to_csv(element + "_df")
print("------------------------------- \n", file = globals()["file_out"])
print("------------------------------- \n")
print("..END..", file = globals()["file_out"])
print("..END..")
file_out.close()