def getSearchTerms():
import json
from poloniex import Poloniex
polo = Poloniex()
eventNames = [
"upgrade", "updates", "releas", "testing", "aplha", "beta", "announce",
"interview", "major", "launch", "add", "improve", "v1"
]
coinMarketList = [
market[market.index("_") + 1:]
for market in polo.return24hVolume().keys() if "BTC_" in market
]
coinList = polo.returnCurrencies()
coinNames = []
ignoredCoins = [
"burst", "clams", "counterparty", "expanse", "dash", "horizon", "magi",
"nem", "nexium", "nxt", "omni", "radium", "ripple", "shadow",
"stellar", "tether"
]
for coin in coinList:
if not coinList[coin]["name"].lower(
) in ignoredCoins and coin in coinMarketList:
coinNames.append(coinList[coin]["name"].lower())
return [coinNames, eventNames]
def getCoinNames():
from poloniex import Poloniex
polo = Poloniex()
config = getConfig()
coinMarketList = [
market[market.index("_") + 1:]
for market in polo.return24hVolume().keys() if "BTC_" in market
]
coinList = polo.returnCurrencies()
coinNames = {}
ignoredCoins = config["ignoredCoins"]
for coin in coinList:
if not coinList[coin]["name"].lower(
) in ignoredCoins and coin in coinMarketList:
coinNames[coinList[coin]["name"].lower()] = "BTC_" + coin.upper()
return coinNames
class PoloniexDriver:
fee = 0.0025
name = "Poloniex"
def __init__(self, api_key, secret):
api_p=''
key_p=''
self.api=Poloniex(api_key, secret)
pass
def get_market_info(self):
market={}
try:
tickers=self.api.returnTicker()
for pairs, ticker in tickers.iteritems():
pair=pairs.split('_')
pair=tuple(pair)
market[pair]=(float(ticker['lowestAsk']), float(ticker['highestBid']))
except:
market={}
return market
def get_txFee(self):
txfee=list()
currencies=self.api.returnCurrencies()
for k, v in currencies.iteritems():
txfee.append(( k, v['name'], v['txFee'])) # create list of tuples(len = 2) 1 - trade, 2- trade name 3 - txFee
return txfee
def get_order(self, pairs): # return from lists sells and buy
pair = '{0}_{1}'.format(pairs[0], pairs[1])
try:
orders=self.api.returnOrderBook(pair,1) # return lists sells and buy
ask_rate=float(orders['asks'][0][0]) # ask-sell rate
bid_rate=float(orders['bids'] [0][0])#bid-buy rate
ask_quanity=float(orders['asks'][0][1])
bid_quanity=float(orders['bids'] [0][1])
order={'ask_rate': ask_rate, 'ask_quanity': ask_quanity, 'bid_rate': bid_rate, 'bid_quanity': bid_quanity}
except:
order={'ask_rate': '0', 'ask_quanity': '0', 'bid_rate': '0', 'bid_quanity': '0'}
return order
class DataManager():
def __init__(self):
self.polo = Poloniex()
self.currencies = self._get_currencies()
self.pairs = self._get_pairs()
self.btc_pairs = self._get_btc_pairs()
# dates
self.dateformat = "%d/%m/%Y %H:%M:%S"
self._today = date.today() # datetime object
self._prev_month = self._today - datedelta.MONTH # datetime object
self.today = self._today.strftime(self.dateformat) # string
self.prev_month = self._prev_month.strftime(self.dateformat) # string
def check_date_format(self):
return self.dateformat
def _get_currencies(self):
'''
Returns list of currency available in Poloniex
'''
currency_dict = self.polo.returnCurrencies()
currencies = list(currency_dict.keys())
return currencies
def _get_pairs(self):
'''
Returns list of currency-pair available in Poloniex
'''
ticker_dict = self.polo.returnTicker()
pairs = list(ticker_dict.keys())
return pairs
def _get_btc_pairs(self):
btc_pairs = []
for pair in self.pairs:
lhs, _ = pair.split("_")
if lhs == "BTC":
btc_pairs.append(pair)
return btc_pairs
def get_chart_data(self, pair, start=None, end=None, period=None):
'''
Returns candlestick chart data.
Args:
pair: Currency Pair
start: string specifying start time in the format "dd/mm/yyyy hh:mm:ss"
end: string specifying end time in the format "dd/mm/yyyy hh:mm:ss"
period: Candlestick period in seconds; valid values are 300, 900, 1800,
7200, 14400, 86400.
Returns:
Dictionary containing candlestick chart data
date: The UTC date for this candle in miliseconds since the Unix epoch.
high: The highest price for this asset within this candle.
low: The lowest price for this asset within this candle.
open: The price for this asset at the start of the candle.
close: The price for this asset at the end of the candle.
volume: The total amount of this asset transacted within this candle.
quoteVolume: The total amount of base currency transacted for this asset within this candle.
weightedAverage: The average price paid for this asset within this candle.
'''
if period is None:
raise ValueError(
"datamanager: argument 'period' should not be None")
start = utils.date2unix(start)
end = utils.date2unix(end)
data = self.polo.returnChartData(pair,
start=start,
end=end,
period=period)
return data
def get_trading_volume(self, pair, start=None, end=None, period=None):
'''
Returns volume and quote volume for currency pair in a given time interval for
the specified time period.
Args:
pair: Currency pair
start: string specifying start time in the format "date/month/year"
end: string specifying end time in the format "date/month/year"
period: Candlestick period in seconds; valud vlaues are 300, 900, 1800,
7200, 14400, 86400.
Returns:
vol: trade volume
qvol: quote volume
'''
start = utils.date2unix(start)
end = utils.date2unix(end)
data = self.polo.returnChartData(pair,
start=start,
end=end,
period=period)
vol = data[0]['volume']
qvol = data[0]['quoteVolume']
return vol, qvol
def get_monthly_trading_volume(self, pair, date=None):
'''
Get monthly trading volume for specified currency-pair starting from
date to one month in the past. If date is None, then evaluates assets based on
trading volume from TODAY to one month in the past.
Args:
pair: Currency pair
date: datetime object or string. If string, must be in the form
'dd/mm/yyyy hh:mm:ss'.
Returns:
tvol: Monthly volume for specified currency-pair
tqvol: Monthly quoted volume for specified currency-pair
'''
if isinstance(date, type(self._today)): # check if datetime object
end = date.strftime(self.dateformat)
prev_month = (date - datedelta.MONTH).strftime(self.dateformat)
elif isinstance(date, str): # check if string
end = date
datetimeobj = datetime.strptime(date,
self.dateformat) - datedelta.MONTH
prev_month = datetimeobj.strftime(
self.dateformat) # convert to string
elif date is None:
prev_month = self.prev_month # needs to be string
end = self.today # needs to be string
else:
raise ValueError('UNKNOWN ERROR in get_monthly_trading_volume()')
period24hr = 86400
data = self.get_chart_data(pair,
start=prev_month,
end=end,
period=period24hr)
tvol = sum([item['volume'] for item in data])
tqvol = sum([item['quoteVolume'] for item in data])
return tvol, tqvol
def save2json(self, filename, data):
'''
Save data to json file called 'filename'
Args:
filename: destination file in json format
data: dictionary of data
'''
with open(filename, 'w') as fp:
json.dump(data, fp)
print("Data saved to {}".format(filename))
def load_json(self, filename):
'''
Loads dictionary object from json file called 'filename'
Args:
filename:
Returns:
data: Dictionary object
'''
with open(filename) as json_file:
data = json.load(json_file)
return data
def save2pickle(self, filename, data):
'''
Pickles data to file called 'filename'
Args:
filename: destination file
data: AutoCastDict object created by poloniex. Can be any python object
'''
def _convert2dict(data):
'''
convert list of AutoCastDict object to list of dictionaries.
Args:
data: List of AutoCastDicts
'''
if isinstance(data[0], str): i = 1
else: i = 0
data = [dict(entry) for entry in data[i:]]
return data
data = _convert2dict(data)
with open(filename, 'wb') as fp:
pickle.dump(data, fp)
def load_pickle(self, filename):
'''
Load pickled data object from file called 'filename'.
Args:
filename: target file called 'filename'
Returns:
data: AutoCastDict object created by poloniex.
'''
with open(filename, 'rb') as fp:
data = pickle.load(fp)
return data
def import_data(self,
date,
pair,
ftype='.pickle',
datadir='./data',
ReturnData=False):
'''
Import data for the specified currency-pair and date from local data storage.
Args:
date: string or datetime object. If string, must be in the format 'dd/mm/yyyy 00:00:00'
pair: string specifying name of currency-pair to load
type: string specifying data file type. Either '.pickle' or '.json'
'''
if type(date) == type(date.today()):
date, _ = date.strftime(self.dateformat).split(' ')
date = date.replace('/', '_')
filename = pair + ftype
path = Path(datadir).joinpath(date, filename).resolve()
# print('Loading data from {}'.format(path))
data = self.load_pickle(path)
data.pop(0) # get rid of label at the start of data.
data[:] = [d for d in data if d.get('date') != 0]
return data
def import_data_btwn(self,
date1,
date2,
pair,
ftype='.pickle',
datadir='./data'):
'''
Imports data between the months specified between date1 and date2.
If dates are strings, should be in format 'dd/mm/yyyy hh:mm:ss'.
Args:
date1: preceding date. Either string or datetime object.
date2: later date. Either string or datetime object.
pair: currency-pair.
ftype: '.pickle' or '.json'
datadir: local data directory
'''
# deal with string type.
convert_to_datetime = lambda d: datetime.strptime(d, self.dateformat)
if isinstance(date1, str): date1 = convert_to_datetime(date1)
if isinstance(date2, str): date2 = convert_to_datetime(date2)
assert date1 < date2, ValueError("date1 has to precede date 2")
data = []
d = date1
one_month = datedelta.MONTH
while d < date2:
new_data = self.import_data(d, pair, ftype=ftype, datadir=datadir)
data = data[0:-1] + new_data
d += one_month
return data
class Ticker:
def __init__(self, ex_ev):
self.ex_ev = ex_ev
self.log = None
self.loop = asyncio.new_event_loop()
self.polo = Poloniex()
self.tickers = []
self.bulk_size = 100
async def eternity(self, timeout):
while not self.ex_ev.is_set():
await asyncio.sleep(timeout)
@staticmethod
async def calculate_points(charts):
def walk(data):
low = None
low_ts = None
high = None
high_ts = None
for char in data:
if not low or char.get('low') < low:
low = char.get('low')
low_ts = char.get('date')
if not high or char.get('high') > high:
high = char.get('high')
high_ts = char.get('date')
return low, low_ts, high, high_ts
return walk(charts[:30]), walk(charts[-20:])
async def get_all_coins(self):
response = self.polo.returnCurrencies()
if 'error' not in response:
for item in response:
try:
data = {
'id': response[item]['id'],
'short_name': item,
'full_name': response[item]['name']
}
coin = await get_first_by_filter(session=session,
model=Coin,
**data)
if not coin:
data.update({'fee': response[item]['txFee']})
await create(session=session, model=Coin, **data)
await asyncio.sleep(0.01)
except Exception as e:
self.log.error(f'Add coin {item} error: {repr(e)}')
async def set_coin_priority(self, ue_id):
ue = await get_first_by_filter(session=session,
model=UserExchange,
exchange_id=ue_id)
coins = await get_all(session=session, model=ExchangeCoin)
for coin in coins:
try:
coin_prior = await get_first_by_filter(
session=session,
model=UserMainCoinPriority,
main_coin_id=coin.id)
if not coin_prior:
data = {
'main_coin_id': coin.id,
'priority': 1,
'user_exchange_id': ue.id,
'is_active': True
}
await create(session=session,
model=UserMainCoinPriority,
**data)
except Exception as e:
self.log.error(f'Set coin priority Error: {e}')
await asyncio.sleep(0.01)
async def group_pull_exchanges(self, pair, item, exchange):
quote, base = pair.split('_')
coins = [quote, base]
for coin in coins:
try:
data = {'exchange_id': exchange.id, 'symbol': coin.lower()}
instance = await get_first_by_filter(session=session,
model=ExchangeCoin,
**data)
if not instance:
await create(session=session, model=ExchangeCoin, **data)
await asyncio.sleep(0.01)
except Exception as e:
self.log.error(f'Pull Exchange Error: {e}')
# !!! AFTER ADD ALL COINS
await asyncio.sleep(0.01)
try:
is_active = not bool(int(item.get('isFrozen', 0)))
main_coin = await get_first_by_filter(session=session,
model=ExchangeCoin,
exchange_id=exchange.id,
symbol=coins[0].lower())
second_coin = await get_first_by_filter(session=session,
model=ExchangeCoin,
exchange_id=exchange.id,
symbol=coins[1].lower())
if main_coin and second_coin:
old_pair = await get_first_by_filter(
session=session,
model=Pair,
main_coin_id=main_coin.id,
second_coin_id=second_coin.id)
if old_pair:
await update_model(session=session,
model=Pair,
pk=old_pair.id,
data={'is_active': is_active})
else:
data = {
'main_coin_id': main_coin.id,
'second_coin_id': second_coin.id,
'is_active': is_active
}
await create(session=session, model=Pair, **data)
except Exception as e:
self.log.error(f'Update/Create Pair Error: {e}')
await asyncio.sleep(0.01)
async def pull_exchanges(self):
while not self.ex_ev.is_set():
exchanges = await get_all(session=session, model=Exchanges)
if not exchanges:
await create(session=session,
model=Exchanges,
name='Poloniex',
info_frozen_key='')
exchanges = await get_all(session=session, model=Exchanges)
for exchange in exchanges:
try:
ue = await get_first_by_filter(session=session,
model=UserExchange,
exchange_id=exchange.id)
markets = self.polo.returnTicker()
self.log.info('Start filling pairs')
[
asyncio.ensure_future(self.group_pull_exchanges(
pair, item, exchange),
loop=self.loop)
for pair, item in markets.items()
]
if ue:
await self.set_coin_priority(ue.id)
except Exception as e:
self.log.error(f'Pull Exchanges Error: {e}')
if self.ex_ev.is_set():
break
try:
await asyncio.wait_for(self.eternity(10), timeout=3600.0)
except asyncio.TimeoutError:
...
self.log.info('=== Pull Exchanges(60s) Exited ===')
async def group_pull_exchange_tickers(self, item, value, exchange):
keys = ['high24hr', 'low24hr', 'highestBid', 'lowestAsk']
if not any(value.get(x) for x in keys):
return
pair_str = re.match(r'([a-zA-Z0-9]+)_([a-zA-Z0-9]+)', item)
if pair_str:
try:
main_coin = await get_first_by_filter(session=session,
model=ExchangeCoin,
symbol=pair_str.group(1))
second_coin = await get_first_by_filter(
session=session,
model=ExchangeCoin,
symbol=pair_str.group(2))
pair = None
if main_coin and second_coin:
pair = await get_first_by_filter(
session=session,
model=Pair,
main_coin_id=main_coin.id,
second_coin_id=second_coin.id)
if pair:
data = {
'exchange_id': exchange.id,
'pair_id': pair.id,
'high': value['high24hr'],
'low': value['low24hr'],
'bid': value['highestBid'],
'ask': value['lowestAsk'],
'base_volume': value['baseVolume'],
'last': value['last'],
'percent_change': value['percentChange'],
'date_time': datetime.now(timezone.utc),
}
self.tickers.append(ExchangeTicker(**data))
# DELETE OLD TICKERS
to_delete = session.query(ExchangeTicker).filter(
ExchangeTicker.date_time <= datetime.now(timezone.utc) -
dt.timedelta(days=3))
_ = to_delete.delete()
session.commit()
await asyncio.sleep(0.01)
except Exception as e:
self.log.error(f'Pair {pair_str}. Pull tickers error: {e}')
async def pull_exchanges_tickers(self):
while not self.ex_ev.is_set():
if len(self.tickers) >= self.bulk_size:
try:
await send_bulk(session, self.tickers)
self.tickers = []
except Exception as e:
self.log.error(f'Create Bulk Error: {e}')
exchanges = await get_all(session=session, model=Exchanges)
if not exchanges:
await create(session=session,
model=Exchanges,
name='Poloniex',
info_frozen_key='')
exchanges = await get_all(session=session, model=Exchanges)
for exchange in exchanges:
try:
tickers = self.polo.returnTicker()
if tickers:
[
asyncio.ensure_future(
self.group_pull_exchange_tickers(
item, value, exchange),
loop=self.loop)
for item, value in tickers.items()
]
except Exception as e:
self.log.exception(f'Ticker Error:{e}')
continue
if self.ex_ev.is_set():
break
try:
await asyncio.wait_for(self.eternity(10), timeout=60.0)
except asyncio.TimeoutError:
...
self.log.info('=== Pull Exchanges Tickers(60s) Exited ===')
async def group_pull_exchange_charts(self, pair, exchange_id):
main_coin = pair.main_coin.symbol.upper()
second_coin = pair.second_coin.symbol.upper()
chart_data = []
currency_pair = f'{main_coin}_{second_coin}'
try:
chart_data = self.polo.returnChartData(
currencyPair=currency_pair,
period=7200,
start=int(
datetime.timestamp(datetime.now().replace(
hour=0, minute=0, second=0, microsecond=0) -
timedelta(hours=2 * 50))),
end=int(time.time()))
except:
pass
if not chart_data:
return
for chart in chart_data:
try:
if not await get_first_by_filter(
session=session,
model=ExchangeCharts,
exchange_id=exchange_id,
pair_id=pair.id,
date_time=datetime.fromtimestamp(
chart.get('date'), timezone.utc)):
await create(session=session,
model=ExchangeCharts,
exchange_id=exchange_id,
pair_id=pair.id,
period=86400,
high=chart.get('high'),
low=chart.get('low'),
open=chart.get('open'),
close=chart.get('close'),
volume=chart.get('volume'),
quote_volume=chart.get('quoteVolume'),
weighted_average=chart.get('weightedAverage'),
date_time=datetime.fromtimestamp(
chart.get('date'), timezone.utc))
await asyncio.sleep(0.01)
except Exception as e:
self.log.exception(f'Bulk data error: {e}')
try:
first, last = await self.calculate_points(chart_data)
data = {
'point_low_first':
first[0],
'point_low_last':
last[0],
'point_high_first':
first[2],
'point_high_last':
last[2],
'point_low_first_date':
datetime.fromtimestamp(first[1], timezone.utc),
'point_low_last_date':
datetime.fromtimestamp(last[1], timezone.utc),
'point_high_first_date':
datetime.fromtimestamp(first[3], timezone.utc),
'point_high_last_date':
datetime.fromtimestamp(last[3], timezone.utc)
}
await update_model(session=session,
model=Pair,
pk=pair.id,
data=data)
except Exception as e:
self.log.exception(f'Calculation of Points error: {e}')
async def pull_exchanges_charts(self):
while not self.ex_ev.is_set():
if datetime.now().hour % 2 != 0:
try:
await asyncio.wait_for(self.eternity(10), timeout=3600.0)
except asyncio.TimeoutError:
...
continue
exchanges = await get_all(session=session, model=Exchanges)
if not exchanges:
await create(session=session,
model=Exchanges,
name='Poloniex',
info_frozen_key='')
exchanges = await get_all(session=session, model=Exchanges)
for exchange in exchanges:
try:
pairs = await get_all(session=session, model=Pair)
[
asyncio.ensure_future(self.group_pull_exchange_charts(
pair, exchange.id),
loop=self.loop) for pair in pairs
]
except Exception as e:
self.log.exception(f'Pull Chart Error:{e}')
continue
if self.ex_ev.is_set():
break
try:
await asyncio.wait_for(self.eternity(10), timeout=3600.0)
except asyncio.TimeoutError:
...
self.log.info('=== Pull Exchange Charts(3600s) Exited ===')
async def fill_user_orders(self, pair_str, data):
main_coin, second_coin = pair_str.split('_')
main_coin = await get_first_by_filter(session=session,
model=ExchangeCoin,
symbol=main_coin.lower())
second_coin = await get_first_by_filter(session=session,
model=ExchangeCoin,
symbol=second_coin.lower())
pair = await get_first_by_filter(session=session,
model=Pair,
main_coin_id=main_coin.id,
second_coin_id=second_coin.id)
exchange = await get_first_by_filter(session=session,
model=Exchanges,
name='poloniex')
ue = await get_first_by_filter(session=session,
model=UserExchange,
exchange_id=exchange.id)
for order_data in data:
try:
data = {
'order_number': order_data['orderNumber'],
'pair_id': pair.id,
}
order = await get_first_by_filter(session=session,
model=UserOrder,
**data)
main_coin_balance = await get_first_by_filter(
session=session,
model=UserBalance,
ue_id=ue.id,
coin=main_coin.symbol)
second_coin_balance = await get_first_by_filter(
session=session,
model=UserBalance,
ue_id=ue.id,
coin=second_coin.symbol)
if not order:
data = {
'ue_id': ue.id,
'pair_id': pair.id,
'order_type': order_data['type'],
'order_number': order_data['orderNumber'],
'price': order_data['rate'],
'amount': order_data['amount'],
'total': order_data['total'],
'date_created': order_data['date'],
'interim_main_coin': 0,
'main_coin_before_total': main_coin_balance.total,
'main_coin_before_free': main_coin_balance.free,
'main_coin_before_used': main_coin_balance.used,
'second_coin_before_total': second_coin_balance.total,
'second_coin_before_free': second_coin_balance.free,
'second_coin_before_used': second_coin_balance.used,
'date_updated': order_data['date'],
'fee': order_data['fee'],
'cancel_desc': 'Worked'
}
await create(session=session, model=UserOrder, **data)
await asyncio.sleep(0.01)
except Exception as e:
self.log.error(f'Add order {order_data} error: {repr(e)}')
async def pull_trade_history(self):
try:
start = datetime.now(timezone.utc) - dt.timedelta(days=DAY_HISTORY)
end = time.time()
history = Poloniex(POLONIEX_API_KEY,
POLONIEX_API_SECRET).returnTradeHistory(
start=start.timestamp(), end=end)
[
asyncio.ensure_future(self.fill_user_orders(pair, data),
loop=self.loop)
for pair, data in history.items()
]
except Exception as e:
self.log.error(f'pull_trade_history Error: {e}')
def run(self):
self.log = get_logger('ticker')
self.log.info('Start Ticker')
asyncio.set_event_loop(self.loop)
tasks = [
self.get_all_coins(),
self.pull_trade_history(),
self.pull_exchanges(),
self.pull_exchanges_tickers(),
self.pull_exchanges_charts()
]
group = asyncio.gather(*tasks)
self.loop.run_until_complete(group)
if self.ex_ev.is_set():
self.log.info('=======TICKER RUN EXIT EVENT=============')
self.loop.stop()
self.loop.close()