def refresh(self): coinmarketcap = Market() dict = coinmarketcap.stats() rows = [] for key, value in dict.items(): rows.append((key, value, 0)) data = pd.DataFrame(data=rows, columns=['key', 'value', 'index']) self.__dataframe = data.pivot(columns='key', values='value', index='index') self.__dataframe['insertion_time'] = self. __insertion_time
def display_Charts(): epd = epd2in7.EPD() epd.init() #epd.rotate(3) # Rotate the Display by 270 degree from coinmarketcap import Market coinmarketcap = Market() CurrencyData = coinmarketcap.ticker(start=0, limit=5, convert='EUR') LINEHEIGHT = 20 price = 0 currency_symbol = '€' # For simplicity, the arguments are explicit numerical coordinates image = Image.new('1', (epd2in7.EPD_WIDTH, epd2in7.EPD_HEIGHT), 255) # 255: clear the image with white draw = ImageDraw.Draw(image) font = ImageFont.truetype( '/usr/share/fonts/truetype/freefont/FreeMonoBold.ttf', 18) font = ImageFont.truetype( '/usr/share/fonts/truetype/freefont/FreeMonoBold.ttf', 12) font_table_title = ImageFont.truetype('fonts/Grand9KPixel.ttf', 10) font_titles = ImageFont.truetype('fonts/slkscr.ttf', 16) font_num = ImageFont.truetype('fonts/PixelSplitter-Bold.ttf', 14) font_date = ImageFont.truetype('fonts/PixelSplitter-Bold.ttf', 12) draw.rectangle((0, 0, 264, 28), fill=0) draw.rectangle((0, 160, 264, 176), fill=0) draw.rectangle((0, 28, 264, 50), fill=0) draw.text((2, 8), " Cryptocurrency Market ", font=font_titles, fill=255) draw.text((5, 32), "NAME PRICE CHANGE(24h)", font=font_table_title, fill=255) for item in CurrencyData: #price=round(float(),4) draw.text((5, 40 + LINEHEIGHT), item['symbol'], font=font_titles, fill=0) draw.text((60, 40 + LINEHEIGHT), "€" + item['price_eur'], font=font_num, fill=0) draw.text((200, 40 + LINEHEIGHT), item['percent_change_24h'] + "%", font=font_num, fill=0) LINEHEIGHT += 20 draw.text((0, 160), str(" UPDATED ON:" + time.strftime("%c")), font=font_date, fill=255) epd.display_frame(epd.get_frame_buffer(image))
def __init__(self, n_streams, master_lock, mode, keychain, loki): self.nP = n_streams self.mode = mode self.totalScore = 0 self.lock = master_lock self.oracle = Oracle(keychain, loki) self.json_db_filename = json_db self.SQL_db_filename = sql_db self.nbatches = 0 self.market = Market() self.initResources()
def get_btc_change(): coinmarketcap = Market() try: market_data = coinmarketcap.ticker(start=0, limit=3, convert='EUR') # pprint.pprint(market_data) daily_percentage_change = market_data['data']['1']['quotes']['USD'][ 'percent_change_24h'] return daily_percentage_change except: print('Problem with func get_btc_change()') raise
def get_btc_usd(): try: coinmarketcap = Market() except Exception as e: print "Exception at coinmarketcap: " + str(e) raise Exception("Retry!") try: btc_usd = (json.loads(coinmarketcap.ticker("bitcoin")))[0]["price_usd"] return btc_usd except Exception as e: print "Exception while parsing coinmarketcap Json: " + str(e)
def run(): coinmarketcap = Market() l = 1 jsonpack = coinmarketcap.ticker(limit=l, convert='CNY') for k in jsonpack: price = k['name'] + u'价格' + k['price_usd'] + u'涨跌' + k[ 'percent_change_24h'] + '%' speak(price)
def main(arguments): parser = argparse.ArgumentParser() parser.add_argument('currency', help="curency") parser.add_argument('key', help="Выводимый ключ") parser.add_argument('convert', help="convert to", default='USD') args = parser.parse_args(arguments) print(args.currency) coinmarketcap = Market() print(coinmarketcap.ticker(args.currency, limit=1, convert=args.currency)) print( coinmarketcap.ticker(args.currency, limit=1, convert=args.currency)[0].get(args.key))
def bitcoin_usd(): try: coinmarketcap = Market() except: return 'Неполадки на сервере coinmarketcap' get_m = coinmarketcap.ticker(limit=900, convert='USD') for i in get_m: if i['symbol'] == "BTC": return i['price_usd'] else: continue return 'Такой команды или валюты нету'
def eth_usd(): try: coinmarketcap = Market() except: return 'Неполадки на сервере coinmarketcap' get_m = coinmarketcap.ticker(limit=900, convert='USD') for i in get_m: if i['symbol'] == "ETH": return "ETH: " + i['price_usd'] + " USD. " + i['price_btc'] + " BTC" else: continue return 'Такой команды или валюты нету'
def __init__(self, token, superadminid, adminsid=[], exchangers=[], updateInterval=300, debuglevel=0): super(CryptoBot, self).__init__(token, superadminid, adminsid, debuglevel) self._debugLevel = debuglevel if (self._debugLevel > 0): print "Debug Level: " + str(debuglevel) self.__exchangerList = exchangers self.__updateInterval = updateInterval if (self._debugLevel >= 1): print "Update Interval: " + str(updateInterval) self.__coinmarketcap = Market() for currency in self.__supportedCurrencies: if (self._debugLevel >= 1): print "Supported Currency: " + currency self.__lastUpdatedStats[currency] = datetime.datetime.min self.__lastUpdatedTickers[currency] = datetime.datetime.min self.__getStats(currency) self.__getTickers(currency) if (self._debugLevel >= 1): print "Adding Handlers" self._addCommandHandler('currency', self.__setCurrency, pass_args=True) self._addCommandHandler('fiatPrice', self.__getPriceFiat, pass_args=True) self._addCommandHandler('price', self.__getPriceCoin, pass_args=True) self._addCommandHandler('rank', self.__getRank, pass_args=True) self._addCommandHandler('marketcap', self.__getMarketCap, pass_args=True) self._addCommandHandler('volume', self.__getVolume, pass_args=True) self._addCommandHandler('change1h', self.__getChangeLastHour, pass_args=True) self._addCommandHandler('change24h', self.__getChangeLastDay, pass_args=True) self._addCommandHandler('change7d', self.__getChangeLastSevenDays, pass_args=True) self._addCommandHandler('coinInfo', self.__getInfoCoin, pass_args=True) self._removeHandler(self._unknownHandler) self._addHandler(self._unknownHandler) return
def __init__(self, exchanges): '''exchanges = comma separated list of exchanges you want to monitor''' self.exchanges = [e.lower() for e in exchanges] # Depending on which exchanges got included, connect to APIs if 'kraken' in self.exchanges: self.kraken_api = krakenex.API() self.kraken = KrakenAPI(self.kraken_api) if 'gdax' in self.exchanges: self.gdax = gdax_api.PublicClient() if 'bittrex' in self.exchanges: self.bittrex = Bittrex(None, None) if 'coinmarketcap' in self.exchanges: self.coinmarketcap = Market()
def make_money(sc): coinmarketcap = Market() current_btc_price = coinmarketcap.ticker( start=0, limit=1)['data']['1']['quotes']['USD']['price'] price_window.append(current_btc_price) if len(price_window) > 1000: del price_window[0] if len(price_window) == 1000: RSI = calculateRSI(price_window) current_run = list(zip(RSI, price_window)) print(current_run) s.enter(300, 1, make_money, (sc, )) print(price_window) print(len(price_window))
def setup(config): print('Fetching cryptocurrency tokens...', end='') total_coins = 0 global currency_mapping coin_market = Market() try: data = coin_market.ticker('', limit=0) for coin in data: currency_mapping[coin["symbol"].upper()] = coin["id"] total_coins += 1 print(' %d tokens added!' % (total_coins)) except Exception as err: print(' error fetching tokens!') print('%s\n' % err)
def __init__( self, n_coins=20, cap=0.1, mcap_limit=50, min_trade_limit=0.001): self.BITTREX_API_KEY = os.getenv('BITTREX_API_KEY') self.BITTREX_API_SECRET = os.getenv('BITTREX_API_SECRET') self.coinmarketcap = Market() self.v1_bittrex = Bittrex('', '', api_version=API_V1_1) self.v2_bittrex = Bittrex( self.BITTREX_API_KEY, self.BITTREX_API_SECRET, api_version=API_V2_0) self.blacklist = pd.Series(['USDT', 'BCC']) self.n_coins = n_coins self.cap = cap self.mcap_limit = mcap_limit self.min_trade_limit = min_trade_limit
def __init__(self, config, subreddits): self.__coinmarketcap = Market() tickers = self.__coinmarketcap.ticker() for ticker in tickers: self.__currencies[ ticker['symbol'].lower()] = ticker['name'].lower() self.__words.append(ticker['symbol'].lower()) self.__words.append(ticker['name'].lower()) print self.__words super(CryptoCountBot, self).__init__(config, subreddits, self.__words) return
def large_cap(): ''' return: Dictionary of Coin:market_cap proportion ''' #instantiate coinmarketcap object cmc = Market() market_df = pd.DataFrame(cmc.ticker()) #filter out coinse where rank above cieling market_df['rank'] = market_df['rank'].astype(int) #now weight based on market cap market_df['market_cap_usd'] = market_df['market_cap_usd'].astype(float) market_df['index_weight'] = market_df['market_cap_usd']/market_df['market_cap_usd'].sum() #create dictionary of market and portfolio weights weight_dict = dict(zip(market_df.symbol,market_df.index_weight)) return weight_dict
def display_Charts(): epd = epd2in7b.EPD() epd.init() epd.set_rotate(3) # Rotate the Display by 270 degree from coinmarketcap import Market coinmarketcap = Market() CurrencyData = coinmarketcap.ticker(start=0, limit=5, convert='EUR') # clear the frame buffer frame_black = [0] * (epd.width * epd.height / 8) frame_red = [0] * (epd.width * epd.height / 8) LINEHEIGHT = 20 price = 0 currency_symbol = '€' # For simplicity, the arguments are explicit numerical coordinates epd.draw_filled_rectangle(frame_black, 0, 0, 264, 28, COLORED) epd.draw_filled_rectangle(frame_black, 0, 160, 264, 176, COLORED) epd.draw_filled_rectangle(frame_red, 0, 28, 264, 50, COLORED) # draw strings to the buffer font = ImageFont.truetype( '/usr/share/fonts/truetype/freefont/FreeMonoBold.ttf', 12) font_table_title = ImageFont.truetype('fonts/Grand9KPixel.ttf', 10) font_titles = ImageFont.truetype('fonts/slkscr.ttf', 16) font_num = ImageFont.truetype('fonts/PixelSplitter-Bold.ttf', 14) font_date = ImageFont.truetype('fonts/PixelSplitter-Bold.ttf', 12) epd.draw_string_at(frame_black, 2, 8, " Cryptocurrency Market ", font_titles, UNCOLORED) epd.draw_string_at( frame_red, 5, 32, "NAME PRICE CHANGE(24h)", font_table_title, UNCOLORED) print(CurrencyData[0]['id']) for item in CurrencyData: #price=round(float(),4) epd.draw_string_at(frame_black, 5, 40 + LINEHEIGHT, item['symbol'], font_titles, COLORED) epd.draw_string_at(frame_black, 60, 40 + LINEHEIGHT, "€" + item['price_eur'], font_num, COLORED) epd.draw_string_at(frame_black, 200, 40 + LINEHEIGHT, item['percent_change_24h'] + "%", font_num, COLORED) LINEHEIGHT += 20 epd.draw_string_at(frame_black, 0, 160, str(" UPDATED ON:" + time.strftime("%c")), font_date, UNCOLORED) # display the frames epd.display_frame(frame_black, frame_red)
def get_market_caps(top=100): coins = Market().ticker(start=0, limit=top)["data"].values() caps = [{ "symbol": coin["symbol"], "value": coin["quotes"]["USD"]["market_cap"], } for coin in coins] return caps
class Coins(object): def __init__(self): self.coinmarketcap = Market() self.current_reference = 'BTC' def _parseresponse(self, response): result = [] for coin in response['data'].keys(): res_coin = {} for field, val in response['data'][coin].items(): if isinstance(val, dict): for currency in response['data'][coin][field].keys(): res_coin.update({ x + '_' + currency: y for x, y in response['data'][coin][field] [currency].items() }) else: res_coin[field] = val result.append(res_coin) return result def get_coins_summary(self, start=0, limit=100): raw_data = self.coinmarketcap.ticker(start=start, limit=limit, convert=self.current_reference) data = pd.DataFrame(self._parseresponse(raw_data)) data['DATETIME'] = pd.to_datetime('today').strftime("%Y-%m-%d") return data
class Statistics: def __init__(self, default): self.market = Market() self.default = default def get_best_asset(self, period, ticker_range): best_asset_symbol = self.default best_asset_percentage = 0 response = self.market.ticker(limit=100, convert='USD') ticker_list_length = len(response) for i in range(0, ticker_list_length): item = response[i] if item["symbol"] in str(ticker_range): if period == PeriodRange.HOUR: if float(item["percent_change_1h"]) > float(best_asset_percentage): best_asset_symbol = item["symbol"] best_asset_percentage = item["percent_change_1h"] elif period == PeriodRange.DAY: if float(item["percent_change_24h"]) > float(best_asset_percentage): best_asset_symbol = item["symbol"] best_asset_percentage = item["percent_change_24h"] elif period == PeriodRange.WEEK: if float(item["percent_change_7d"]) > float(best_asset_percentage): best_asset_symbol = item["symbol"] best_asset_percentage = item["percent_change_7d"] return best_asset_symbol
def get_currency_stats(cur_list): req = urllib.request.Request("https://api.coinmarketcap.com/v1/ticker/") opener = urllib.request.build_opener() f = opener.open(req) jso = json.loads(f.read().decode('utf-8')) total_bal = [] for ci in cur_list: for cp in jso: if cp['symbol'] == ci['currency']: price_usd = float(cp['price_usd'].encode('utf-8')) cur_bal = float(ci['Balance']) usd_balance = price_usd * cur_bal total_bal.append(usd_balance) print(cp['name']+ ' $'+ str(usd_balance)) coin_market = CMC.ticker(cp['id'])[0] print('Rank '+ coin_market['rank'],end=' | ') print('1HR %'+ coin_market['percent_change_1h'],end=' | ') print('24HR % '+ coin_market['percent_change_24h'],end=' | ') try: print('7D %'+ coin_market['percent_change_7d']) except: print('No Data') print("="*60) total_bal = reduce(lambda x, y: x+y, total_bal) total_bal = 'Balance $'+str(total_bal) return total_bal
class Strategy: def __init__(self, exchange=None, **kwargs): print('Strategy initializing...') self.exchange = exchange if exchange else DEFAULT_EXCHANGE self.type = ROLLBACK self.purchase_different_coins = 4 # 3-10 recommended self.drop_range_to_buy_pct = range(-40, -10) self.deposit_threshold_pct = 50 self.capitalization_threshold_usd = float(20000000.0) # > 30 billions recommended self.market_volume_threshold_usd_24h = 500000 self.your_margin_pct = 5 self.market = Market() self.drops = None self.suitable_coins_marketcap = {} self.suitable_tickers = [] self.coins_listing = {item['symbol']: {'id': item['id'], 'name': item['name']} for item in self.market.listings()['data']} self.crypto_only = False # include tokens, ico etc. self.currencies = None self.profit_policy = HALF_REINVEST for key in kwargs: if hasattr(self, key): setattr(self, key, kwargs[key]) def fetch_currencies(self): curs = self.exchange.fetch_currencies() if self.crypto_only: curs = dict((k + '/' + BASE_TICKER, v) for k, v in curs.items() if v['type'] == 'crypto') else: curs = dict((k + '/' + BASE_TICKER, v) for k, v in curs.items()) self.currencies = curs def fetch_suitable_coins(self): self.fetch_currencies() try: tickers = self.exchange.fetch_tickers() except: tickers = {} self.drops = {pair: data for pair, data in tickers.items() if pair.endswith(BASE_TICKER) and data['percentage'] is not None and int(data['percentage']) in self.drop_range_to_buy_pct and pair in self.currencies} if self.market: for pair, market_data in self.drops.items(): ticker = pair.split('/')[0] if ticker in self.coins_listing: market_cap_id = self.coins_listing[ticker]['id'] if bool(market_cap_id): market_data = self.market.ticker(market_cap_id, convert=USD)['data']['quotes'][USD] capital = market_data['market_cap'] if isinstance(capital, float): if capital >= self.capitalization_threshold_usd: print(ticker, self.coins_listing[ticker]['name'], capital) self.suitable_coins_marketcap[ticker] = capital else: print('Capitalization is unknown: {}... pass'.format(ticker)) if len(self.suitable_coins_marketcap) > 0: scm = self.suitable_coins_marketcap self.suitable_tickers = sorted(scm, key=scm.get, reverse=True)[:self.purchase_different_coins]
def __init__(self, name, buy_price, quantity, buy_currency='USD'): ''' INITIALIZE: name (str): name of the coin bought buy_price (float) - the price per coin in any currency quantity(float) - the quantity bought buy_currency(STR) - the currency used to buy it ''' market = Market() self.name = name self.buy_price = buy_price self.buy_currency = buy_currency self.quantity = quantity self.online_result = market.ticker(self.name)[0] self.now_price = self._get_price() self.status_usd = self.now_price * self.quantity self.start_price_usd = self._convert_usd(self.buy_currency)
def __new__(cls): if CryptoToFiatConverter.__instance is None: CryptoToFiatConverter.__instance = object.__new__(cls) try: CryptoToFiatConverter._coinmarketcap = Market() except BaseException: CryptoToFiatConverter._coinmarketcap = None return CryptoToFiatConverter.__instance
def updatecryptos(): t0 = time.time() coinmarketcap = Market() cryptocoins = coinmarketcap.ticker(start=0, limit=0) Allcryptos = {} cryptobets = {} for x in range(0, len(cryptocoins) - 1): name = cryptocoins[x] Allcryptos[name['symbol']] = cryptocoins[x] cryptobets[name['symbol']] = 0 print('All coins have been updated') t1 = time.time() total = t1 - t0 print(total) return Allcryptos
async def crypto(ctx, *symbol): """Get info about crypto currencies. '!crypto btc' to get info about one specific currency.""" coins = Market() listings = coins.ticker(start=0, limit=10, convert='GBP') t = listings["data"] coinOutputs = [] for x in t: coin = listings["data"][x] output = "" output += coin["name"] output += " (" output += coin["symbol"] output += ") £" output += str(round(coin["quotes"]["GBP"]["price"], 3)) output += "\t1hr % " hr = coin["quotes"]["GBP"]["percent_change_1h"] output += str(hr) output += cryptoChange(hr) output += "\t24hr % " hr = coin["quotes"]["GBP"]["percent_change_24h"] output += str(hr) output += cryptoChange(hr) output += "\t7d % " hr = coin["quotes"]["GBP"]["percent_change_7d"] output += str(hr) output += cryptoChange(hr) output += "\n" coinOutputs.append(output) output = "" for x in coinOutputs: output += x if symbol: for x in coinOutputs: if x.find(str(symbol[0].upper())) >= 0: output = x await BOT.say(output)
def chooseCurrency(bot, update): coinmarketcap = Market() array = coinmarketcap.ticker(limit=6) kb = [] i = 0 for fila in range(2): kb.append([]) for col in range(3): kb[fila].append(KeyboardButton(text=array[i]["symbol"])) i += 1 bot.sendMessage( update.message.chat.id, "Choose or type the desired cryptocurrency symbol:", reply_markup=ReplyKeyboardMarkup( keyboard=kb, one_time_keyboard=True ) )
def get_coin_rate(coin_id): """ Get Crypto Coin USD rate from CoinMarketCap by coin ID :param string coin_id: CoinMarketCap coin ID like 'bitcoin' for BTC :return: crypto coin USD rate """ market = Market() data = market.ticker(coin_id) if len(data) > 0: data = data[0] result = "Unavailable" if 'price_usd' in data: result = data['price_usd'] return result
def get_weights(): coinmarketcap = Market() market_cap = {} for c in coins: ticker = coinmarketcap.ticker(currency=coins[c]) market_cap[c] = float(ticker[0]['market_cap_usd']) total_market_cap = sum(market_cap.values()) weights = {} for c in coins: weights[c] = market_cap[c] / total_market_cap print('coin weights:') for w in weights: print(' {0}: {1:.4f}'.format(w, weights[w])) print() return weights
class CryptoCountBot(CountBot): __currencies = {} __words = [] __coinMarketCap = None def __init__(self, config, subreddits): self.__coinmarketcap = Market() tickers = self.__coinmarketcap.ticker() for ticker in tickers: self.__currencies[ ticker['symbol'].lower()] = ticker['name'].lower() self.__words.append(ticker['symbol'].lower()) self.__words.append(ticker['name'].lower()) print self.__words super(CryptoCountBot, self).__init__(config, subreddits, self.__words) return def print_cryptos_by_popularity(self, mentiontype=MentionType.POST, timeinterval=TimeInterval.DAY): resultdict = self.run_count_post_comment(mentiontype=mentiontype, timeinterval=timeinterval) postdict = resultdict[MentionType.POST] commentdict = resultdict[MentionType.COMMENT] cryptoresults = {} for word in postdict: if (word in self.__currencies): cryptoresults[self.__currencies[word]] = cryptoresults.get( self.__currencies[word], 0) + postdict[word] else: cryptoresults[word] = cryptoresults.get(word, 0) + postdict[word] for word in commentdict: if (word in self.__currencies): cryptoresults[self.__currencies[word]] = cryptoresults.get( self.__currencies[word], 0) + commentdict[word] else: cryptoresults[word] = cryptoresults.get(word, 0) + commentdict[word] sortedcryptos = sorted(cryptoresults.items(), key=operator.itemgetter(1), reverse=True) for crypto, result in sortedcryptos: print(crypto + ": " + str(result)) return
import xml.etree.ElementTree as ET from decimal import * import collections from sopel.module import commands, example, NOLIMIT from sortedcontainers import SortedDict from forex_python.converter import CurrencyCodes from forex_python.converter import CurrencyRates from forex_python.bitcoin import BtcConverter from pymarketcap import Pymarketcap from coinmarketcap import Market import coinmarketcap c = CurrencyRates() b=BtcConverter() m=Market() cc=CurrencyCodes() allcoins={} curlist=list(c.get_rates("USD").keys()) #print(c.get_rates("USD")) curlist.append("USD") curlist.sort() for cur in curlist: allcoins[cur]=cc.get_currency_name(cur) altcoinmap={} #print (coinmarketcap.price("BTC")) json=m.ticker(convert='EUR') #print(json) for currency in json: altcoinmap[currency["symbol"]]=currency["id"] allcoins[currency["symbol"]]=currency["name"] #print(altcoinmap) #print(json)
import json import os from datetime import datetime from coinmarketcap import Market from apscheduler.schedulers.blocking import BlockingScheduler coin_market = Market() json_data = coin_market.ticker().decode('utf-8') data = json.loads(json_data) path = 'Data' poloniex_symbols = ['ETH', 'XRP', 'ETC', 'GNT', 'STR', 'DOGE', 'XEM', 'LTC', 'DGB', 'XMR', 'ZEC', 'SC', 'BCN', 'DASH', 'BTS', 'BTM', 'FCT', 'STEEM', 'STRAT', 'REP', 'LSK', 'SYS', 'NXT', 'MAID', 'GAME', 'ARDR', 'RIC', 'GNO', 'DCR', 'AMP', 'VTC', 'CLAM', 'BURST', 'BCY', 'LBC', 'SJCX', 'NAV', 'PINK', 'BTCD', 'EXP', 'PPC', 'XCP', 'NEOS', 'NAUT', 'POT', 'PASC', 'BELA', 'EMC2', 'BLK', 'FLO', 'FLDC', 'NMC', 'GRC', 'XVC', 'XPM', 'OMNI', 'SBD', 'XBC', 'NXC', 'VRC', 'HUC', 'NOTE' ] def increasing(): for coin in data: if coin['percent_change_1h'] is not None: #check for volume over $1 million dollars if float(coin['24h_volume_usd']) > 1000000: percent_change_1h = float(coin['percent_change_1h']) if percent_change_1h > 10: print (coin['name'])
if __name__ == '__main__': # start date current_date = datetime.now().replace(second=0, microsecond=0) # config loading config = configparser.ConfigParser() config.read(os.path.join(os.path.expanduser('~'), '.config', 'me.ini')) # prepare database connection dburl = config['db']['url'] engine = create_engine(dburl) # get coin market cap data coinmarketcap = Market() # Since data is falling quite a bit, temporarly check if we got # it, and if not, increase the limit of records returned. contains_data = False limit = 125 while not contains_data: markets = coinmarketcap.ticker(limit=limit, convert='EUR') mktdf = pd.DataFrame(markets) if len(mktdf[mktdf.symbol == 'DATA']) > 0: contains_data = True limit += 10 mktdf = pd.DataFrame(markets)
class Cerberus(object): def __init__(self, n_streams, master_lock, mode, keychain, loki): self.nP = n_streams self.mode = mode self.totalScore = 0 self.lock = master_lock self.oracle = Oracle(keychain, loki) self.json_db_filename = json_db self.SQL_db_filename = sql_db self.nbatches = 0 self.market = Market() self.initResources() def initResources(self): self.stacks = [Queue() for _ in range(self.nP)] if self.mode == 'mongo': self.client = MongoClient() self.db = self.client[mongo_db] if self.mode == 'SQL': self.db = self.SQL_db_filename self.conn = _sqlite3.connect(self.SQL_db_filename) self.curs = self.conn.cursor() self.curs.execute("CREATE TABLE IF NOT EXISTS tweets (tid integer, username text, cat text, content text, coordinates text, sentiment text, source text)") if self.mode == 'JSON': self.db = self.json_db_filename def db_count(self, table): r_query = "SELECT Count() FROM %s" % table self.curs.execute(r_query) n_rows = self.curs.fetchone()[0] return n_rows def handleNewTweet(self, pID, pDesc, tweet): self.lock.acquire() try: self.stacks[int(pID)].put(tweet) except Exception as e: print 'caught', e, 'while handling new entry' finally: self.lock.release() def printScore(self, batchc): if self.mode == 'mongo': name = self.mode + '.' + self.db.name dbc = sum([self.db[cname].count() for cname in self.db.collection_names()]) else: name = self.db dbc = self.db_count('tweets') batch_number = 'Batch ' + str(self.nbatches) announcement = ' Wrote ' + str(sum(batchc)) + \ ' entities to ' + str(name) + ' for a total of ' + str(dbc) + ' ' statistics = str(batchc) l = len(announcement) t1 = int((l - len(batch_number))/2) t2 = int((l - len(statistics))/2) print '' print t1*'-', batch_number, (l-len(batch_number)-t1)*'-' print announcement print t2*'-', statistics, (l-len(statistics)-t2)*'-' def readFromJSON(self): json_file = open(self.json_db_filename,'r') json_data = json.load(json_file) json_file.close() return json_data, len(json_data) def writeToJSON(self, json_data): json_file = open(self.json_db_filename,'w') json_file.write(json.dumps(json_data)) json_file.close() return len(json_data) def executeBatch(self, dbs): self.lock.acquire() self.dbs = dbs self.nbatches += 1 buffers = [] batch_cnt = [] try: for i, stack in enumerate(self.stacks): buf = [] while not stack.empty(): tweet = stack.get(timeout = 3) buf.append(tweet) buffers.append(buf) batch_cnt.append(len(buf)) if len(buffers): if self.mode == 'JSON': json_data = [] try: json_data, u = self.readFromJSON() except Exception as e: print e, 'exception caught while writing to JSON' for buf in buffers: for tweet in buf: json_data.append({'user': tweet.userID, 'tweet': tweet.tweetID, 'text': tweet.text, 'created_at': str(tweet.createdAt), 'location': tweet.location, 'source': tweet.device}) self.writeToJSON(json_data) score = len(buf) self.totalScore += score self.printScore(score, self.totalScore) if self.mode == 'SQL': for buf in buffers: for tweet in buf: self.curs.execute("insert into tweets (tid, username, content, cat, /" "coordinates, sentiment, source) values(?, ?, ?, ?, ?, ?, ?)", (tweet.tweetID, tweet.userID, tweet.text, str(tweet.createdAt), tweet.location, tweet.sentiment, tweet.device)) self.conn.commit() self.printScore(batch_cnt) if self.mode == 'mongo': score = 0 for i, buf in enumerate(buffers): col = self.dbs[i] objs = [] for obj in buf: cat = date_convert(obj.json['created_at']) objs.append({'created': cat, 'payload':obj.json}) try: if len(objs): self.db[col].insert_many(objs) except Exception as e: print(e) raise try: price = {'created': datetime.now() - timedelta(hours = 3), 'payload':self.market.ticker(col)[0]} self.db[col+'_price'].insert_one(price) except Exception as e: print(e) raise self.printScore(batch_cnt) except Exception as e: print type(e).__name__,'exception caught while writing to database' finally: self.lock.release() return batch_cnt def reboot(self): del self.stacks self.executeBatch() self.initResources()