def conapi_data(api, symbol_id, period, start_data): data = DataFrame( api.ohlcv_historical_data('{}'.format(symbol_id), { 'period_id': period, 'time_start': start_data })) to_pickle(data, 'coinapi', 'symbols', '{}_{}'.format(symbol_id, period))
def save_one(symbol): client = c() data = client.get_dataframe(symbol, startDate='1980-01-01') if data is not None: to_pickle(data, 'tiingo', '{}'.format(symbol)) print(colored.green('Got it.')) else: print(colored.red('Nothing for this symbols.'))
def run_quandl(check_latest=True): for s in QUANDL_SYMBOLS: data = get(s[0]) name = s[0].replace('/', '_') if check_latest: if s[2]: ensure_latest(df=data) to_pickle(data, 'futures', name) print(colored.green(name))
def generate_crypto_timeframes(exchange, name): try: tfs = exchange.timeframes.keys() df = DataFrame([k for k in tfs]) to_pickle(df, join('ccxt', 'tfs'), name) print(colored.green(name)) except AttributeError: pass except HTTPError: pass except Exception as err: print(colored.red(err))
def run_tiingo(i=0): for s in Market.select()[i:]: try: data = get_data(s=s) if data is not None: to_pickle(data, 'tiingo', '{}'.format(s.symbol)) i += 1 except FileNotFoundError as err: print(colored.red('Retrying due to disk error: {}'.format(err))) run_tiingo(i=i) except KeyError as err: print(colored.red(err))
def cboe_download(check_latest=True): for sym in CBOE_DATA: url = 'http://www.cboe.com/publish/scheduledtask/mktdata/datahouse/%s.csv' % sym[ 1] content = get(url).content data = read_csv(StringIO(content.decode('utf-8')), skiprows=4, names=sym[2], parse_dates=True) data.index = to_datetime(data['Date'], errors='coerce') del data['Date'] if check_latest: ensure_latest(df=data) to_pickle(data, 'indicators', sym[0]) print(colored.green(sym[0]))
def generate_crypto_symbols(exchange, name): try: syms = exchange.load_markets() out = [key for key in syms] df = DataFrame(out) to_pickle(df, join('ccxt', 'symbols'), name) print(colored.green(name)) for i in range(len(out)): df = DataFrame(syms[out[i]]) to_pickle(df, join('ccxt', 'conditions'), '{}_{}'.format(name, out[i].replace('/', '_'))) print(colored.green(out[i])) except HTTPError: pass except Exception as err: print(colored.red(err))
def download_all_crypto(first=False): for e in EXCHANGES: if first: generate_crypto_timeframes(exchange=e[1], name=e[0]) generate_crypto_symbols(exchange=e[1], name=e[0]) syms = get_syms(e=e) tf = get_tfs(e=e) if (tf is not None) & (syms is not None): for s in syms: for t in tf: data = fetch_ohlc(exchange=e[1], symbol=s, timeframe=t) if data is not None: to_pickle(data, 'ccxt', '{}_{}_{}'.format(e[0], s.replace('/', '_'), t)) print(colored.green('%s %s %s' % (e[0], s, t)))
def download_eurex(): source = 'http://www.stoxx.com/download/historical_values/' es_url = source + 'hbrbcpe.txt' vs_url = source + 'h_vstoxx.txt' es_path = join(STORAGE_PATH, 'eurex', 'es.txt') vs_path = join(STORAGE_PATH, 'eurex', 'vs.txt') urlretrieve(es_url, es_path) # ES5 urlretrieve(vs_url, vs_path) # VSTOXX columns = ['Date', 'SX5P', 'SX5E', 'SXXP', 'SXXE', 'SXXF', 'SXXA', 'DK5F', 'DKXF', 'DEL'] es = read_csv(es_path, index_col=0, parse_dates=True, dayfirst=True, header=None, skiprows=4, names=columns, sep=';') del es['DEL'] vs = read_csv(vs_path, index_col=0, parse_dates=True, dayfirst=True, header=2) to_pickle(es, 'eurex', 'esx') to_pickle(vs, 'eurex', 'vsx')
def get_all_statements(sym): stypes = ['pl', 'bs', 'cf'] ptypes = ['Q1', 'Q2', 'Q3', 'Q4', 'TTM', 'FY'] years = range(2007, datetime.now().year) id = get_id(sym=sym) if id is not None: for stype in stypes: print(stype) try: for year in years: print(year) for ptype in ptypes: print(ptype) df = get_statements(id=id, stype=stype, ptype=ptype, fyear=year) if df is not None: name = construct_name(sym=sym, stype=stype, ptype=ptype, fyear=year) to_pickle(df, 'fundamentals', name) print(colored.green('Saved %s' % name)) except Exception as err: print(colored.red(err))
def coinapi_exchanges(api): exchanges = DataFrame(api.metadata_list_exchanges()) to_pickle(exchanges, 'coinapi', 'exchanges')
def run_stooq(): for s in Index.select(): data = DataReader(s.symbol, 'stooq') to_pickle(data, 'stooq', s.symbol) sleep(10)
def write_resampled_df(df, folder, s, p): to_pickle(data=df, folder=folder, name='{}_{}'.format(s, p)) print(colored.green('Resampled {} {}'.format(s, p)))
def run_derivatives(): Y = recounstruct_vxx() to_pickle(Y, join('futures', 'VX', 'output'), 'VXX')
def run_gf(): for s in Market.select(): data = DataReader(s.symbol, 'google') to_pickle(data, 'gf', s.symbol) sleep(10)
def coinapi_periods(api): periods = DataFrame(api.ohlcv_list_all_periods()) to_pickle(periods, 'coinapi', 'periods')
def run_morningstar(): for s in Market.select(): data = DataReader(s.symbol, 'morningstar') to_pickle(data, 'morningstar', s.symbol) sleep(10)
def _cap(date): snaps = historical_snapshots(date) df = DataFrame( snaps[date], columns=['id', 'symbol', 'name', 'cap', 'price', 'supply', 'volume']) to_pickle(df, 'cmc', date)
def get(keyword): data = fred.get_series_all_releases(keyword[0]) data.set_index('date', inplace=True) to_pickle(data, 'fred', keyword[0])
def coinapi_coins(api): symbols = DataFrame(api.metadata_list_symbols()) to_pickle(symbols, 'coinapi', 'symbols')