def conapi_data(api, symbol_id, period, start_data):
data = DataFrame(
api.ohlcv_historical_data('{}'.format(symbol_id), {
'period_id': period,
'time_start': start_data
}))
to_pickle(data, 'coinapi', 'symbols', '{}_{}'.format(symbol_id, period))
def save_one(symbol):
client = c()
data = client.get_dataframe(symbol, startDate='1980-01-01')
if data is not None:
to_pickle(data, 'tiingo', '{}'.format(symbol))
print(colored.green('Got it.'))
else:
print(colored.red('Nothing for this symbols.'))
def run_quandl(check_latest=True):
for s in QUANDL_SYMBOLS:
data = get(s[0])
name = s[0].replace('/', '_')
if check_latest:
if s[2]:
ensure_latest(df=data)
to_pickle(data, 'futures', name)
print(colored.green(name))
def generate_crypto_timeframes(exchange, name):
try:
tfs = exchange.timeframes.keys()
df = DataFrame([k for k in tfs])
to_pickle(df, join('ccxt', 'tfs'), name)
print(colored.green(name))
except AttributeError:
pass
except HTTPError:
pass
except Exception as err:
print(colored.red(err))
def run_tiingo(i=0):
for s in Market.select()[i:]:
try:
data = get_data(s=s)
if data is not None:
to_pickle(data, 'tiingo', '{}'.format(s.symbol))
i += 1
except FileNotFoundError as err:
print(colored.red('Retrying due to disk error: {}'.format(err)))
run_tiingo(i=i)
except KeyError as err:
print(colored.red(err))
def cboe_download(check_latest=True):
for sym in CBOE_DATA:
url = 'http://www.cboe.com/publish/scheduledtask/mktdata/datahouse/%s.csv' % sym[
1]
content = get(url).content
data = read_csv(StringIO(content.decode('utf-8')),
skiprows=4,
names=sym[2],
parse_dates=True)
data.index = to_datetime(data['Date'], errors='coerce')
del data['Date']
if check_latest:
ensure_latest(df=data)
to_pickle(data, 'indicators', sym[0])
print(colored.green(sym[0]))
def generate_crypto_symbols(exchange, name):
try:
syms = exchange.load_markets()
out = [key for key in syms]
df = DataFrame(out)
to_pickle(df, join('ccxt', 'symbols'), name)
print(colored.green(name))
for i in range(len(out)):
df = DataFrame(syms[out[i]])
to_pickle(df, join('ccxt', 'conditions'), '{}_{}'.format(name, out[i].replace('/', '_')))
print(colored.green(out[i]))
except HTTPError:
pass
except Exception as err:
print(colored.red(err))
def download_all_crypto(first=False):
for e in EXCHANGES:
if first:
generate_crypto_timeframes(exchange=e[1], name=e[0])
generate_crypto_symbols(exchange=e[1], name=e[0])
syms = get_syms(e=e)
tf = get_tfs(e=e)
if (tf is not None) & (syms is not None):
for s in syms:
for t in tf:
data = fetch_ohlc(exchange=e[1], symbol=s, timeframe=t)
if data is not None:
to_pickle(data, 'ccxt', '{}_{}_{}'.format(e[0], s.replace('/', '_'), t))
print(colored.green('%s %s %s' % (e[0], s, t)))
def download_eurex():
source = 'http://www.stoxx.com/download/historical_values/'
es_url = source + 'hbrbcpe.txt'
vs_url = source + 'h_vstoxx.txt'
es_path = join(STORAGE_PATH, 'eurex', 'es.txt')
vs_path = join(STORAGE_PATH, 'eurex', 'vs.txt')
urlretrieve(es_url, es_path) # ES5
urlretrieve(vs_url, vs_path) # VSTOXX
columns = ['Date', 'SX5P', 'SX5E', 'SXXP', 'SXXE', 'SXXF', 'SXXA', 'DK5F', 'DKXF', 'DEL']
es = read_csv(es_path, index_col=0, parse_dates=True, dayfirst=True, header=None, skiprows=4, names=columns, sep=';')
del es['DEL']
vs = read_csv(vs_path, index_col=0, parse_dates=True, dayfirst=True, header=2)
to_pickle(es, 'eurex', 'esx')
to_pickle(vs, 'eurex', 'vsx')
def get_all_statements(sym):
stypes = ['pl', 'bs', 'cf']
ptypes = ['Q1', 'Q2', 'Q3', 'Q4', 'TTM', 'FY']
years = range(2007, datetime.now().year)
id = get_id(sym=sym)
if id is not None:
for stype in stypes:
print(stype)
try:
for year in years:
print(year)
for ptype in ptypes:
print(ptype)
df = get_statements(id=id, stype=stype, ptype=ptype, fyear=year)
if df is not None:
name = construct_name(sym=sym, stype=stype, ptype=ptype, fyear=year)
to_pickle(df, 'fundamentals', name)
print(colored.green('Saved %s' % name))
except Exception as err:
print(colored.red(err))
def coinapi_exchanges(api):
exchanges = DataFrame(api.metadata_list_exchanges())
to_pickle(exchanges, 'coinapi', 'exchanges')
def run_stooq():
for s in Index.select():
data = DataReader(s.symbol, 'stooq')
to_pickle(data, 'stooq', s.symbol)
sleep(10)
def write_resampled_df(df, folder, s, p):
to_pickle(data=df, folder=folder, name='{}_{}'.format(s, p))
print(colored.green('Resampled {} {}'.format(s, p)))
def run_derivatives():
Y = recounstruct_vxx()
to_pickle(Y, join('futures', 'VX', 'output'), 'VXX')
def run_gf():
for s in Market.select():
data = DataReader(s.symbol, 'google')
to_pickle(data, 'gf', s.symbol)
sleep(10)
def coinapi_periods(api):
periods = DataFrame(api.ohlcv_list_all_periods())
to_pickle(periods, 'coinapi', 'periods')
def run_morningstar():
for s in Market.select():
data = DataReader(s.symbol, 'morningstar')
to_pickle(data, 'morningstar', s.symbol)
sleep(10)
def _cap(date):
snaps = historical_snapshots(date)
df = DataFrame(
snaps[date],
columns=['id', 'symbol', 'name', 'cap', 'price', 'supply', 'volume'])
to_pickle(df, 'cmc', date)
def get(keyword):
data = fred.get_series_all_releases(keyword[0])
data.set_index('date', inplace=True)
to_pickle(data, 'fred', keyword[0])
def coinapi_coins(api):
symbols = DataFrame(api.metadata_list_symbols())
to_pickle(symbols, 'coinapi', 'symbols')