class TestToolsSQLA(object):
def setup_method(self, test_method):
self.eng = SetupTrump()
self.sm = SymbolManager(self.eng)
def test_repr_mixin(self):
sym = self.sm.create("testsym", overwrite=True)
assert repr(
sym
) == """Symbol(name=u'testsym', description=None, units=None, agg_method=u'priority_fill', freshthresh=0)"""
class TestToolsSQLA(object):
def setup_method(self, test_method):
self.eng = SetupTrump()
self.sm = SymbolManager(self.eng)
def test_repr_mixin(self):
sym = self.sm.create("testsym", overwrite=True)
assert repr(sym) == """Symbol(name=u'testsym', description=None, units=None, agg_method=u'priority_fill', freshthresh=0)"""
class TestToolsSQLA(object):
def setup_method(self, test_method):
self.eng = SetupTrump()
self.sm = SymbolManager(self.eng)
def test_metamatrix(self):
tikrs = [x * 3 for x in list('jkl')]
syms = []
for s in tikrs:
sym = self.sm.create(s, overwrite=True)
sym.add_meta(alpha='a', beta='b', charlie='c')
syms.append(sym)
mm = MetaMatrix(syms)
mmr = mm(['alpha','beta'])
exp = pd.DataFrame({'alpha' : ['a'] * 3,
'beta' : ['b'] * 3}, index = tikrs)
exp.index.name = 'trumpsym'
assert_frame_equal(mmr, exp)
# -*- coding: utf-8 -*-
from trump.orm import SymbolManager
sm = SymbolManager()
oil = sm.get("oil_front_month")
oil.cache()
print oil.df.tail()
sm.finish()
def setup_method(self, test_method):
self.eng = SetupTrump()
self.sm = SymbolManager(self.eng)
# -*- coding: utf-8 -*-
from trump.orm import SymbolManager
from trump.templating import QuandlFT, QuandlSecureFT, SQLFT
from datetime import datetime as dt
sm = SymbolManager()
oil = sm.create(
name="oil_front_month",
description="Crude Oil Futures, Continuous Contract #2 (CL2) (Front Month)",
units='$ / barrel')
oil.add_alias("OILA")
oil.add_tags(['commodity', 'oil'])
f1 = QuandlFT(r"CHRIS/CME_CL2", fieldname='Settle')
f2 = QuandlSecureFT(r"CHRIS/CME_CL3", fieldname='Settle')
f3 = SQLFT("SELECT date,data FROM test_oil_data;")
oil.add_feed(f1)
oil.add_feed(f2)
oil.add_feed(f3)
oil.add_override(dt(2015, 1, 6), 50.0, user='******')
oil.add_fail_safe(dt(2015, 1, 6), 45.0, user='******')
df = oil.cache()
print oil.df.tail()
# -*- coding: utf-8 -*- from trump.orm import SymbolManager from trump.templating import QuandlFT, QuandlSecureFT, SQLFT from trump.tools.bitflags import BitFlag from datetime import datetime as dt sm = SymbolManager() oil = sm.create(name = "oil_front_month", overwrite=True) f1 = QuandlFT(r"CHRIS/CME_CL3",fieldname='Settle') oil.add_feed(f1) oil.cache() print oil.df.tail() oil._reset_datatable() oil.cache() print oil.df.tail()
class TestORM(object):
@classmethod
def setup_class(cls):
cls.eng = SetupTrump()
def setup_method(self, test_method):
self.sm = SymbolManager(self.eng)
def teardown_method(self, test_method):
self.sm.complete()
def test_setuptrump(self):
SymbolManager("sqlite://")
SymbolManager()
@requires_net
def test_symbol_creation(self):
sm = self.sm
sym = sm.create("TSLA", overwrite=True)
fd = GoogleFinanceFT("TSLA", start='2015-04-01', end='2015-04-13')
sym.add_feed(fd)
sym.cache()
assert sym.df.ix['2015-04-10'][0] == 210.90
@requires_net
def test_symbol_munging_mod(self):
sm = self.sm
sym = sm.create("TSLA", overwrite=True)
fdtemp = GoogleFinanceFT("TSLA")
mgtemp = SimpleExampleMT(5, 5)
sym.add_feed(fdtemp, munging=mgtemp)
sym.cache()
df = sym.df
assert round(df.ix['2015-03-20'][0], 4) == round(0.031638, 4)
@requires_net
def test_symbol_pct_change_munge(self):
sm = self.sm
sym = sm.create("GOOG", overwrite=True)
fdtemp = GoogleFinanceFT("GOOG")
mgtemp = PctChangeMT()
sym.add_feed(fdtemp, munging=mgtemp)
sym.cache()
df = sym.df
assert round(df.ix['2015-05-08'][0], 4) == round(0.014170, 4)
print df.tail(5)
@requires_net
def test_symbol_frequency_mod(self):
sm = self.sm
sym = sm.create("TSLA", overwrite=True)
fd = GoogleFinanceFT("TSLA", start='2015-02-01', end='2015-04-05')
sym.add_feed(fd)
sym.index.case = "asfreq"
sym.index.setkwargs(freq='M', method='ffill')
sm.ses.commit()
sym.cache()
df = sym.df
assert df.ix['2015-03-31'][0] == 188.77
assert df.index.freq == 'M'
@requires_net
def test_two_feeds(self):
sm = self.sm
sym = sm.create("MSFT", overwrite=True)
fdgoog = GoogleFinanceFT("MSFT", start='2015-03-01', end='2015-03-10')
fdyhoo = YahooFinanceFT("MSFT", start='2015-03-01', end='2015-03-14')
sym.add_feed(fdgoog)
sym.add_feed(fdyhoo)
sym.cache()
ans = sym._all_datatable_data()
# the 13th is the last row, and it should be blank because
# we only fetched through the 10th.
# As of now, the third column from the last, is the 'google' feed.
assert (ans[-1][-3] is None) or pd.isnull(ans[-1][-3])
df = sym.df
assert sym.n_feeds == 2
assert round(df.ix['2015-03-13'][0], 2) == 41.38
@requires_net
def test_tags_and_search(self):
sm = self.sm
sym = sm.create("MSFT", overwrite=True)
fdmsft = YahooFinanceFT("MSFT")
sym.add_feed(fdmsft)
sym.add_tags(['tech', 'software'])
results = sm.search_tag('tech')
msft = results[0]
results = sm.search_tag('soft%')
msft2 = results[0]
assert msft2 is msft
msft.del_tags('tech')
results = sm.search_tag('tech')
assert len(results) == 0
def test_tags_and_search_feeds(self):
sm = self.sm
for s in ['vaf', 'vbf', 'vcg']:
sym = sm.create(s, overwrite=True)
testdata = os.path.join(curdir, 'testdata', 'testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
if s == 'vaf':
sym.add_tags('vvvvv') #Should not be returned in search.
sym.feeds[0].add_tags([x * 5 for x in list(s)])
syms = sm.search_tag('vvvvv', symbols=True, feeds=False)
assert len(syms) == 1
assert syms[0].name == 'vaf'
syms = sm.search_tag('vvvvv', symbols=False, feeds=True)
assert len(syms) == 3
syms = None
syms = sm.search_tag('vvvvv', symbols=True, feeds=True)
assert len(syms) == 3
def test_symbol_event_log(self):
sm = self.sm
s = 'evlg'
sym = sm.create(s, overwrite=True)
origdata = os.path.join(curdir, 'testdata', 'testdailydata.csv')
tmpdata = os.path.join(curdir, 'testdata', 'testdailydatatmp.csv')
sh.copy2(origdata, tmpdata)
fdtemp = CSVFT(tmpdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
sym.cache()
sym.cache(staleonly=False)
gc = sym.last_cache()
os.remove(tmpdata)
try:
sym.cache(staleonly=False)
except:
print "Expected to fail..."
assert sym.last_cache() == gc
lasttry = sym.last_cache('START')
assert lasttry > gc
def test_general_search(self):
sm = self.sm
for s in ['gsvaf', 'gsvbf', 'gsvcg']:
sym = sm.create(s, overwrite=True)
testdata = os.path.join(curdir, 'testdata', 'testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
if s in ('gsvaf', 'gsvcg'):
sym.add_tags('gsvvv') #Should not be returned in search.
sym.set_description('exampledesc' + s)
sym.add_meta(keyattr=s[::-1])
syms = sm.search("gsvaf", name=True)
assert len(syms) == 1
syms = sm.search("gsvvv", tags=True)
assert len(syms) == 2
syms = sm.search("exampledesc%", desc=True)
assert len(syms) == 3
syms = sm.search(s[::-1], meta=True)
assert len(syms) == 1
assert isinstance(syms[0], Symbol)
syms = sm.search(s[::-1], meta=True, stronly=True)
assert len(syms) == 1
assert isinstance(syms[0], (str, unicode))
@requires_net
def test_existence_deletion(self):
sm = self.sm
sym = sm.create("new", overwrite=True)
fdtemp = YahooFinanceFT("NEW")
sym.add_feed(fdtemp)
assert sm.exists("new")
assert sm.exists(sym)
sm.delete(sym)
assert not sm.exists('new')
with pytest.raises(Exception) as excinfo:
sm.delete(0)
assert 'Invalid symbol' in excinfo.value.message
assert not sm.exists(sym)
def test_pydata_csv(self):
sm = self.sm
sym = sm.create("new", overwrite=True)
testdata = os.path.join(curdir, 'testdata', 'testdata.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sym.cache()
df = sym.df
assert isinstance(df.index, pd.DatetimeIndex)
assert df.iloc[2][0] == 3
def test_datetime_float_override_failsafe(self):
sm = self.sm
sym = sm.create("dtflor", overwrite=True)
testdata = os.path.join(curdir, 'testdata', 'testdata.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sm.add_override(sym,
dt.date(2012, 12, 31),
5,
user='******',
comment='testcomment')
sym.cache()
df = sym.df
assert isinstance(df.index, pd.DatetimeIndex)
assert df.iloc[2][0] == 5
sm.add_fail_safe(sym,
dt.date(2011, 12, 31),
-1,
user='******',
comment='testcomment2')
sym.cache()
df = sym.df
assert df.iloc[1][0] == -1
sm.add_override(sym,
dt.date(2012, 12, 31),
4,
user='******',
comment='testcomment3')
sm.add_fail_safe(sym,
dt.date(2011, 12, 31),
-2,
user='******',
comment='testcomment4')
sym.cache()
df = sym.df
assert df.iloc[2][0] == 4
assert df.iloc[1][0] == -2
def test_int_index_string_data_override_failsafe(self):
sm = self.sm
sym = sm.create("intstrdtflor", overwrite=True)
testdata = os.path.join(curdir, 'testdata', 'teststrdata.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sym.index.indimp = "IntIndexImp"
sym.dtype.datadef = "StrDataDef"
sm.complete()
testdata = os.path.join(curdir, 'testdata', 'teststrdata2.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sm.add_override(sym, 2015, 'z', user='******', comment='testcomment6')
sym.cache()
df = sym.df
assert isinstance(df.index, pd.Int64Index)
assert isinstance(df.intstrdtflor[2010], str)
assert df.intstrdtflor[2014] == 'e'
assert df.intstrdtflor[2015] == 'z'
def test_add_feed_post_cache(self):
sm = self.sm
sym = sm.create("onetwo", overwrite=True)
testdata = os.path.join(curdir, 'testdata', 'teststrdata.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sym.dtype.datadef = "StrDataDef"
sm.complete()
sym.cache()
testdata = os.path.join(curdir, 'testdata', 'teststrdata2.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sym.cache()
df = sym.df
assert df.onetwo['2015-12-31'] == 'f'
onetwo = sm.get("onetwo")
#Why is this needed in postgres, but not sqlite?
sm.complete()
sm.delete("onetwo")
sym = sm.create("onetwo", overwrite=True)
sm.delete("onetwo")
def test_symbol_describe(self):
sm = self.sm
sym = sm.create("describer", overwrite=True)
fdtemp = CSVFT("fakefile.csv", 'Amount', index_col=0)
sym.add_feed(fdtemp)
fdtemp = CSVFT("fakefile2.csv", 'Amount', index_col=0)
sym.add_feed(fdtemp)
sym.add_tags(['atag', 'btag', 'ctag'])
result = sym.describe
exp_result = """Symbol = describer
tagged = atag, btag, ctag
aliased = describer
feeds:
0. CSVFT
1. CSVFT"""
def stripit(s):
return s.replace(" ", "").replace("\n", "")
assert stripit(result) == stripit(exp_result)
def test_update_handle(self):
sm = self.sm
sym = sm.create("uht", overwrite=True)
testdata = os.path.join(curdir, 'testdata', 'testdata.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
assert not sym.handle.validity_check.warn
assert not sym.feeds[0].handle.api_failure.warn
sym.update_handle({'validity_check': 2})
sym.feeds[0].update_handle({'api_failure': 2})
sym.cache()
assert sym.handle.validity_check.warn
assert sym.feeds[0].handle.api_failure.warn
def test_index_templating(self):
sm = self.sm
sym = sm.create("indt", overwrite=True)
weekly = FFillIT('W')
sym.set_indexing(weekly)
testdata = os.path.join(curdir, 'testdata', 'testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
sym.cache()
assert sym.df.index.freq == 'W'
def test_validity_feed_match(self):
sm = self.sm
sym = sm.create("fmvttf", overwrite=True)
fm = FeedsMatchVT(1, 2)
sym.add_validator(fm)
testdata = os.path.join(curdir, 'testdata', 'testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
sym.cache()
def test_validity_date_exists(self):
sm = self.sm
sym = sm.create("devt", overwrite=True)
fm = DateExistsVT(dt.date(2010, 02, 15))
sym.add_validator(fm)
testdata = os.path.join(curdir, 'testdata', 'testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
sym.cache()
def test_index_kwargs(self):
sm = self.sm
sym = sm.create("tinkw", overwrite=True)
testdata = os.path.join(curdir, 'testdata', 'testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
tkwargs = {
'A': None,
'B': 10,
'C': 10.0,
'D': 'test',
'E': False,
'F': dt.datetime(2010, 10, 10)
}
sym.index.setkwargs(**tkwargs)
sm.complete()
symn = sm.get('tinkw')
actkwargs = symn.index.getkwargs()
assert tkwargs == actkwargs
def test_view_creation(self):
sm = self.sm
for s in ['va', 'vb', 'vc']:
sym = sm.create(s, overwrite=True)
sym.add_tags('testtagz')
testdata = os.path.join(curdir, 'testdata', 'testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
sm.complete()
sym.cache()
sm.build_view_from_tag('testtagz')
def test_fx_converting(self):
sm = self.sm
fxdata = os.path.join(curdir, 'testdata', 'fxdata3.csv')
for pair in ['EURUSD', 'GBPUSD', 'CADUSD', 'AUDEUR', 'USDJPY']:
print pair
sym = sm.create(pair, overwrite=True)
fdtemp = CSVFT(fxdata, pair, index_col=0)
sym.add_feed(fdtemp)
business_day = FFillIT('B')
sym.set_indexing(business_day)
#USDCAD -> "CAD/USD"
sym.set_units("{}/{}".format(pair[3:], pair[:3]))
sym.add_tags('forex')
sym.cache()
cm = ConversionManager(self.eng, 'FX', 'forex')
#Should be the same as GBP...
df = cm.get_converted('GBPUSD', 'USD')
assert floats_equal(df.ix['2015-05-15'][0], 1.57370)
#Should be ~1.88
df = cm.get_converted('GBPUSD', 'CAD')
assert floats_equal(df.ix['2015-05-15'][0], 1.88357)
#Should be 1.0000
df = cm.get_converted('GBPUSD', 'GBP')
assert floats_equal(df.ix['2015-05-15'][0], 1.0)
#Should be ~0.97
df = cm.get_converted('AUDEUR', 'CAD')
assert floats_equal(df.ix['2015-05-15'][0], 0.96678)
#Should be ~187
df = cm.get_converted('GBPUSD', 'JPY')
assert floats_equal(df.ix['2015-05-15'][0], 187.7817525)
#Should error...since CHF wasn't added.
try:
df = cm.get_converted('GBPUSD', 'CHF')
assert False
except Exception, exp:
assert exp.message == "Converter has insufficient data to process USD to CHF"
# -*- coding: utf-8 -*-
from trump.orm import SymbolManager
from trump.templating import QuandlFT, QuandlSecureFT, SQLFT
from datetime import datetime as dt
sm = SymbolManager()
oil = sm.create(name = "oil_front_month",
description = "Crude Oil Futures, Continuous Contract #2 (CL2) (Front Month)",
units = '$ / barrel')
oil.add_alias("OILA")
oil.add_tags(['commodity','oil'])
f1 = QuandlFT(r"CHRIS/CME_CL2",fieldname='Settle')
f2 = QuandlSecureFT(r"CHRIS/CME_CL3",fieldname='Settle')
f3 = SQLFT("SELECT date,data FROM test_oil_data;")
oil.add_feed(f1)
oil.add_feed(f2)
oil.add_feed(f3)
oil.add_override(dt(2015,1,6),50.0,user='******')
oil.add_fail_safe(dt(2015,1,6),45.0,user='******')
df = oil.cache()
print oil.df.tail()
from trump.templating import QuandlFT, mAddFFillMult
from datetime import datetime as dt
from collections import namedtuple
info = namedtuple('info',['description','ticker'])
oil = {"EU" : [info("Europe Brent Crude Oil Spot Price FOB", r"DOE/RBRTE")],
"US" : [info("WTI Crude Oil Spot Price Cushing, OK FOB", r"DOE/RWTC")],
"CONT" : [info("Crude Oil Futures, Continuous Contracts",r"CHRIS/CME_CL1"),
info("Crude Oil Futures, Continuous Contract #2 (CL2) (Front Month)",r"CHRIS/CME_CL2")]}
sm = SymbolManager()
rawmunging = False
templatedmunging = True
if rawmunging:
for oil_sym,oil_info_l in oil.iteritems():
new_sym = sm.create("oil_" + oil_sym,"Quandl - " + oil_info_l[0].description,'D','$ / barrel')
for oil_info in oil_info_l:
f = QuandlFT(oil_info.ticker,fieldname='Settle')
f.addMunging('crop_dates',{'start' : '2000-01-01', 'end' : '2015-01-01'})
f.addMunging('multiply_const',3)
new_sym.add_feed(f,fillna={'method' : 'ffill'},exp_const=(3,['a','b','c'],{'n' : 123}),add_const={'n' : 55})
new_sym.add_alias(oil_sym + "_oil")
# -*- coding: utf-8 -*-
"""
Created on Sat Jun 6 16:02:28 2015
@author: jnm
"""
from trump.orm import SymbolManager
from trump.templating.templates import FFillIT, WorldBankFT, StLouisFEDFT, QuandlFT, GoogleFinanceFT, YahooFinanceFT
sm = SymbolManager()
xiu = sm.create('XIU', overwrite=True)
xiu.add_tags(['ETF', 'iShares'])
xiu.add_meta(Geography='Canada', AssetClass='Equity', MER="0.07")
xiu.set_description("S&P/TSX 60 Index Fund")
xiu.set_units("CAD/unit")
#xiu.add_feed(GoogleFinanceFT('TSE:XIU'))
xiu.add_feed(YahooFinanceFT('XIU.TO'))
xiu.cache()
spy = sm.create('SPY', overwrite=True)
spy.add_tags(['ETF', 'equity'])
spy.set_description("SPDR S&P 500 ETF Trust")
spy.add_meta(Geography='USA', AssetClass='Equity', Index='S&P 500', MER="0.11", CUSIP="78462F103", Exchange="NYSE")
spy.set_units("USD/unit")
spy.add_feed(YahooFinanceFT('SPY'))
spy.cache()
tsla = sm.create('TSLA', overwrite=True)
tsla.add_tags(['ETF', 'equity', 'Company'])
"""
Created on Sat Jun 6 16:02:28 2015
@author: jnm
"""
from trump.orm import SymbolManager
from trump.templating.templates import FFillIT, WorldBankFT, StLouisFEDFT, QuandlSecureFT, QuandlFT, GoogleFinanceFT, YahooFinanceFT
import requests
dofed = False
doraymond = False
donvca = True
sm = SymbolManager()
if donvca:
r = requests.get('https://www.quandl.com/api/v2/datasets.json?query=*&source_code=NVCA&per_page=300&page=1&auth_token=y9RJp7f7DRW3Aj7FEqq1')
r = r.json()
for doc in r['docs']:
code = doc['code']
if code == 'VENTURE_3_10':
cols = doc['column_names']
del cols[cols.index('Year')]
del cols[cols.index('Unknown')]
name = doc['name']
class TestORM(object):
@classmethod
def setup_class(cls):
cls.eng = SetupTrump()
def setup_method(self, test_method):
self.sm = SymbolManager(self.eng)
def teardown_method(self, test_method):
self.sm.complete()
def test_setuptrump(self):
SymbolManager("sqlite://")
SymbolManager()
@requires_net
def test_symbol_creation(self):
sm = self.sm
sym = sm.create("TSLA", overwrite=True)
fd = GoogleFinanceFT("TSLA", start='2015-04-01', end='2015-04-13')
sym.add_feed(fd)
sym.cache()
assert sym.df.ix['2015-04-10'][0] == 210.90
@requires_net
def test_symbol_munging_mod(self):
sm = self.sm
sym = sm.create("TSLA", overwrite=True)
fdtemp = GoogleFinanceFT("TSLA")
mgtemp = SimpleExampleMT(5,5)
sym.add_feed(fdtemp,munging=mgtemp)
sym.cache()
df = sym.df
assert round(df.ix['2015-03-20'][0], 4) == round(0.031638, 4)
@requires_net
def test_symbol_pct_change_munge(self):
sm = self.sm
sym = sm.create("GOOG", overwrite=True)
fdtemp = GoogleFinanceFT("GOOG")
mgtemp = PctChangeMT()
sym.add_feed(fdtemp, munging=mgtemp)
sym.cache()
df = sym.df
assert round(df.ix['2015-05-08'][0], 4) == round(0.014170, 4)
print df.tail(5)
@requires_net
def test_symbol_frequency_mod(self):
sm = self.sm
sym = sm.create("TSLA", overwrite=True)
fd = GoogleFinanceFT("TSLA", start='2015-02-01', end='2015-04-05')
sym.add_feed(fd)
sym.index.case = "asfreq"
sym.index.setkwargs(freq='M', method='ffill')
sm.ses.commit()
sym.cache()
df = sym.df
assert df.ix['2015-03-31'][0] == 188.77
assert df.index.freq == 'M'
@requires_net
def test_two_feeds(self):
sm = self.sm
sym = sm.create("MSFT", overwrite=True)
fdgoog = GoogleFinanceFT("MSFT", start='2015-03-01', end='2015-03-10')
fdyhoo = YahooFinanceFT("MSFT", start='2015-03-01', end='2015-03-14')
sym.add_feed(fdgoog)
sym.add_feed(fdyhoo)
sym.cache()
ans = sym._all_datatable_data()
# the 13th is the last row, and it should be blank because
# we only fetched through the 10th.
# As of now, the third column from the last, is the 'google' feed.
assert (ans[-1][-3] is None) or pd.isnull(ans[-1][-3])
df = sym.df
assert sym.n_feeds == 2
assert round(df.ix['2015-03-13'][0], 2) == 41.38
@requires_net
def test_tags_and_search(self):
sm = self.sm
sym = sm.create("MSFT", overwrite=True)
fdmsft = YahooFinanceFT("MSFT")
sym.add_feed(fdmsft)
sym.add_tags(['tech','software'])
results = sm.search_tag('tech')
msft = results[0]
results = sm.search_tag('soft%')
msft2 = results[0]
assert msft2 is msft
msft.del_tags('tech')
results = sm.search_tag('tech')
assert len(results) == 0
def test_tags_and_search_feeds(self):
sm = self.sm
for s in ['vaf', 'vbf', 'vcg']:
sym = sm.create(s, overwrite=True)
testdata = os.path.join(curdir,'testdata','testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
if s == 'vaf':
sym.add_tags('vvvvv') #Should not be returned in search.
sym.feeds[0].add_tags([x * 5 for x in list(s)])
syms = sm.search_tag('vvvvv', symbols=True, feeds=False)
assert len(syms) == 1
assert syms[0].name == 'vaf'
syms = sm.search_tag('vvvvv', symbols=False, feeds=True)
assert len(syms) == 3
syms = None
syms = sm.search_tag('vvvvv', symbols=True, feeds=True)
assert len(syms) == 3
def test_symbol_event_log(self):
sm = self.sm
s = 'evlg'
sym = sm.create(s, overwrite=True)
origdata = os.path.join(curdir,'testdata','testdailydata.csv')
tmpdata = os.path.join(curdir,'testdata','testdailydatatmp.csv')
sh.copy2(origdata,tmpdata)
fdtemp = CSVFT(tmpdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
sym.cache()
sym.cache(staleonly=False)
gc = sym.last_cache()
os.remove(tmpdata)
try:
sym.cache(staleonly=False)
except:
print "Expected to fail..."
assert sym.last_cache() == gc
lasttry = sym.last_cache('START')
assert lasttry > gc
def test_general_search(self):
sm = self.sm
for s in ['gsvaf', 'gsvbf', 'gsvcg']:
sym = sm.create(s, overwrite=True)
testdata = os.path.join(curdir,'testdata','testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
if s in ('gsvaf', 'gsvcg'):
sym.add_tags('gsvvv') #Should not be returned in search.
sym.set_description('exampledesc' + s)
sym.add_meta(keyattr=s[::-1])
syms = sm.search("gsvaf", name=True)
assert len(syms) == 1
syms = sm.search("gsvvv", tags=True)
assert len(syms) == 2
syms = sm.search("exampledesc%", desc=True)
assert len(syms) == 3
syms = sm.search(s[::-1], meta=True)
assert len(syms) == 1
assert isinstance(syms[0], Symbol)
syms = sm.search(s[::-1], meta=True, stronly=True)
assert len(syms) == 1
assert isinstance(syms[0], (str, unicode))
@requires_net
def test_existence_deletion(self):
sm = self.sm
sym = sm.create("new", overwrite=True)
fdtemp = YahooFinanceFT("NEW")
sym.add_feed(fdtemp)
assert sm.exists("new")
assert sm.exists(sym)
sm.delete(sym)
assert not sm.exists('new')
with pytest.raises(Exception) as excinfo:
sm.delete(0)
assert 'Invalid symbol' in excinfo.value.message
assert not sm.exists(sym)
def test_pydata_csv(self):
sm = self.sm
sym = sm.create("new", overwrite=True)
testdata = os.path.join(curdir,'testdata','testdata.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sym.cache()
df = sym.df
assert isinstance(df.index, pd.DatetimeIndex)
assert df.iloc[2][0] == 3
def test_datetime_float_override_failsafe(self):
sm = self.sm
sym = sm.create("dtflor", overwrite=True)
testdata = os.path.join(curdir,'testdata','testdata.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sm.add_override(sym, dt.date(2012, 12, 31), 5, user='******',
comment='testcomment')
sym.cache()
df = sym.df
assert isinstance(df.index, pd.DatetimeIndex)
assert df.iloc[2][0] == 5
sm.add_fail_safe(sym, dt.date(2011, 12, 31), -1, user='******',
comment='testcomment2')
sym.cache()
df = sym.df
assert df.iloc[1][0] == -1
sm.add_override(sym, dt.date(2012, 12, 31), 4, user='******',
comment='testcomment3')
sm.add_fail_safe(sym, dt.date(2011, 12, 31), -2, user='******',
comment='testcomment4')
sym.cache()
df = sym.df
assert df.iloc[2][0] == 4
assert df.iloc[1][0] == -2
def test_int_index_string_data_override_failsafe(self):
sm = self.sm
sym = sm.create("intstrdtflor", overwrite=True)
testdata = os.path.join(curdir,'testdata','teststrdata.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sym.index.indimp = "IntIndexImp"
sym.dtype.datadef = "StrDataDef"
sm.complete()
testdata = os.path.join(curdir,'testdata','teststrdata2.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sm.add_override(sym, 2015, 'z', user='******',
comment='testcomment6')
sym.cache()
df = sym.df
assert isinstance(df.index, pd.Int64Index)
assert isinstance(df.intstrdtflor[2010], str)
assert df.intstrdtflor[2014] == 'e'
assert df.intstrdtflor[2015] == 'z'
def test_add_feed_post_cache(self):
sm = self.sm
sym = sm.create("onetwo", overwrite=True)
testdata = os.path.join(curdir,'testdata','teststrdata.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sym.dtype.datadef = "StrDataDef"
sm.complete()
sym.cache()
testdata = os.path.join(curdir,'testdata','teststrdata2.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
sym.cache()
df = sym.df
assert df.onetwo['2015-12-31'] == 'f'
onetwo = sm.get("onetwo")
#Why is this needed in postgres, but not sqlite?
sm.complete()
sm.delete("onetwo")
sym = sm.create("onetwo", overwrite=True)
sm.delete("onetwo")
def test_symbol_describe(self):
sm = self.sm
sym = sm.create("describer", overwrite=True)
fdtemp = CSVFT("fakefile.csv", 'Amount', index_col=0)
sym.add_feed(fdtemp)
fdtemp = CSVFT("fakefile2.csv", 'Amount', index_col=0)
sym.add_feed(fdtemp)
sym.add_tags(['atag', 'btag', 'ctag'])
result = sym.describe
exp_result = """Symbol = describer
tagged = atag, btag, ctag
aliased = describer
feeds:
0. CSVFT
1. CSVFT"""
def stripit(s):
return s.replace(" ", "").replace("\n","")
assert stripit(result) == stripit(exp_result)
def test_update_handle(self):
sm = self.sm
sym = sm.create("uht", overwrite=True)
testdata = os.path.join(curdir,'testdata','testdata.csv')
fdtemp = CSVFT(testdata, 'Amount', index_col=0)
sym.add_feed(fdtemp)
assert not sym.handle.validity_check.warn
assert not sym.feeds[0].handle.api_failure.warn
sym.update_handle({'validity_check' : 2})
sym.feeds[0].update_handle({'api_failure' : 2})
sym.cache()
assert sym.handle.validity_check.warn
assert sym.feeds[0].handle.api_failure.warn
def test_index_templating(self):
sm = self.sm
sym = sm.create("indt", overwrite=True)
weekly = FFillIT('W')
sym.set_indexing(weekly)
testdata = os.path.join(curdir,'testdata','testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
sym.cache()
assert sym.df.index.freq == 'W'
def test_validity_feed_match(self):
sm = self.sm
sym = sm.create("fmvttf", overwrite=True)
fm = FeedsMatchVT(1,2)
sym.add_validator(fm)
testdata = os.path.join(curdir,'testdata','testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
sym.cache()
def test_validity_date_exists(self):
sm = self.sm
sym = sm.create("devt", overwrite=True)
fm = DateExistsVT(dt.date(2010,02,15))
sym.add_validator(fm)
testdata = os.path.join(curdir,'testdata','testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
sym.cache()
def test_index_kwargs(self):
sm = self.sm
sym = sm.create("tinkw", overwrite=True)
testdata = os.path.join(curdir,'testdata','testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
tkwargs = {'A' : None, 'B' : 10, 'C' : 10.0, 'D' : 'test',
'E' : False, 'F' : dt.datetime(2010,10,10)}
sym.index.setkwargs(**tkwargs)
sm.complete()
symn = sm.get('tinkw')
actkwargs = symn.index.getkwargs()
assert tkwargs == actkwargs
def test_view_creation(self):
sm = self.sm
for s in ['va', 'vb', 'vc']:
sym = sm.create(s, overwrite=True)
sym.add_tags('testtagz')
testdata = os.path.join(curdir,'testdata','testdailydata.csv')
fdtemp = CSVFT(testdata, 'Amount', parse_dates=0, index_col=0)
sym.add_feed(fdtemp)
sm.complete()
sym.cache()
sm.build_view_from_tag('testtagz')
def test_fx_converting(self):
sm = self.sm
fxdata = os.path.join(curdir,'testdata','fxdata3.csv')
for pair in ['EURUSD', 'GBPUSD', 'CADUSD', 'AUDEUR', 'USDJPY']:
print pair
sym = sm.create(pair, overwrite=True)
fdtemp = CSVFT(fxdata, pair, index_col=0)
sym.add_feed(fdtemp)
business_day = FFillIT('B')
sym.set_indexing(business_day)
#USDCAD -> "CAD/USD"
sym.set_units("{}/{}".format(pair[3:], pair[:3]))
sym.add_tags('forex')
sym.cache()
cm = ConversionManager(self.eng, 'FX', 'forex')
#Should be the same as GBP...
df = cm.get_converted('GBPUSD', 'USD')
assert floats_equal(df.ix['2015-05-15'][0], 1.57370)
#Should be ~1.88
df = cm.get_converted('GBPUSD', 'CAD')
assert floats_equal(df.ix['2015-05-15'][0], 1.88357)
#Should be 1.0000
df = cm.get_converted('GBPUSD', 'GBP')
assert floats_equal(df.ix['2015-05-15'][0], 1.0)
#Should be ~0.97
df = cm.get_converted('AUDEUR', 'CAD')
assert floats_equal(df.ix['2015-05-15'][0], 0.96678)
#Should be ~187
df = cm.get_converted('GBPUSD', 'JPY')
assert floats_equal(df.ix['2015-05-15'][0], 187.7817525)
#Should error...since CHF wasn't added.
try:
df = cm.get_converted('GBPUSD', 'CHF')
assert False
except Exception, exp:
assert exp.message == "Converter has insufficient data to process USD to CHF"
def test_setuptrump(self):
SymbolManager("sqlite://")
SymbolManager()
def setup_method(self, test_method):
self.eng = SetupTrump()
self.sm = SymbolManager(self.eng)
# -*- coding: utf-8 -*-
from trump.orm import SymbolManager
from trump.templating import QuandlFT, GoogleFinanceFT, YahooFinanceFT
sm = SymbolManager()
TSLA = sm.create(name = "TSLA", description = "Tesla Closing Price USD",
units = '$ / share')
TSLA.add_tags(["stocks","US"])
#Try Google First
#If Google's feed has a problem, try Quandl's backup
#If all else fails, use Yahoo's data...
TSLA.add_feed(GoogleFinanceFT("TSLA"))
TSLA.add_feed(QuandlFT("GOOG/NASDAQ_TSLA",fieldname='Close'))
TSLA.add_feed(YahooFinanceFT("TSLA"))
#All three are downloaded, with every cache instruction
TSLA.cache()
# Trump's plans for various aggregation, munging, and validity checking can
# use the intelligence of multiple feeds to compare and clean up the data
# if a variety common problems exist. These features are few months off,
# but a priority in Trump's plans.
# In the end, you're left with one clean pandas Series representing TSLA's closing
# price, with source, munging, validity checks, all stored persistently for future
# re-caching.
# -*- coding: utf-8 -*- from trump.orm import SymbolManager from trump.templating import QuandlFT, QuandlSecureFT, SQLFT from trump.tools.bitflags import BitFlag from datetime import datetime as dt sm = SymbolManager() oil = sm.create(name="oil_front_month", overwrite=True) oil.index.case = "asfreq" oil.index.setkwargs(freq='W', method='ffill') sm.ses.commit() f1 = QuandlFT(r"CHRIS/CME_CL3", fieldname='Settle') oil.add_feed(f1) oil.cache() print oil.df.tail() print oil.df.index
# -*- coding: utf-8 -*-
from trump.orm import SymbolManager
from trump.templating import QuandlFT, GoogleFinanceFT, YahooFinanceFT
sm = SymbolManager()
TSLA = sm.create(name="TSLA",
description="Tesla Closing Price USD",
units='$ / share')
TSLA.add_tags(["stocks", "US"])
#Try Google First
#If Google's feed has a problem, try Quandl's backup
#If all else fails, use Yahoo's data...
TSLA.add_feed(GoogleFinanceFT("TSLA"))
TSLA.add_feed(QuandlFT("GOOG/NASDAQ_TSLA", fieldname='Close'))
TSLA.add_feed(YahooFinanceFT("TSLA"))
#All three are downloaded, with every cache instruction
TSLA.cache()
# Trump's plans for various aggregation, munging, and validity checking can
# use the intelligence of multiple feeds to compare and clean up the data
# if a variety common problems exist. These features are few months off,
# but a priority in Trump's plans.
# In the end, you're left with one clean pandas Series representing TSLA's closing
# price, with source, munging, validity checks, all stored persistently for future
