def test_checkPricing(self):
import time
instrumentList = [
Instrument(symbol='A', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAAL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAPL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='ABT', asset_type=AssetType.us_equity, currency=Currency.usd),
]
for instrument in instrumentList:
self.vector_service.vectorDao.delete(instrument)
self.vector_service.vectorizedDataService.historicalMarketDataService.deleteBar(instrument=instrument,
period=Period.day,
number_of_periods=1)
start = time.time()
dictPricing = {}
wrongInstrumentsSymbolLists = []
for instrument in instrumentList:
self.vector_service.vectorizedDataService.__downloadHistoricalPrices__(
wrongInstrumentsSymbolLists=wrongInstrumentsSymbolLists,
outputDict=dictPricing,
fromDate=datetime.datetime(year=2010,
day=1,
month=1),
toDate=self.toDate,
instrument=instrument
)
dict_of_matrix = self.vector_service.getDataDictOfMatrix(instrumentList=instrumentList,
ratioList=[],
fromDate=datetime.datetime(year=2010, day=1, month=1),
toDate=self.toDate,
)
end = time.time()
elapsed = end - start
print('1-Took %d seconds to complete 4 instrument downloading' % (elapsed))
self.assertIsNotNone(dict_of_matrix)
self.assertEqual(5, len(dict_of_matrix))
keysOutput = dict_of_matrix.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
# %%
for key in dictPricing.keys():
if key == 'wrong':
continue
for symbol in dict_of_matrix[key].columns:
print('testing pricing equality in %s:%s' % (key, symbol))
isBusinessDay = BDay().onOffset
match_series = pd.to_datetime(dictPricing[key].index).map(isBusinessDay)
dictPricing_df = dictPricing[key][match_series].copy()
dictPricing_df.fillna(method='ffill', inplace=True)
df = dict_of_matrix[key][symbol]
df_pricing = dictPricing_df[symbol]
# self.assertEqual(len(df), len(df_pricing.dropna()))
sum1 = df.sum()
sum2 = df_pricing.sum()
class InstrumentTest(unittest.TestCase):
instrumentList = [
Instrument(symbol='ABC',
asset_type=AssetType.us_equity,
currency=Currency.usd),
Instrument(symbol='ACN',
asset_type=AssetType.us_equity,
currency=Currency.usd),
Instrument(symbol='AAPL',
asset_type=AssetType.us_equity,
currency=Currency.usd),
Instrument(symbol='GOOGL',
asset_type=AssetType.us_equity,
currency=Currency.usd),
Instrument(symbol='MSFT',
asset_type=AssetType.us_equity,
currency=Currency.usd),
]
def test_checkUnstackStackInstrumentList(self):
symbolList, assetType, currency = __unstackInstrumentList__(
self.instrumentList)
self.assertIsNotNone(symbolList)
self.assertIsNotNone(assetType)
self.assertIsNotNone(currency)
self.assertEqual(len(symbolList), len(self.instrumentList))
instrumentListBack = __stackInstrumentList__(symbolList, assetType,
currency)
self.assertIsNotNone(instrumentListBack)
self.assertEqual(len(instrumentListBack), len(self.instrumentList))
class AlphaVantageHistoricalDataTestCase(unittest.TestCase):
symbol = 'MSFT'
broker = Broker.manual_email
currency = Currency.usd
asset_type = AssetType.future
instrument = Instrument(symbol,
asset_type=asset_type,
broker=broker,
currency=currency)
instrument_wrong = Instrument(symbol + 'losif',
asset_type=asset_type,
broker=broker,
currency=currency)
fromDate = datetime.datetime.today() - datetime.timedelta(days=5)
user_settings = user_settings_tests
market_data_object = AlphaVantageHistoricalMarketData(user_settings)
# def setUp(self):
# pass
#
# def tearDown(self):
# pass
@unittest.skip("error premium account")
def test_download_data_minute(self):
# premium
fromDateMin = datetime.datetime.today() - datetime.timedelta(days=1)
dataframe = self.market_data_object.download(self.instrument,
period=Period.minute,
number_of_periods=1,
fromDate=fromDateMin)
self.assertIsNotNone(dataframe)
def test_download_data_day(self):
dataframe = self.market_data_object.download(self.instrument,
period=Period.day,
number_of_periods=1,
fromDate=self.fromDate)
self.assertIsNotNone(dataframe)
def test_download_data_day_wrong(self):
dataframe = self.market_data_object.download(self.instrument_wrong,
period=Period.day,
number_of_periods=1,
fromDate=self.fromDate)
self.assertIsNone(dataframe)
def test_download_data_minute_wrong(self):
dataframe = self.market_data_object.download(self.instrument_wrong,
period=Period.minute,
number_of_periods=1,
fromDate=self.fromDate)
self.assertIsNone(dataframe)
class OandaHistoricalDataTestCase(unittest.TestCase):
symbol = 'EUR'
broker = Broker.manual_email
currency = Currency.usd
asset_type = AssetType.forex
instrument = Instrument(symbol,
asset_type=asset_type,
broker=broker,
currency=currency)
instrument_wrong = Instrument(symbol + 'losif',
asset_type=asset_type,
broker=broker,
currency=currency)
fromDate = datetime.datetime.today() - datetime.timedelta(days=5)
user_settings = user_settings_tests
market_data_object = OandaHistoricalMarketData(user_settings)
# def setUp(self):
# pass
#
# def tearDown(self):
# pass
def test_download_data_minute(self):
dataframe = self.market_data_object.download(
self.instrument,
period=Period.minute,
number_of_periods=1,
fromDate=datetime.datetime(2018, 6, 20),
toDate=datetime.datetime(2018, 6, 21))
self.assertIsNotNone(dataframe)
def test_download_data_day(self):
dataframe = self.market_data_object.download(self.instrument,
period=Period.day,
number_of_periods=1,
fromDate=self.fromDate)
self.assertIsNotNone(dataframe)
def test_download_data_day_wrong(self):
dataframe = self.market_data_object.download(self.instrument_wrong,
period=Period.day,
number_of_periods=1,
fromDate=self.fromDate)
self.assertIsNone(dataframe)
def test_download_data_minute_wrong(self):
dataframe = self.market_data_object.download(self.instrument_wrong,
period=Period.minute,
number_of_periods=1,
fromDate=self.fromDate)
self.assertIsNone(dataframe)
class CryptoCompareHistoricalDataTestCase(unittest.TestCase):
symbol = 'BTC'
broker = Broker.gdax
currency = Currency.eur
instrument = Instrument(symbol, broker, currency)
asset_type = AssetType.crypto
instrument_wrong = Instrument(symbol + 'losif',
asset_type=asset_type,
broker=broker,
currency=currency)
fromDate = datetime.datetime.today() - datetime.timedelta(days=5)
user_settings = user_settings_tests
market_data_object = CryptoCompareHistoricalMarketData(user_settings)
# def setUp(self):
# pass
#
# def tearDown(self):
# pass
def test_download_data_minute(self):
dataframe = self.market_data_object.download(self.instrument,
period=Period.minute,
number_of_periods=1,
fromDate=self.fromDate)
self.assertIsNotNone(dataframe)
def test_download_data_day(self):
dataframe = self.market_data_object.download(self.instrument,
period=Period.day,
number_of_periods=1,
fromDate=self.fromDate)
self.assertIsNotNone(dataframe)
def test_download_data_day_wrong(self):
dataframe = self.market_data_object.download(self.instrument_wrong,
period=Period.day,
number_of_periods=1,
fromDate=self.fromDate)
self.assertIsNone(dataframe)
def test_download_data_minute_wrong(self):
dataframe = self.market_data_object.download(self.instrument_wrong,
period=Period.minute,
number_of_periods=1,
fromDate=self.fromDate)
self.assertIsNone(dataframe)
def test_getAllBatch_wrong(self):
fromDate = datetime.datetime(year=2010, day=1, month=1)
instrument = Instrument('A123APL', asset_type=AssetType.us_equity, currency=Currency.usd)
df_batch = self.fundamental_data_object.downloadBatch(instrument=instrument,
fundamental_ratio_list=Ratio.fundamental_list_Y,
fromDate=fromDate, toDate=self.toDate)
self.assertIsNone(df_batch)
class QuandlQuantCalculatorTestCase(unittest.TestCase):
instrument = Instrument('AAPL',
asset_type=AssetType.us_equity,
currency=Currency.usd)
user_settings = user_settings_tests
quant_data_object = QuantDataImpl(user_settings)
fromDate = datetime.datetime(year=2017, day=1, month=1)
toDate = datetime.datetime(year=2018, day=26, month=6)
def test_calculateReturnDiff(self):
ratio = Ratio.quant_returnsDiff_120
data = self.quant_data_object.download(self.instrument,
quant_ratio=ratio,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(data)
self.assertTrue(Ratio.ratio in data.columns)
self.assertEqual(len(data.columns), 1)
def test_all_quants(self):
ratioList = Ratio.quant_list
for ratio in ratioList:
print('testing ratio %s ' % ratio)
downloadTest = self.quant_data_object.download(
self.instrument,
quant_ratio=ratio,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(downloadTest)
self.assertNotEquals(len(downloadTest), 0)
self.assertTrue(Ratio.ratio in downloadTest.columns)
self.assertEqual(len(downloadTest.columns), 1)
def getDataFile_instrument_from_to_period_numberPeriods(self, file):
import datetime
filename = file.split(os.sep)[-1][:-4]
fileSplitted = filename.split('_')
symbol = fileSplitted[0][:3]
currency = fileSplitted[0][3:]
periodDukas = fileSplitted[1]
if periodDukas == self.period_dict[Period.tick]:
# tick
number_periods = int(1)
period = Period.tick
dates = fileSplitted[2].split('-')
fromDate = datetime.datetime.strptime(dates[0],
self.filenameDateFormat)
toDate = datetime.datetime.strptime(dates[1],
self.filenameDateFormat)
else:
# bars
number_periods = int(fileSplitted[2])
period = {
key
for key, value in self.period_dict.items()
if value == fileSplitted[3]
}.pop() # self.period_dict[fileSplitted[3]]
dates = fileSplitted[5].split('-')
fromDate = datetime.datetime.strptime(dates[0],
self.filenameDateFormat)
toDate = datetime.datetime.strptime(dates[1],
self.filenameDateFormat)
toDate = self.getToDateFromFile(toDate, period)
instrument = Instrument(symbol=symbol,
currency=currency,
asset_type=AssetType.forex)
return instrument, fromDate, toDate, period, number_periods
class HistoricalMarketDataServiceTestCase(unittest.TestCase):
user_settings = user_settings_tests
historicalMarketService = HistoricalMarketDataService(user_settings)
instrument = Instrument(symbol='ETH',
broker=Broker.gdax,
asset_type=AssetType.crypto,
currency=Currency.eur)
period = Period.hour
number_of_periods = 1
def setUp(self):
pass
def tearDown(self):
# delete
# self.historicalMarketService.deleteBar(instrument=self.instrument,period=self.period,number_of_periods=self.number_of_periods)
pass
def test_getHistorical_hour(self):
# clean first
# self.historicalMarketService.deleteBar(instrument=self.instrument, period=self.period,
# number_of_periods=self.number_of_periods)
import datetime
from_date = datetime.datetime.today() - datetime.timedelta(days=2)
to_date = datetime.datetime.today() - datetime.timedelta(days=1)
data = self.historicalMarketService.getHistoricalData(
self.instrument,
period=self.period,
number_of_periods=self.number_of_periods,
fromDate=from_date,
toDate=to_date,
force_download=True)
self.assertIsNotNone(data)
def test_getHistorical__dailyClean(self):
# clean first
# self.historicalMarketService.deleteBar(instrument=self.instrument, period=self.period,
# number_of_periods=self.number_of_periods)
import datetime
instrument = Instrument(symbol='AAPL',
broker=Broker.manual_email,
asset_type=AssetType.us_equity,
currency=Currency.usd)
from_date = datetime.datetime(year=2010, month=1, day=1)
to_date = datetime.datetime(year=2012, month=1, day=1)
period = Period.day
number_of_periods = 1
data = self.historicalMarketService.getHistoricalData(
instrument,
period=period,
number_of_periods=number_of_periods,
fromDate=from_date,
toDate=to_date,
force_download=True,
endOfDayData=True)
self.assertIsNotNone(data)
def _pricing_iter():
sid = 0
with maybe_show_progress(
symbols,
show_progress,
label='Downloading Tradea Database pricing data ') as it, \
requests.Session() as session:
for symbol in it:
logger.debug('zipline bundle downloading %s' % symbol)
try:
instrument = Instrument(
symbol=symbol, asset_type=AssetType.us_equity)
df = self.historical_market_data_service.getHistoricalData(
instrument,
period=Period.day,
number_of_periods=1,
fromDate=start,
toDate=end,
bar_type=BarType.time_bar,
force_download=False,
cleanOutliers=False)
except Exception as e:
logger.error(
'Error downloading bundle zipline %s : %s' %
(symbol, str(e)))
print('Error downloading bundle zipline %s : %s' %
(symbol, str(e)))
df = None
continue
# the start date is the date of the first trade and
# the end date is the date of the last trade
indexSet = df.index.copy()
indexSet = (indexSet + pd.DateOffset(hours=3)
) - pd.DateOffset(days=1)
df.index = indexSet
start_date = df.index[0]
end_date = df.index[-1]
# The auto_close date is the day after the last trade.
ac_date = end_date + pd.Timedelta(days=1)
metadata.iloc[
sid] = start_date, end_date, ac_date, symbol
df.rename(
columns={
Bar.open: 'open',
Bar.high: 'high',
Bar.low: 'low',
Bar.close: 'close',
Bar.volume: 'volume',
},
inplace=True,
)
yield sid, df
sid += 1
class InteractiveBrokersHistoricalDataTestCase(unittest.TestCase):
symbol = 'EUR'
broker = Broker.manual_email
currency = Currency.usd
asset_type = AssetType.forex
instrument = Instrument(symbol, asset_type=asset_type, broker=broker, currency=currency)
fromDate = datetime.datetime.today() - datetime.timedelta(days=5)
user_settings = user_settings_tests
market_data_object = InteractiveBrokersHistoricalMarketData(user_settings)
@unittest.skip # no reason needed
def test_getPeriodsRequests(self):
fromDate = datetime.datetime.today() - datetime.timedelta(days=40)
toDate = datetime.datetime.today()
periodSplit = Period.hour
durationStr_list, toDateString_list = self.market_data_object.getPeriodsRequests(periodSplit, fromDate, toDate)
self.assertGreater(len(durationStr_list), 1)
self.assertGreater(len(toDateString_list), 1)
pass
@unittest.skip # no reason needed
def test_download_fx(self):
symbol = 'EUR'
broker = Broker.manual_email
currency = Currency.usd
asset_type = AssetType.forex
instrument = Instrument(symbol, asset_type=asset_type, broker=broker, currency=currency)
fromDate = datetime.datetime.today() - datetime.timedelta(days=40)
toDate = datetime.datetime.today()
periodSplit = Period.hour
dataframe = self.market_data_object.download(instrument, periodSplit, 1, fromDate, toDate)
self.assertIsNotNone(dataframe)
self.assertGreater(len(dataframe), 1)
pass
@unittest.skip # no reason needed
def test_download_equity(self):
symbol = 'IBM'
broker = Broker.manual_email
currency = Currency.usd
asset_type = AssetType.equity
instrument = Instrument(symbol, asset_type=asset_type, broker=broker, currency=currency)
fromDate = datetime.datetime.today() - datetime.timedelta(days=40)
toDate = datetime.datetime.today()
periodSplit = Period.hour
dataframe = self.market_data_object.download(instrument, periodSplit, 1, fromDate, toDate)
self.assertIsNotNone(dataframe)
self.assertGreater(len(dataframe), 1)
pass
def test_download_wrong(self):
instrument = Instrument('AAPqwL',
asset_type=AssetType.us_equity,
currency=Currency.usd)
downloadTest = self.fundamental_data_object.download(
instrument,
fundamental_ratio=Ratio.fundamental_ebitda_Y,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNone(downloadTest)
class BarDaoTestCase(unittest.TestCase):
symbol = 'aapl_test'
instrument = Instrument(symbol, 'quandl')
period = Period.hour
barDataframe = None
dao_test = None
collection = None
def setUp(self):
self.dao_test = BarDao(user_settings_tests)
self.barDataframe = self.generateRandomData()
def tearDown(self):
self.dao_test.delete(instrument=self.instrument,
period=self.period,
number_of_periods=1)
def generateRandomData(self):
import numpy as np
import pandas as pd
import datetime
columns = Bar.columns
output = pd.DataFrame(np.random.rand(20, len(columns)),
columns=columns)
base = datetime.datetime.today()
date_list = [base - datetime.timedelta(days=x) for x in range(0, 20)]
output[Bar.index] = date_list
output.set_index(Bar.index, inplace=True)
return output
def test_save_load(self):
initial_size = self.dao_test.load(instrument=self.instrument,
period=self.period,
number_of_periods=1)
if initial_size is None:
initial_size = 0
self.dao_test.save(self.barDataframe,
instrument=self.instrument,
period=self.period,
number_of_periods=1)
dataframeLoaded = self.dao_test.load(instrument=self.instrument,
period=self.period,
number_of_periods=1)
self.assertIsNotNone(dataframeLoaded)
final_size = len(dataframeLoaded)
self.assertEqual(initial_size + len(self.barDataframe), final_size,
'incorrect size after save')
def test___getInputKeys__(self):
import time
pk = self.vectorized_data_service.__getInputKeys__(instrumentList=self.instrumentList, fromDate=self.fromDate,
toDate=self.toDate)
time.sleep(1)
pk1 = self.vectorized_data_service.__getInputKeys__(instrumentList=self.instrumentList, fromDate=self.fromDate,
toDate=self.toDate)
time.sleep(1)
instrumentList2 = [
Instrument(symbol='ABC', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='ACN', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAPL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='GOOGL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='MSFT', asset_type=AssetType.us_equity, currency=Currency.usd),
]
pk2 = self.vectorized_data_service.__getInputKeys__(instrumentList=instrumentList2, fromDate=self.fromDate,
toDate=self.toDate)
self.assertEqual(pk, pk2)
class RatioDataServiceTestCase(unittest.TestCase):
user_settings = user_settings_tests
ratio_fundamental_service = RatioDataService(user_settings)
instrument = Instrument(symbol='AAPL',
asset_type=AssetType.us_equity,
currency=Currency.usd)
ratio_fundamental = Ratio.fundamental_ebitda_Y
ratio_quant = Ratio.quant_returnsDiff_120
fromDate = datetime.datetime(year=2017, day=1, month=1)
toDate = datetime.datetime(year=2018, day=26, month=6)
def test_getRatio_fund(self):
# clean first
data = self.ratio_fundamental_service.getRatioData(
self.instrument,
ratio=self.ratio_fundamental,
fromDate=self.fromDate,
toDate=self.toDate,
force_download=True)
self.assertIsNotNone(data)
def test_getRatio_quant(self):
# clean first
data = self.ratio_fundamental_service.getRatioData(
self.instrument,
ratio=self.ratio_quant,
fromDate=self.fromDate,
toDate=self.toDate,
force_download=True)
self.assertIsNotNone(data)
def test_getRatioList_fund(self):
# clean first
data = self.ratio_fundamental_service.getRatioDataBatch(
self.instrument,
ratio_list=[
Ratio.fundamental_ebit_Y, Ratio.fundamental_net_income_Y
],
fromDate=self.fromDate,
toDate=self.toDate,
force_download=True)
self.assertIsNotNone(data)
self.assertEqual(len(data.columns), 2)
def getInstrument(self, filename):
import os
filenameArray = filename.split(os.sep)
lastPart = filenameArray[-1]
nameFile = os.path.splitext(lastPart)[0]
nameFileList = nameFile.split('_')
symbol = nameFileList[0]
currency = nameFileList[1]
if currency == Currency.eur:
assetType = AssetType.es_equity
elif currency == Currency.usd:
assetType = AssetType.us_equity
instrument = Instrument(symbol=symbol,
asset_type=assetType,
currency=currency)
return instrument
def test_download_equity(self):
symbol = 'IBM'
broker = Broker.manual_email
currency = Currency.usd
asset_type = AssetType.equity
instrument = Instrument(symbol, asset_type=asset_type, broker=broker, currency=currency)
fromDate = datetime.datetime.today() - datetime.timedelta(days=40)
toDate = datetime.datetime.today()
periodSplit = Period.hour
dataframe = self.market_data_object.download(instrument, periodSplit, 1, fromDate, toDate)
self.assertIsNotNone(dataframe)
self.assertGreater(len(dataframe), 1)
pass
def __getInstrumentData__(self, instrument_symbol, instrument_currency,
instrument_assetType, columnList, fromDate,
toDate, outputDict, wrongInstrumentsSymbolLists):
instrument = Instrument(symbol=instrument_symbol,
currency=instrument_currency,
asset_type=instrument_assetType)
if self.useFunctionTemp:
functionTemp = self.cacher.cache(
self.___getInstrumentData___,
ignore=['self', 'outputDict', 'wrongInstrumentsSymbolLists'])
columnList.sort()
functionTemp(instrument, columnList, fromDate, toDate, outputDict,
wrongInstrumentsSymbolLists)
else:
# logger.debug('Not using functionTemp cache in __getInstrumentData__ in vector_service for %s' % instrument)
self.___getInstrumentData___(instrument, columnList, fromDate,
toDate, outputDict,
wrongInstrumentsSymbolLists)
class DukasCopyHistoricalDataTestCase(unittest.TestCase):
symbol = 'EUR'
broker = Broker.manual_email
currency = Currency.usd
instrument = Instrument(symbol=symbol, asset_type=AssetType.forex, broker=broker, currency=currency)
fromDate = datetime.datetime(year=2018, day=1, month=6)
fromDate_tick = datetime.datetime(year=2018, day=26, month=6)
toDate = datetime.datetime(year=2018, day=26, month=6)
user_settings = user_settings_tests
market_data_object = DukasCopyFileHistoricalMarketData(user_settings)
def setUp(self):
import os
os.environ[
"TRADEA_DUKASCOPY_INPUT_PATH"] = "D:\javif\Coding\Python\AInvesting\tradeasystems_connector\tests\historical_market_data\dukascopy_test"
pass
def tearDown(self):
import os
filesInDirectory = glob.glob(
self.user_settings.dukascopy_source_folder + os.sep + "*" + self.market_data_object.processedFilesPattern)
for processedFiles in filesInDirectory:
os.rename(processedFiles, processedFiles[:-len(self.market_data_object.processedFilesPattern)])
pass
@unittest.skip # no reason needed
def test_download_data_tick(self):
dataframe = self.market_data_object.download(self.instrument, period=Period.tick,
number_of_periods=1, fromDate=self.fromDate_tick,
toDate=self.toDate)
self.assertIsNotNone(dataframe)
@unittest.skip # no reason needed
def test_download_data_day(self):
dataframe = self.market_data_object.download(self.instrument, period=Period.day,
number_of_periods=1, fromDate=self.fromDate, toDate=self.toDate)
self.assertIsNotNone(dataframe)
def test_getHistorical__dailyClean(self):
# clean first
# self.historicalMarketService.deleteBar(instrument=self.instrument, period=self.period,
# number_of_periods=self.number_of_periods)
import datetime
instrument = Instrument(symbol='AAPL',
broker=Broker.manual_email,
asset_type=AssetType.us_equity,
currency=Currency.usd)
from_date = datetime.datetime(year=2010, month=1, day=1)
to_date = datetime.datetime(year=2012, month=1, day=1)
period = Period.day
number_of_periods = 1
data = self.historicalMarketService.getHistoricalData(
instrument,
period=period,
number_of_periods=number_of_periods,
fromDate=from_date,
toDate=to_date,
force_download=True,
endOfDayData=True)
self.assertIsNotNone(data)
class EmailConnectorTestCase(unittest.TestCase):
symbol = 'MSFT'
broker = Broker.manual_email
currency = Currency.usd
asset_type = AssetType.equity
instrument = Instrument(symbol,
asset_type=asset_type,
broker=broker,
currency=currency)
order = Order(instrument,
666,
Side.buy,
10.66,
order_type=OrderType.fill_or_kill)
email_connector = EmailConnector(user_settings_tests)
@unittest.skip # no reason needed
def test_send_buy_cancel(self):
self.email_connector.sendOrder(self.order)
# self.email_connector.cancelOrder(self.order)
@unittest.skip # no reason needed
def test_send_html(self):
text = ""
html = """\
<html>
<head></head>
<body>
<p>Hi!<br>
How are you?<br>
Here is the <a href="http://www.python.org">link</a> you wanted.
</p>
</body>
</html>
"""
self.email_connector.__sendEmail__(
recipient=user_settings_tests.email_notify,
subject='test',
body=text,
html=html)
@unittest.skip # no reason needed
def test_ipython_to_html(self):
ipythonFile = 'test'
htmlPythonFile = getNotebookHTML(ipythonFile)
self.assertIsNotNone(htmlPythonFile)
@unittest.skip # no reason needed
def test_ipython_exec(self):
ipythonFile = 'test'
result = updateNotebook(ipythonFile)
self.assertIsNotNone(result)
self.assertTrue(result)
@unittest.skip # no reason needed
def test_sendNoteBook(self):
ipythonFile = 'test'
self.email_connector.sendNotebook(
recipient=user_settings_tests.email_notify,
notebookName=ipythonFile)
class VectorDaoTestCase(unittest.TestCase):
symbol = 'aapl_test'
instrument = Instrument(symbol, 'quandl')
period = Period.hour
vectorDataframe = None
dao_test = None
collection = None
columnList = None
def setUp(self):
self.dao_test = VectorDao(user_settings_tests)
self.vectorDataframe = self.generateRandomData()
def tearDown(self):
self.dao_test.delete(instrument=self.instrument)
def generateRandomData(self):
import numpy as np
import pandas as pd
import datetime
columns = DataDictKeys.keys.copy()
columns += ([Ratio.fundamental_ebit_Y, Ratio.quant_std1Y])
self.columnList = columns
output = pd.DataFrame(np.random.rand(20, len(columns)),
columns=columns)
base = datetime.datetime.today().replace(hour=0,
minute=0,
second=0,
microsecond=0)
date_list = [base - datetime.timedelta(days=x) for x in range(0, 20)]
output[DataDictKeys.index] = date_list
output.set_index(DataDictKeys.index, inplace=True)
return output
def generateRandomData2(self):
import numpy as np
import pandas as pd
import datetime
columns = DataDictKeys.keys.copy()
columns += ([
Ratio.fundamental_ebit_Y, Ratio.quant_std1Y,
Ratio.fundamental_shares_Y
])
output = pd.DataFrame(np.random.rand(30, len(columns)),
columns=columns)
base = datetime.datetime.today().replace(hour=0,
minute=0,
second=0,
microsecond=0)
date_list = [base - datetime.timedelta(days=x) for x in range(0, 30)]
output[DataDictKeys.index] = date_list
output.set_index(DataDictKeys.index, inplace=True)
return output
def test_save_load(self):
initial_size = self.dao_test.load(instrument=self.instrument,
columnList=self.columnList)
if initial_size is None:
initial_size = 0
isSave = self.dao_test.save(self.vectorDataframe,
instrument=self.instrument)
self.assertTrue(isSave)
dataframeLoaded = self.dao_test.load(instrument=self.instrument,
columnList=self.columnList)
self.assertIsNotNone(dataframeLoaded)
final_size = len(dataframeLoaded)
self.assertEqual(initial_size + len(self.vectorDataframe), final_size,
'incorrect size after save')
def test_save_append(self):
initial_size = self.dao_test.load(instrument=self.instrument,
columnList=self.columnList)
if initial_size is None:
initial_size = 0
else:
initial_size = len(initial_size)
isSave = self.dao_test.save(self.vectorDataframe,
instrument=self.instrument)
self.assertTrue(isSave)
dataframeLoaded = self.dao_test.load(instrument=self.instrument,
columnList=self.columnList)
self.assertIsNotNone(dataframeLoaded)
middle_size = len(dataframeLoaded)
self.assertEqual(initial_size + len(self.vectorDataframe), middle_size,
'incorrect size after save')
newVector = self.generateRandomData2()
isSave = self.dao_test.save(newVector, instrument=self.instrument)
self.assertTrue(isSave)
dataframeLoaded = self.dao_test.load(instrument=self.instrument,
columnList=self.columnList)
self.assertIsNotNone(dataframeLoaded)
end_size = len(dataframeLoaded)
self.assertEqual(initial_size + len(newVector), end_size,
'incorrect size after save')
class QuandlFundamentalCalculatorTestCase(unittest.TestCase):
instrument = Instrument('AAPL', asset_type=AssetType.us_equity, currency=Currency.usd)
user_settings = user_settings_tests
fundamental_data_object = QuandlFundamentalData(user_settings)
fromDate = datetime.datetime(year=2017, day=1, month=1)
toDate = datetime.datetime(year=2018, day=26, month=6)
fundamental_period = FundamentalPeriod.yearly
def test_getAllBatch(self):
import time
start = time.time()
fromDate = datetime.datetime(year=2010, day=1, month=1)
df_batch = self.fundamental_data_object.downloadBatch(instrument=self.instrument,
fundamental_ratio_list=Ratio.fundamental_list_Y,
fromDate=fromDate, toDate=self.toDate)
end = time.time()
secondsElapsed = end - start
print('Get all fundamentals of %s took %d seconds' % (self.instrument, secondsElapsed))
self.assertIsNotNone(df_batch)
for ratio in Ratio.fundamental_list_Y:
if ratio not in list(df_batch.columns):
print('Ratio %s is not in output!' % ratio)
self.assertTrue(ratio in list(df_batch.columns))
self.assertEqual(len(Ratio.fundamental_list_Y) + 1, len(df_batch.columns)) # +1 because of close_temp
self.assertLess(secondsElapsed, 30) # less 30 seconds per instrument
def test_getROABatch(self):
import time
start = time.time()
fromDate = datetime.datetime(year=2010, day=1, month=1)
df_batch = self.fundamental_data_object.downloadBatch(instrument=self.instrument, fundamental_ratio_list=[
Ratio.fundamental_return_over_assets_Y], fromDate=fromDate, toDate=self.toDate)
end = time.time()
secondsElapsed = end - start
print('test_getROABatch of %s took %d seconds' % (self.instrument, secondsElapsed))
self.assertIsNotNone(df_batch)
self.assertEqual(1, len(df_batch.columns))
def test_getBVBatch(self):
import time
start = time.time()
fromDate = datetime.datetime(year=2010, day=1, month=1)
df_batch = self.fundamental_data_object.downloadBatch(instrument=self.instrument, fundamental_ratio_list=[
Ratio.fundamental_book_value_per_share_Y], fromDate=fromDate, toDate=self.toDate)
end = time.time()
secondsElapsed = end - start
print('test_getBVBatch of %s took %d seconds' % (self.instrument, secondsElapsed))
self.assertIsNotNone(df_batch)
self.assertEqual(1, len(df_batch.columns))
def test_getBV_BVLiabBatch(self):
import time
start = time.time()
fromDate = datetime.datetime(year=2010, day=1, month=1)
df_batch = self.fundamental_data_object.downloadBatch(instrument=self.instrument, fundamental_ratio_list=[
Ratio.fundamental_book_value_liabilities_Y, Ratio.fundamental_book_value_per_share_Y], fromDate=fromDate,
toDate=self.toDate)
end = time.time()
secondsElapsed = end - start
print('test_getBV_BVLiabBatch of %s took %d seconds' % (self.instrument, secondsElapsed))
self.assertIsNotNone(df_batch)
self.assertEqual(3, len(df_batch.columns))
def test_getAllBatch_wrong(self):
fromDate = datetime.datetime(year=2010, day=1, month=1)
instrument = Instrument('A123APL', asset_type=AssetType.us_equity, currency=Currency.usd)
df_batch = self.fundamental_data_object.downloadBatch(instrument=instrument,
fundamental_ratio_list=Ratio.fundamental_list_Y,
fromDate=fromDate, toDate=self.toDate)
self.assertIsNone(df_batch)
def test_getCapitalization(self):
capitalization = getCapitalization(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period, fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(capitalization)
def test_getNetAssetValue(self):
nav = getNetAssetValue(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period)
self.assertIsNotNone(nav)
def test_getAdjustedBookValue(self):
adj_book_value = getAdjustedBookValue(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period)
self.assertIsNotNone(adj_book_value)
def test_getDaySalesReceivables(self):
day_sales_receivables = getDaySalesReceivables(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period)
self.assertIsNotNone(day_sales_receivables)
def test_getEnterpriseValue(self):
ev = getEnterpriseValue(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period)
self.assertIsNotNone(ev)
def test_getReturnOverAssets(self):
roa = getReturnOverAssets(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period, fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(roa)
def test_getOperatingAssets(self):
op_assets = getOperatingAssets(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period)
self.assertIsNotNone(op_assets)
def test_getReturnOverCapital(self):
roc = getReturnOverCapital(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period, fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(roc)
def test_getSalesGrowth(self):
sales_growth = getSalesGrowth(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period)
self.assertIsNotNone(sales_growth)
def test_getBookValueLiabilities(self):
bv_liabilities = getBookValueLiabilities(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period)
self.assertIsNotNone(bv_liabilities)
def test_getLongTermDebt(self):
lt_debt = getLongTermDebt(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period)
self.assertIsNotNone(lt_debt)
def test_getShortTermDebt(self):
st_debt = getShortTermDebt(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period)
self.assertIsNotNone(st_debt)
def test_getWrongInstrument(self):
instrument_wrong = Instrument('A123APL', asset_type=AssetType.us_equity, currency=Currency.usd)
st_debt = getShortTermDebt(self.fundamental_data_object, instrument=instrument_wrong,
fundamental_period=self.fundamental_period)
self.assertIsNone(st_debt)
def test_getAccrual(self):
data = getAccrual(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period, fromDate=self.fromDate, toDate=self.toDate)
self.assertIsNotNone(data)
def test_getDaySalesReceivablesIndex(self):
data = getDaySalesReceivablesIndex(self.fundamental_data_object, instrument=self.instrument,
fundamental_period=self.fundamental_period, fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(data)
def test_get4InstrumentCompleteDeleting(self):
import time
instrumentList = [
Instrument(symbol='A', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAAL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAPL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='ABT', asset_type=AssetType.us_equity, currency=Currency.usd),
]
for instrument in instrumentList:
self.vector_service.vectorDao.delete(instrument)
start = time.time()
dictPricing = {}
for instrument in instrumentList:
self.vector_service.vectorizedDataService.historicalMarketDataService.deleteBar(instrument=instrument,
period=Period.day,
number_of_periods=1)
self.vector_service.vectorizedDataService.__downloadHistoricalPrices__(wrongInstrumentsSymbolLists=[],
outputDict=dictPricing,
fromDate=datetime.datetime(year=2010,
day=1,
month=1),
toDate=self.toDate,
instrument=instrument
)
dict_of_matrix = self.vector_service.getDataDictOfMatrix(instrumentList=instrumentList,
ratioList=Ratio.allRatios,
fromDate=datetime.datetime(year=2010, day=1, month=1),
toDate=self.toDate,
)
end = time.time()
elapsed = end - start
print('1-Took %d seconds to complete 4 instrument downloading' % (elapsed))
self.assertIsNotNone(dict_of_matrix)
self.assertEqual(len(Ratio.allRatios) + 5, len(dict_of_matrix))
keysOutput = dict_of_matrix.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
for key in self.ratio_list:
self.assertEqual(key in keysOutput, True)
# %% from DDBB
start2 = time.time()
self.vector_service.force_download = False
dict_of_matrix_2 = self.vector_service.getDataDictOfMatrix(instrumentList=instrumentList,
ratioList=Ratio.allRatios,
fromDate=datetime.datetime(year=2010, day=1,
month=1),
toDate=self.toDate,
)
end2 = time.time()
elapsed2 = end2 - start2
self.vector_service.force_download = True
print('2-Took %d seconds to complete an instrument from DDBB' % (elapsed2))
self.assertIsNotNone(dict_of_matrix_2)
self.assertEqual(len(Ratio.allRatios) + 5, len(dict_of_matrix_2))
keysOutput = dict_of_matrix_2.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
for key in self.ratio_list:
self.assertEqual(key in keysOutput, True)
# %%
for key in dict_of_matrix.keys():
print('testing equality in %s' % key)
df = dict_of_matrix[key]
df2 = dict_of_matrix_2[key]
self.assertEqual(len(df), len(df2))
sum1 = df.sum().sum()
sum2 = df2.sum().sum()
self.assertEqual(sum1, sum2)
df_dropna = df.dropna()
self.assertFalse(df_dropna.empty)
# %%
for key in dictPricing.keys():
if key == 'wrong':
continue
for symbol in dict_of_matrix[key].columns:
print('testing pricing equality in %s:%s' % (key, symbol))
isBusinessDay = BDay().onOffset
match_series = pd.to_datetime(dictPricing[key].index).map(isBusinessDay)
dictPricing[key] = dictPricing[key][match_series]
dictPricing[key].fillna(method='ffill', inplace=True)
df = dict_of_matrix[key][symbol]
df_pricing = dictPricing[key][symbol]
# self.assertEqual(len(df), len(df_pricing.dropna()))
sum1 = df.sum()
sum2 = df_pricing.sum()
self.assertEqual(sum1, sum2)
lastSum = None
for fundamental_key in Ratio.allRatios:
sum = dict_of_matrix[fundamental_key].sum().sum()
if lastSum is not None:
self.assertNotEquals(sum, lastSum)
lastSum = sum
def test_getWrongInstrument(self):
instrument_wrong = Instrument('A123APL', asset_type=AssetType.us_equity, currency=Currency.usd)
st_debt = getShortTermDebt(self.fundamental_data_object, instrument=instrument_wrong,
fundamental_period=self.fundamental_period)
self.assertIsNone(st_debt)
class VectorServiceTestCase(unittest.TestCase):
user_settings = user_settings_tests
vector_service = VectorService(user_settings)
instrumentList = [
Instrument(symbol='ABC', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='ACN', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAPL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='GOOGL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='MSFT', asset_type=AssetType.us_equity, currency=Currency.usd),
]
ratio_list = [Ratio.fundamental_ebitda_Y, Ratio.fundamental_enterprise_value_Y, Ratio.fundamental_ms_Y]
fromDate = datetime.datetime(year=2017, day=1, month=1)
toDate = datetime.datetime(year=2018, day=26, month=6)
def test_getOneInstrumentCompleteDeleting(self):
import time
instrument = self.instrumentList[2]
self.vector_service.vectorDao.delete(instrument)
self.vector_service.useFunctionTemp = False
start = time.time()
dict_of_matrix = self.vector_service.getDataDictOfMatrix(instrumentList=[instrument],
ratioList=Ratio.allRatios,
fromDate=datetime.datetime(year=2010, day=1, month=1),
toDate=self.toDate,
)
end = time.time()
elapsed = end - start
print('1-Took %d seconds to complete an instrument downloading' % (elapsed))
self.assertIsNotNone(dict_of_matrix)
self.assertEqual(len(Ratio.allRatios) + 5, len(dict_of_matrix))
keysOutput = dict_of_matrix.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
for key in self.ratio_list:
self.assertEqual(key in keysOutput, True)
self.assertLess(elapsed, 40)
# %% from DDBB
start2 = time.time()
self.vector_service.force_download = False
dict_of_matrix_2 = self.vector_service.getDataDictOfMatrix(instrumentList=[instrument],
ratioList=Ratio.allRatios,
fromDate=datetime.datetime(year=2010, day=1,
month=1),
toDate=self.toDate,
)
end2 = time.time()
elapsed2 = end2 - start2
self.vector_service.force_download = True
print('2-Took %d seconds to complete an instrument from DDBB' % (elapsed2))
self.assertIsNotNone(dict_of_matrix_2)
self.assertEqual(len(Ratio.allRatios) + 5, len(dict_of_matrix_2))
keysOutput = dict_of_matrix_2.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
for key in self.ratio_list:
self.assertEqual(key in keysOutput, True)
self.assertLess(elapsed2, 10)
# %%
for key in dict_of_matrix.keys():
df = dict_of_matrix[key]
df2 = dict_of_matrix_2[key]
self.assertEqual(len(df), len(df2))
sum1 = df.sum().sum()
sum2 = df2.sum().sum()
self.assertEqual(sum1, sum2)
df_dropna = df.dropna()
self.assertFalse(df_dropna.empty)
def test_get4InstrumentCompleteDeleting(self):
import time
instrumentList = [
Instrument(symbol='A', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAAL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAPL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='ABT', asset_type=AssetType.us_equity, currency=Currency.usd),
]
for instrument in instrumentList:
self.vector_service.vectorDao.delete(instrument)
start = time.time()
dictPricing = {}
for instrument in instrumentList:
self.vector_service.vectorizedDataService.historicalMarketDataService.deleteBar(instrument=instrument,
period=Period.day,
number_of_periods=1)
self.vector_service.vectorizedDataService.__downloadHistoricalPrices__(wrongInstrumentsSymbolLists=[],
outputDict=dictPricing,
fromDate=datetime.datetime(year=2010,
day=1,
month=1),
toDate=self.toDate,
instrument=instrument
)
dict_of_matrix = self.vector_service.getDataDictOfMatrix(instrumentList=instrumentList,
ratioList=Ratio.allRatios,
fromDate=datetime.datetime(year=2010, day=1, month=1),
toDate=self.toDate,
)
end = time.time()
elapsed = end - start
print('1-Took %d seconds to complete 4 instrument downloading' % (elapsed))
self.assertIsNotNone(dict_of_matrix)
self.assertEqual(len(Ratio.allRatios) + 5, len(dict_of_matrix))
keysOutput = dict_of_matrix.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
for key in self.ratio_list:
self.assertEqual(key in keysOutput, True)
# %% from DDBB
start2 = time.time()
self.vector_service.force_download = False
dict_of_matrix_2 = self.vector_service.getDataDictOfMatrix(instrumentList=instrumentList,
ratioList=Ratio.allRatios,
fromDate=datetime.datetime(year=2010, day=1,
month=1),
toDate=self.toDate,
)
end2 = time.time()
elapsed2 = end2 - start2
self.vector_service.force_download = True
print('2-Took %d seconds to complete an instrument from DDBB' % (elapsed2))
self.assertIsNotNone(dict_of_matrix_2)
self.assertEqual(len(Ratio.allRatios) + 5, len(dict_of_matrix_2))
keysOutput = dict_of_matrix_2.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
for key in self.ratio_list:
self.assertEqual(key in keysOutput, True)
# %%
for key in dict_of_matrix.keys():
print('testing equality in %s' % key)
df = dict_of_matrix[key]
df2 = dict_of_matrix_2[key]
self.assertEqual(len(df), len(df2))
sum1 = df.sum().sum()
sum2 = df2.sum().sum()
self.assertEqual(sum1, sum2)
df_dropna = df.dropna()
self.assertFalse(df_dropna.empty)
# %%
for key in dictPricing.keys():
if key == 'wrong':
continue
for symbol in dict_of_matrix[key].columns:
print('testing pricing equality in %s:%s' % (key, symbol))
isBusinessDay = BDay().onOffset
match_series = pd.to_datetime(dictPricing[key].index).map(isBusinessDay)
dictPricing[key] = dictPricing[key][match_series]
dictPricing[key].fillna(method='ffill', inplace=True)
df = dict_of_matrix[key][symbol]
df_pricing = dictPricing[key][symbol]
# self.assertEqual(len(df), len(df_pricing.dropna()))
sum1 = df.sum()
sum2 = df_pricing.sum()
self.assertEqual(sum1, sum2)
lastSum = None
for fundamental_key in Ratio.allRatios:
sum = dict_of_matrix[fundamental_key].sum().sum()
if lastSum is not None:
self.assertNotEquals(sum, lastSum)
lastSum = sum
def test_checkPricing(self):
import time
instrumentList = [
Instrument(symbol='A', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAAL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAPL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='ABT', asset_type=AssetType.us_equity, currency=Currency.usd),
]
for instrument in instrumentList:
self.vector_service.vectorDao.delete(instrument)
self.vector_service.vectorizedDataService.historicalMarketDataService.deleteBar(instrument=instrument,
period=Period.day,
number_of_periods=1)
start = time.time()
dictPricing = {}
wrongInstrumentsSymbolLists = []
for instrument in instrumentList:
self.vector_service.vectorizedDataService.__downloadHistoricalPrices__(
wrongInstrumentsSymbolLists=wrongInstrumentsSymbolLists,
outputDict=dictPricing,
fromDate=datetime.datetime(year=2010,
day=1,
month=1),
toDate=self.toDate,
instrument=instrument
)
dict_of_matrix = self.vector_service.getDataDictOfMatrix(instrumentList=instrumentList,
ratioList=[],
fromDate=datetime.datetime(year=2010, day=1, month=1),
toDate=self.toDate,
)
end = time.time()
elapsed = end - start
print('1-Took %d seconds to complete 4 instrument downloading' % (elapsed))
self.assertIsNotNone(dict_of_matrix)
self.assertEqual(5, len(dict_of_matrix))
keysOutput = dict_of_matrix.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
# %%
for key in dictPricing.keys():
if key == 'wrong':
continue
for symbol in dict_of_matrix[key].columns:
print('testing pricing equality in %s:%s' % (key, symbol))
isBusinessDay = BDay().onOffset
match_series = pd.to_datetime(dictPricing[key].index).map(isBusinessDay)
dictPricing_df = dictPricing[key][match_series].copy()
dictPricing_df.fillna(method='ffill', inplace=True)
df = dict_of_matrix[key][symbol]
df_pricing = dictPricing_df[symbol]
# self.assertEqual(len(df), len(df_pricing.dropna()))
sum1 = df.sum()
sum2 = df_pricing.sum()
# self.assertEqual(sum1, sum2)
def test_getDict(self):
dict_of_matrix = self.vector_service.getDataDictOfMatrix(instrumentList=self.instrumentList,
ratioList=self.ratio_list,
fromDate=self.fromDate,
toDate=self.toDate,
)
self.assertIsNotNone(dict_of_matrix)
self.assertEqual(8, len(dict_of_matrix))
keysOutput = dict_of_matrix.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
for key in self.ratio_list:
self.assertEqual(key in keysOutput, True)
@unittest.skip(reason='slow')
def test_getAllDict(self):
ratio_list = Ratio.fundamental_list_Y
ratio_list += (Ratio.quant_list)
dict_of_matrix = self.vector_service.getDataDictOfMatrix(instrumentList=self.instrumentList,
ratioList=ratio_list,
fromDate=self.fromDate,
toDate=self.toDate
)
self.assertIsNotNone(dict_of_matrix)
keysOutput = dict_of_matrix.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
for key in ratio_list:
self.assertEqual(key in keysOutput, True)
@unittest.skip(reason='slow , and not yet')
def test_getAllDict_timeIt(self):
import time
ratio_list = [Ratio.fundamental_shares_Y, Ratio.fundamental_accrual_Y]
startTemp = time.time()
dict_of_matrix_normal = self.vector_service.getDataDictOfMatrix(instrumentList=self.instrumentList,
ratioList=ratio_list,
fromDate=self.fromDate,
toDate=self.toDate
)
endTemp = time.time()
print('TEMP::: Dict matrix of %d ratios of %d symbols took %d seconds ' % (
len(ratio_list), len(self.instrumentList), endTemp - startTemp))
self.assertIsNotNone(dict_of_matrix_normal)
start = time.time()
dict_of_matrix_notTemp = self.vector_service.getDataDictOfMatrix(instrumentList=self.instrumentList,
ratioList=ratio_list,
fromDate=self.fromDate,
toDate=self.toDate,
useFunctionTemp=False
)
end = time.time()
timeInSeconds = end - start
print('Dict matrix of %d ratios of %d symbols took %d seconds ' % (
len(ratio_list), len(self.instrumentList), timeInSeconds))
self.assertIsNotNone(dict_of_matrix_notTemp)
self.assertEqual(dict_of_matrix_normal.keys(), dict_of_matrix_notTemp.keys())
self.assertEqual(dict_of_matrix_normal[DataDictKeys.close].shape,
dict_of_matrix_notTemp[DataDictKeys.close].shape)
self.assertEqual(dict_of_matrix_normal[Ratio.fundamental_shares_Y].shape,
dict_of_matrix_notTemp[Ratio.fundamental_shares_Y].shape)
class QuandlFundamentalDataTestCase(unittest.TestCase):
instrument = Instrument('AAPL',
asset_type=AssetType.us_equity,
currency=Currency.usd)
user_settings = user_settings_tests
fundamental_data_object = QuandlFundamentalData(user_settings)
fromDate = datetime.datetime(year=2017, day=1, month=1)
toDate = datetime.datetime(year=2018, day=26, month=6)
@unittest.skip("deprecated class")
def test_download_data(self):
downloadTest = self.fundamental_data_object.download(
self.instrument,
fundamental_ratio=Ratio.fundamental_ebitda_Y,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(downloadTest)
self.assertNotEquals(len(downloadTest), 0)
def test_download_data_batch(self):
downloadTest = self.fundamental_data_object.downloadBatch(
self.instrument,
fundamental_ratio_list=[
Ratio.fundamental_ebitda_Y, Ratio.fundamental_assets_Y
],
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(downloadTest)
self.assertNotEquals(len(downloadTest), 0)
self.assertEquals(len(downloadTest.columns), 2)
@unittest.skip("deprecated class")
def test_calculate_download_data(self):
downloadTest = self.fundamental_data_object.download(
self.instrument,
fundamental_ratio=Ratio.fundamental_enterprise_value_Y,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(downloadTest)
self.assertNotEquals(len(downloadTest), 0)
@unittest.skip("deprecated class")
def test_download_wrong(self):
instrument = Instrument('AAPqwL',
asset_type=AssetType.us_equity,
currency=Currency.usd)
downloadTest = self.fundamental_data_object.download(
instrument,
fundamental_ratio=Ratio.fundamental_ebitda_Y,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNone(downloadTest)
@unittest.skip("deprecated class")
def test_calculate_gross_margin(self):
downloadTest = self.fundamental_data_object.download(
self.instrument,
fundamental_ratio=Ratio.fundamental_gross_margin_Y,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(downloadTest)
self.assertNotEquals(len(downloadTest), 0)
self.assertTrue(Ratio.ratio in downloadTest.columns)
self.assertEqual(len(downloadTest.columns), 1)
@unittest.skip("deprecated class")
def test_all_fundamentals(self):
ratioList = Ratio.fundamental_list_Y
for ratio in ratioList:
print('testing ratio %s ' % ratio)
downloadTest = self.fundamental_data_object.download(
self.instrument,
fundamental_ratio=ratio,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(downloadTest)
self.assertNotEquals(len(downloadTest), 0)
self.assertTrue(Ratio.ratio in downloadTest.columns)
self.assertEqual(len(downloadTest.columns), 1)
class VectorizedDataServiceTestCase(unittest.TestCase):
user_settings = user_settings_tests
vectorized_data_service = VectorOnlineService(user_settings)
instrumentList = [
Instrument(symbol='ABC', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='ACN', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAPL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='GOOGL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='MSFT', asset_type=AssetType.us_equity, currency=Currency.usd),
]
ratio_list = [Ratio.fundamental_ebitda_Y, Ratio.fundamental_enterprise_value_Y, Ratio.fundamental_ms_Y]
fromDate = datetime.datetime(year=2017, day=1, month=1)
toDate = datetime.datetime(year=2018, day=26, month=6)
@unittest.skip('not used anymore')
def test_getDict(self):
dict_of_matrix = self.vectorized_data_service.getDataDictOfMatrix(instrumentList=self.instrumentList,
ratioList=self.ratio_list,
fromDate=self.fromDate,
toDate=self.toDate,
)
self.assertIsNotNone(dict_of_matrix)
self.assertEqual(8, len(dict_of_matrix))
keysOutput = dict_of_matrix.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
for key in self.ratio_list:
self.assertEqual(key in keysOutput, True)
@unittest.skip(reason='slow')
def test_getAllDict(self):
ratio_list = Ratio.fundamental_list_Y
ratio_list += (Ratio.quant_list)
dict_of_matrix = self.vectorized_data_service.getDataDictOfMatrix(instrumentList=self.instrumentList,
ratioList=ratio_list,
fromDate=self.fromDate,
toDate=self.toDate
)
self.assertIsNotNone(dict_of_matrix)
keysOutput = dict_of_matrix.keys()
for key in DataDictKeys.keys:
self.assertEqual(key in keysOutput, True)
for key in ratio_list:
self.assertEqual(key in keysOutput, True)
@unittest.skip(reason='slow , and not yet')
def test_getAllDict_timeIt(self):
import time
ratio_list = [Ratio.fundamental_shares_Y, Ratio.fundamental_accrual_Y]
startTemp = time.time()
dict_of_matrix_normal = self.vectorized_data_service.getDataDictOfMatrix(instrumentList=self.instrumentList,
ratioList=ratio_list,
fromDate=self.fromDate,
toDate=self.toDate
)
endTemp = time.time()
print('TEMP::: Dict matrix of %d ratios of %d symbols took %d seconds ' % (
len(ratio_list), len(self.instrumentList), endTemp - startTemp))
self.assertIsNotNone(dict_of_matrix_normal)
start = time.time()
dict_of_matrix_notTemp = self.vectorized_data_service.getDataDictOfMatrix(instrumentList=self.instrumentList,
ratioList=ratio_list,
fromDate=self.fromDate,
toDate=self.toDate,
useFunctionTemp=False
)
end = time.time()
timeInSeconds = end - start
print('Dict matrix of %d ratios of %d symbols took %d seconds ' % (
len(ratio_list), len(self.instrumentList), timeInSeconds))
self.assertIsNotNone(dict_of_matrix_notTemp)
self.assertEqual(dict_of_matrix_normal.keys(), dict_of_matrix_notTemp.keys())
self.assertEqual(dict_of_matrix_normal[DataDictKeys.close].shape,
dict_of_matrix_notTemp[DataDictKeys.close].shape)
self.assertEqual(dict_of_matrix_normal[Ratio.fundamental_shares_Y].shape,
dict_of_matrix_notTemp[Ratio.fundamental_shares_Y].shape)
## test time <10 seconds
# self.assertLess(timeInSeconds,10)
def test___getInputKeys__(self):
import time
pk = self.vectorized_data_service.__getInputKeys__(instrumentList=self.instrumentList, fromDate=self.fromDate,
toDate=self.toDate)
time.sleep(1)
pk1 = self.vectorized_data_service.__getInputKeys__(instrumentList=self.instrumentList, fromDate=self.fromDate,
toDate=self.toDate)
time.sleep(1)
instrumentList2 = [
Instrument(symbol='ABC', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='ACN', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAPL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='GOOGL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='MSFT', asset_type=AssetType.us_equity, currency=Currency.usd),
]
pk2 = self.vectorized_data_service.__getInputKeys__(instrumentList=instrumentList2, fromDate=self.fromDate,
toDate=self.toDate)
self.assertEqual(pk, pk2)
# check if parallel downloading is working
@unittest.skip('not used anymore')
def test_getDataDictOfMatrixParallel(self):
# ratio_list = Ratio.fundamental_list_Y
# symbolList = getSharadarTickers(self.user_settings)[20:30]
instrumentList2 = [
Instrument(symbol='ABC', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='ACN', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAPL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='GOOGL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='MSFT', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='GNW', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='SCG', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='KANG', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='S', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='A', asset_type=AssetType.us_equity, currency=Currency.usd),
]
ratio_list = [Ratio.fundamental_ebitda_Y,
Ratio.fundamental_enterprise_value_Y,
Ratio.fundamental_ms_Y,
# Ratio.quant_return1Y,
# Ratio.quant_return1YFrom20,
# Ratio.quant_returnsDiff_120,
# Ratio.quant_std1Y,
# Ratio.fundamental_asset_quality_index_Y,
# Ratio.fundamental_book_value_liabilities_Y,
# Ratio.fundamental_shares_Y
]
instrumentList = instrumentList2 # getInstrumentList(symbolList, asset_type=AssetType.us_equity, currency=Currency.usd)
fromDate = datetime.datetime(year=2015, day=1, month=1)
self.vectorized_data_service.parallelDownloadInstruments = False
self.vectorized_data_service.parallelDownloadRatios = False
self.vectorized_data_service.force_download = True
self.vectorized_data_service.threads = 8
dict_of_matrix_serial = self.vectorized_data_service.getDataDictOfMatrix(instrumentList=instrumentList,
ratioList=ratio_list,
fromDate=fromDate,
toDate=self.toDate,
)
self.assertIsNotNone(dict_of_matrix_serial)
closeDF = dict_of_matrix_serial[DataDictKeys.close]
self.assertIsNotNone(closeDF)
prices = closeDF
columnsRemove = list(prices.sum(axis=0)[prices.sum(axis=0) == 0].index)
if len(columnsRemove) > 0:
prices = prices.drop(columnsRemove, axis=1)
self.assertIsNotNone(prices)
self.assertTrue(len(prices) > 0)
self.vectorized_data_service.parallelDownloadInstruments = True
self.vectorized_data_service.parallelDownloadRatios = True
dict_of_matrix_parallel = self.vectorized_data_service.getDataDictOfMatrix(instrumentList=instrumentList,
ratioList=ratio_list,
fromDate=fromDate,
toDate=self.toDate
)
self.assertIsNotNone(dict_of_matrix_parallel)
self.assertEquals(len(dict_of_matrix_serial), len(dict_of_matrix_parallel))
for key in DataDictKeys.keys:
print('Checking %s' % key)
self.assertEquals(len(dict_of_matrix_serial[key]), len(dict_of_matrix_parallel[key]))
self.assertEquals(dict_of_matrix_serial[key].sum().sum(), dict_of_matrix_parallel[key].sum().sum())
for key in ratio_list:
print('Checking %s' % key)
self.assertEquals(len(dict_of_matrix_serial[key]), len(dict_of_matrix_parallel[key]))
self.assertEquals(dict_of_matrix_serial[key].sum().sum(), dict_of_matrix_parallel[key].sum().sum())
def test_getDataDictOfMatrixParallel(self):
# ratio_list = Ratio.fundamental_list_Y
# symbolList = getSharadarTickers(self.user_settings)[20:30]
instrumentList2 = [
Instrument(symbol='ABC', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='ACN', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='AAPL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='GOOGL', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='MSFT', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='GNW', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='SCG', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='KANG', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='S', asset_type=AssetType.us_equity, currency=Currency.usd),
Instrument(symbol='A', asset_type=AssetType.us_equity, currency=Currency.usd),
]
ratio_list = [Ratio.fundamental_ebitda_Y,
Ratio.fundamental_enterprise_value_Y,
Ratio.fundamental_ms_Y,
# Ratio.quant_return1Y,
# Ratio.quant_return1YFrom20,
# Ratio.quant_returnsDiff_120,
# Ratio.quant_std1Y,
# Ratio.fundamental_asset_quality_index_Y,
# Ratio.fundamental_book_value_liabilities_Y,
# Ratio.fundamental_shares_Y
]
instrumentList = instrumentList2 # getInstrumentList(symbolList, asset_type=AssetType.us_equity, currency=Currency.usd)
fromDate = datetime.datetime(year=2015, day=1, month=1)
self.vectorized_data_service.parallelDownloadInstruments = False
self.vectorized_data_service.parallelDownloadRatios = False
self.vectorized_data_service.force_download = True
self.vectorized_data_service.threads = 8
dict_of_matrix_serial = self.vectorized_data_service.getDataDictOfMatrix(instrumentList=instrumentList,
ratioList=ratio_list,
fromDate=fromDate,
toDate=self.toDate,
)
self.assertIsNotNone(dict_of_matrix_serial)
closeDF = dict_of_matrix_serial[DataDictKeys.close]
self.assertIsNotNone(closeDF)
prices = closeDF
columnsRemove = list(prices.sum(axis=0)[prices.sum(axis=0) == 0].index)
if len(columnsRemove) > 0:
prices = prices.drop(columnsRemove, axis=1)
self.assertIsNotNone(prices)
self.assertTrue(len(prices) > 0)
self.vectorized_data_service.parallelDownloadInstruments = True
self.vectorized_data_service.parallelDownloadRatios = True
dict_of_matrix_parallel = self.vectorized_data_service.getDataDictOfMatrix(instrumentList=instrumentList,
ratioList=ratio_list,
fromDate=fromDate,
toDate=self.toDate
)
self.assertIsNotNone(dict_of_matrix_parallel)
self.assertEquals(len(dict_of_matrix_serial), len(dict_of_matrix_parallel))
for key in DataDictKeys.keys:
print('Checking %s' % key)
self.assertEquals(len(dict_of_matrix_serial[key]), len(dict_of_matrix_parallel[key]))
self.assertEquals(dict_of_matrix_serial[key].sum().sum(), dict_of_matrix_parallel[key].sum().sum())
for key in ratio_list:
print('Checking %s' % key)
self.assertEquals(len(dict_of_matrix_serial[key]), len(dict_of_matrix_parallel[key]))
self.assertEquals(dict_of_matrix_serial[key].sum().sum(), dict_of_matrix_parallel[key].sum().sum())
class QuandlFundamentalDataCSVTestCase(unittest.TestCase):
instrument = Instrument('AAPL',
asset_type=AssetType.us_equity,
currency=Currency.usd)
user_settings = user_settings_tests
fundamental_data_object = QuandlFundamentalDataCSV(user_settings)
fromDate = datetime.datetime(year=2017, day=1, month=1)
toDate = datetime.datetime(year=2018, day=26, month=6)
@unittest.skip
def test_download_data(self):
downloadTest = self.fundamental_data_object.download(
self.instrument,
fundamental_ratio=Ratio.fundamental_ebitda_Y,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(downloadTest)
self.assertNotEquals(len(downloadTest), 0)
@unittest.skip
def test_calculate_download_data(self):
downloadTest = self.fundamental_data_object.download(
self.instrument,
fundamental_ratio=Ratio.fundamental_enterprise_value_Y,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(downloadTest)
self.assertNotEquals(len(downloadTest), 0)
@unittest.skip
def test_download_wrong(self):
instrument = Instrument('AAPqwL',
asset_type=AssetType.us_equity,
currency=Currency.usd)
downloadTest = self.fundamental_data_object.download(
instrument,
fundamental_ratio=Ratio.fundamental_ebitda_Y,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNone(downloadTest)
@unittest.skip
def test_calculate_gross_margin(self):
downloadTest = self.fundamental_data_object.download(
self.instrument,
fundamental_ratio=Ratio.fundamental_gross_margin_Y,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(downloadTest)
self.assertNotEquals(len(downloadTest), 0)
self.assertTrue(Ratio.ratio in downloadTest.columns)
self.assertEqual(len(downloadTest.columns), 1)
@unittest.skip
def test_all_fundamentals(self):
ratioList = Ratio.fundamental_list_Y
for ratio in ratioList:
print('testing ratio %s ' % ratio)
downloadTest = self.fundamental_data_object.download(
self.instrument,
fundamental_ratio=ratio,
fromDate=self.fromDate,
toDate=self.toDate)
self.assertIsNotNone(downloadTest)
self.assertNotEquals(len(downloadTest), 0)
self.assertTrue(Ratio.ratio in downloadTest.columns)
self.assertEqual(len(downloadTest.columns), 1)
@unittest.skip
def test_all_fundamentals_allSymbols(self):
fromDate = datetime.datetime(year=2000, day=1, month=1)
toDate = datetime.datetime(year=2018, day=26, month=6)
symbolList = getSF0Tickers(self.user_settings)
instrumentList = getInstrumentList(symbolList=symbolList,
currency=Currency.usd,
asset_type=AssetType.us_equity)
ratioList = Ratio.fundamental_list_Y
for ratio in ratioList:
for instrument in instrumentList:
print('testing ratio %s for %s ' % (ratio, instrument))
downloadTest = self.fundamental_data_object.download(
instrument,
fundamental_ratio=ratio,
fromDate=fromDate,
toDate=toDate)
self.assertIsNotNone(downloadTest)
self.assertNotEquals(len(downloadTest), 0)
self.assertTrue(Ratio.ratio in downloadTest.columns)
self.assertEqual(len(downloadTest.columns), 1)