class TestSymbols(unittest.TestCase):
def setUp(self):
self.symbols = Symbols()
conn = pymongo.MongoClient()
db = conn.test_marketdata
symbols = db.test_symbols
self.symbols._symbols = symbols # replaces real db with test one
self.symbols.clean()
def test_clean(self):
self.symbols.add(['AAPL'])
self.symbols.clean()
act = self.symbols.symbols()
self.assertEquals(0, len(act))
def test_add(self):
exp = ['AAPL']
self.symbols.add(exp)
self.assertListEqual(exp, self.symbols.symbols())
def test_remove(self):
exp = ['AAPL']
added = ['MSFT']
self.symbols.add(exp + added)
self.symbols.remove(added)
self.assertListEqual(exp, self.symbols.symbols())
class UpdateMarketDataIntegrationTest(unittest.TestCase):
def setUp(self):
self.symbols = Symbols()
conn = pymongo.MongoClient()
db = conn.test_marketdata
symbols = db.test_symbols
self.symbols._symbols = symbols # replaces real db with test one
self.symbols.clean()
self.symbols.add(['AAPL'])
def test_update_marketdata(self):
from_date = datetime(2012, 9, 20)
to_date = datetime(2012, 9, 21)
update_marketdata(from_date, to_date, self.symbols)
res = self.symbols.select_historical_prices('AAPL', from_date, to_date)
self.assertEqual(2, len(res))
self.assertEqual(from_date, res[0]['date'])
self.assertEqual(to_date, res[1]['date'])
self.assertEqual(705.07, res[1]['high'])
class DataSource(object):
'''
Spec for a Stock data source
'''
__metaclass__ = abc.ABCMeta
def __init__(self, name, fields, exchanges=[]):
'''
Constructor
'''
self.name = name
self.exchanges = exchanges
self.rawpath = join(Config.DATA_BASEDIR, "raw/US", name)
self.pklpath = join(Config.DATA_BASEDIR, "processed/US", name)
self.date_from = datetime.datetime(1990, 1, 1)
self.date_to = datetime.datetime.now()
self.symbols = Symbols()
self.csv_delimiter = ","
self.csv_skiprows = 1
self.csv_cols = range(0, 7) # date, o, h, l, c, v, adj_close
self.fields = fields
self.field_name_to_idx = {f[1]: f[0] for f in enumerate(fields)}
self.raw_datefile = join(Config.MYDIR, "NYSE_dates.txt")
self.datefile = join(Config.DATA_BASEDIR, "NYSE_dates.pkl")
self.dates = None
self.dtype = self._setup_dtype()
self.num_workers = 4
if not os.path.exists(self.rawpath):
os.makedirs(self.rawpath)
if not os.path.exists(self.pklpath):
os.makedirs(self.pklpath)
if not os.path.exists(self.datefile):
self._setup_dataframe()
def __repr__(self):
res = "name: " + self.name
res += "\n rawpath: " + self.rawpath
res += "\n pklpath: " + self.pklpath
res += "\n exchanges: " + ", ".join([x.name for x in self.exchanges])
return res
def _setup_dtype(self):
"""
Data types need to be specified for the columns of CSV files.
* Date object is stored in object format.
* All other fields are parsed as float.
"""
names = []
formats = []
for f in self.fields:
names.append(f)
if f == DataItem.DATE:
formats.append("O") # date in object format
else:
formats.append(float)
return {'names': names, 'formats': formats}
def _csvfile(self, sym):
"""
CVS file for a given symbol
"""
return join(self.rawpath, sym + ".csv")
def _pklfile(self, sym):
"""
Pickle file for a given symbol
"""
return join(self.pklpath, sym + ".pkl")
@property
def symbol_list(self):
"""
Returns a list of symbols for all exchanges
"""
return reduce(lambda x, y: x + y.symbol_list(), self.exchanges, [])
@abc.abstractmethod
def _hist_url(self, date_from, date_to, symbol):
"""
Returns a URL for getting the historical data from the web
"""
return
@abc.abstractmethod
def _csv_field_converters(self):
"""
Converters for the CSV columns.
Numpy can do the simple conversions, But converters need to be
specified for complex fields. E.g. parsing a Data string to
datetime object
"""
return
def _setup_dataframe(self, datefile=None):
"""
Setup an empty dataframe using NYSE dates as index.
This will be used to as base frame for the stock data.
"""
if datefile is None:
datefile = self.raw_datefile
dates = np.loadtxt(datefile, dtype='O',
converters={
0: lambda x: dt.strptime(x, "%m/%d/%Y")
})
dateIdx = pandas.tseries.index.DatetimeIndex(dates)
empty_df = pandas.DataFrame(index=dateIdx)
empty_df.save(self.datefile)
def _load_dataframe(self):
"""
Load the previously saved dataframe
"""
return read_pickle(self.datefile)
def download(self, symbol):
"""Download data for a given symbol
This will store the data in pickle file
"""
url = self._hist_url(self.date_from, self.date_to, symbol)
url_get = urllib2.urlopen(url)
# url_get = open(DataSource.TESTFILE, "rb")
csvfile = open(self._csvfile(symbol), "wb")
csvfile.write(url_get.read())
csvfile.close()
self._pickle(symbol)
return symbol
def _load_csv(self, symbol):
"""
Load a CSV file into a pandas Dataframe
"""
data_file = self._csvfile(symbol)
data = np.loadtxt(data_file,
dtype=self.dtype,
delimiter=self.csv_delimiter,
skiprows=self.csv_skiprows,
converters=self._csv_field_converters(),
usecols=self.csv_cols)
data = data[::-1] # reverse the order
fields = data.dtype.names[1:] # all except the date
df = pandas.DataFrame({f: data[f]
for f in fields}, index=data[DataItem.DATE])
return df
def _pickle(self, symbol):
"""
Load CSV data and save the object to a file.
load_csv is expensive, as it needs to convert data to appropriate
datatypes. pickle saves the processed data that can efficiently
reloaded.
"""
df = self._load_csv(symbol)
df.save(self._pklfile(symbol))
return symbol
def load(self, symbol):
"""
load symbol dataframe from a pre saved file
This loads the pre-processed data that was previously stored using
pickle().
"""
return read_pickle(self._pklfile(symbol))
def loadData(self, symbols, field="adj_close", from_=None, to_=None):
"""
Load data for symbols for given date range.
Returns a DataFrame. Columns are indexed by symbols
e.g:
from_date = datetime.datetime.strptime("2012/1/1", "%Y/%M/%d")
to_date = datetime.datetime.now()
syms=["AAPL", "GOOG", "COG"]
ds.loadData(syms, "close", from_date, to_date)
# DataFrame
<class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 248 entries, 2012-01-03 00:00:00 to 2012-12-24 00:00:00
Data columns:
AAPL 245 non-null values
GOOG 245 non-null values
COG 245 non-null values
dtypes: float64(3)
# df[:3]
AAPL GOOG COG
2012-01-03 411.23 665.41 76.80
2012-01-04 413.44 668.28 82.32
2012-01-05 418.03 659.01 82.71
"""
if from_ is None:
from_ = self.date_from
if to_ is None:
to_ = self.date_to
df = self._load_dataframe()[from_:to_]
for s in symbols:
df[s] = self.load(s)[field]
return df
def create_symbol_list(self):
for exch in self.exchanges:
exch.get_symbol_list()
self.symbols.add(exch)
screen = pygame.display.set_mode((300, 300)) # creates the screen
background = pygame.Surface((300, 300)) # creates the background image
background = background.convert()
board = pygame.Surface((300, 300))
pygame.draw.rect(background, WHITE, [50, 50, 201, 201])
draw_lines()
screen.blit(background, (0, 0)) # blits background onto screen
pygame.display.flip() # updates display
symbols = Symbols()
symbol = Symbol()
while not end:
for event in pygame.event.get(): # deteremines which key is pressed
if event.type == pygame.QUIT: # if you click the quit :(
end = True # breaks the while loop
if event.type == pygame.MOUSEBUTTONDOWN:
mouse_pos = pygame.mouse.get_pos()
mx = mouse_pos[0]
my = mouse_pos[1]
if 50 <= mx <= 251 and 50 <= my <= 251:
grid_pos = (get_pos(mx), get_pos(my))
if occupied_check(symbols, grid_pos):
symbol.add(grid_pos)
draw_symbol(symbol)
symbols.add(symbol)
symbol.switch()
# print("line check", symbols.line_check(symbol))
# print(grid_pos)
class TestMarketDataDb(unittest.TestCase):
def setUp(self):
self.symbols = Symbols()
conn = pymongo.MongoClient()
db = conn.test_marketdata
symbols = db.test_symbols
self.symbols._symbols = symbols # replaces real db with test one
self.symbols.clean()
self.symbols.add(['AAPL'])
def test_single_hist_price(self):
dt = datetime(2013, 7, 13)
self.symbols.insert_historical_prices(
'AAPL', [(dt, 100.0, 101.0, 99.0, 100.0, 100, 100.0)])
res = self.symbols.select_historical_prices('AAPL', dt, dt)
self.assertEqual(1, len(res))
self.assertEqual(dt, res[0]['date'])
self.assertEqual(100.0, res[0]['open'])
self.assertEqual(101.0, res[0]['high'])
self.assertEqual(99.0, res[0]['low'])
self.assertEqual(100.0, res[0]['close'])
self.assertEqual(100, res[0]['volume'])
self.assertEqual(100.0, res[0]['adj_close'])
def test_hist_price_for_undefined_date(self):
dt = datetime(2013, 7, 13)
res = self.symbols.select_historical_prices('AAPL', dt, dt)
self.assertEqual([], res)
def test_three_hist_price(self):
d1 = datetime(2013, 7, 13)
d2 = datetime(2013, 7, 14)
d3 = datetime(2013, 7, 15)
self.symbols.insert_historical_prices(
'AAPL', [(d1, 100.0, 101.0, 99.0, 100.0, 100, 100.0)])
self.symbols.insert_historical_prices(
'AAPL', [(d3, 99.0, 100.0, 98.0, 99.0, 99, 99.0)])
self.symbols.insert_historical_prices(
'AAPL', [(d2, 101.0, 102.0, 100.0, 101.0, 101, 101.0)])
res = self.symbols.select_historical_prices('AAPL', d1, d3)
self.assertEqual(3, len(res))
self.assertEqual(d1, res[0]['date'])
self.assertEqual(100.0, res[0]['open'])
self.assertEqual(d2, res[1]['date'])
self.assertEqual(101.0, res[1]['open'])
self.assertEqual(d3, res[2]['date'])
self.assertEqual(99.0, res[2]['open'])
def test_date_filtering(self):
d1 = datetime(2013, 7, 13)
d2 = datetime(2013, 7, 14)
d3 = datetime(2013, 7, 15)
d4 = datetime(2013, 7, 16)
self.symbols.insert_historical_prices(
'AAPL', [(d1, 100.0, 101.0, 99.0, 100.0, 100, 100.0)])
self.symbols.insert_historical_prices(
'AAPL', [(d2, 101.0, 102.0, 100.0, 101.0, 101, 101.0)])
self.symbols.insert_historical_prices(
'AAPL', [(d3, 99.0, 100.0, 98.0, 99.0, 99, 99.0)])
self.symbols.insert_historical_prices(
'AAPL', [(d4, 98.0, 99.0, 97.0, 98.0, 98, 98.0)])
res = self.symbols.select_historical_prices('AAPL', d2, d3)
self.assertEqual(2, len(res))
self.assertEqual(d2, res[0]['date'])
self.assertEqual(101.0, res[0]['open'])
self.assertEqual(d3, res[1]['date'])
self.assertEqual(99.0, res[1]['open'])
def test_reinsert_historical_prices(self):
d1 = datetime(2013, 7, 13)
d2 = datetime(2013, 7, 14)
d3 = datetime(2013, 7, 15)
self.symbols.insert_historical_prices(
'AAPL', [(d1, 100.0, 101.0, 99.0, 100.0, 100, 100.0),
(d2, 101.0, 102.0, 100.0, 101.0, 101, 101.0),
(d3, 99.0, 100.0, 98.0, 99.0, 99, 99.0)])
res = self.symbols.select_historical_prices('AAPL', d1, d3)
self.assertEqual(3, len(res))
self.symbols.insert_historical_prices(
'AAPL', [(d2, 101.0, 102.0, 100.0, 101.0, 101, 101.0)])
res = self.symbols.select_historical_prices('AAPL', d1, d3)
self.assertEqual(3, len(res))
self.assertEqual(d1, res[0]['date'])
self.assertEqual(100.0, res[0]['open'])
self.assertEqual(d2, res[1]['date'])
self.assertEqual(101.0, res[1]['open'])
self.assertEqual(d3, res[2]['date'])
self.assertEqual(99.0, res[2]['open'])
def test_last_date(self):
d = [
datetime(2013, 7, 16),
datetime(2013, 7, 14),
datetime(2013, 7, 13),
datetime(2013, 7, 17),
datetime(2013, 7, 12),
datetime(2013, 7, 15)
]
for x in d:
self.symbols.insert_historical_prices(
'AAPL', [(x, 1.0, 1.0, 1.0, 1.0, 1, 1.0)])
res = self.symbols.last_date('AAPL')
self.assertEquals(datetime(2013, 7, 17), res)
def test_last_date_when_no_data(self):
res = self.symbols.last_date('AAPL')
self.assertEquals(None, res)