def test_all(self):
"""
Test all inside a symbol folder (max 2 year, 3 files)
:return:
"""
for year_folder in glob(os.path.join(self.test_dir, '*'))[:2]:
thinkback_files = glob(os.path.join(year_folder, '*.*'))
for test_file in thinkback_files[:3]:
date, symbol = os.path.basename(test_file)[:-4].split('-StockAndOptionQuoteFor')
raw_data = open(test_file).read()
open_thinkback = OpenThinkBack(date=date, data=raw_data)
stock, options = open_thinkback.format()
print 'stock: %s' % stock
print 'option count: %d' % len(options)
print '.' * 80
print '*' * 100
def test_all(self):
"""
Test all inside a symbol folder (max 2 year, 3 files)
:return:
"""
for year_folder in glob(os.path.join(self.test_dir, '*'))[:2]:
thinkback_files = glob(os.path.join(year_folder, '*.*'))
for test_file in thinkback_files[:3]:
date, symbol = os.path.basename(test_file)[:-4].split(
'-StockAndOptionQuoteFor')
raw_data = open(test_file).read()
open_thinkback = OpenThinkBack(date=date, data=raw_data)
stock, options = open_thinkback.format()
print 'stock: %s' % stock
print 'option count: %d' % len(options)
print '.' * 80
print '*' * 100
def setUp(self):
TestSetUp.setUp(self)
self.symbol = 'WFC'
self.year = '2015'
self.date = '2015-01-30'
self.test_path = list()
for path in glob(os.path.join(THINKBACK_DIR, '*')):
if os.path.isdir(path):
self.test_path.append(path)
self.test_dir = [
path for path in self.test_path if self.symbol.lower() in path
].pop()
# single test file
self.test_file = r'D:\rivers\data' \
r'/csv\%s\%s\%s-StockAndOptionQuoteFor%s.csv' % \
(self.symbol.lower(), self.year, self.date, self.symbol.upper())
self.open_thinkback = OpenThinkBack(date=self.date,
data=open(self.test_file).read())
def setUp(self):
TestSetUp.setUp(self)
self.symbol = 'WFC'
self.year = '2015'
self.date = '2015-01-30'
self.test_path = list()
for path in glob(os.path.join(THINKBACK_DIR, '*')):
if os.path.isdir(path):
self.test_path.append(path)
self.test_dir = [path for path in self.test_path if self.symbol.lower() in path].pop()
# single test file
self.test_file = r'D:\rivers\data' \
r'/csv\%s\%s\%s-StockAndOptionQuoteFor%s.csv' % \
(self.symbol.lower(), self.year, self.date, self.symbol.upper())
self.open_thinkback = OpenThinkBack(
date=self.date,
data=open(self.test_file).read()
)
def data_tos_thinkback_import_view(request, symbol):
"""
Import a symbol folder into db
:param request: request
:param symbol: str
:return: render
"""
template = 'data/run_csv.html'
symbol = symbol.upper()
""":type: str"""
insert_files = list()
# move files into year folder
no_year_files = glob(os.path.join(THINKBACK_DIR, symbol, '*.csv'))
years = sorted(list(set([os.path.basename(f)[:4] for f in no_year_files])))
for year in years:
year_dir = os.path.join(THINKBACK_DIR, symbol, year)
# make dir if not exists
if not os.path.isdir(year_dir):
os.mkdir(year_dir)
# move all year files into dir
for no_year_file in no_year_files:
#print os.path.basename(no_year_file)[:4], year
filename = os.path.basename(no_year_file)
if filename[:4] == year:
#print 'move ', no_year_file, ' -> ', year_dir
os.rename(no_year_file, os.path.join(year_dir, filename))
# get all files in year folder
files = []
for year in glob(os.path.join(THINKBACK_DIR, symbol, '*')):
for csv in glob(os.path.join(year, '*.csv')):
files.append(csv)
# stocks = list()
for csv in files:
inserted_file = dict(path='', stock=None, contracts=0, options=0)
# get date and symbol
date, symbol = os.path.basename(csv)[:-4].split(
'-StockAndOptionQuoteFor')
# insert stock if not exists in db
query = Q(symbol=symbol) & Q(date=date) & Q(source='tos_thinkback')
if not Stock.objects.filter(query).exists() and date not in SKIP_DATES:
# output to console
print 'running %s file...' % os.path.basename(csv)
stock_data, option_data = OpenThinkBack(
date=date, data=open(csv).read()).format()
# for view only
inserted_file['path'] = os.path.basename(csv)
stock = Stock()
stock.symbol = symbol
stock.source = 'tos_thinkback'
stock.data = stock_data
stock.save()
# stocks.append(stock)
# for view only
inserted_file['stock'] = stock
option_codes = [
contract['option_code'] for contract, _ in option_data
]
size = 100
exists_option_codes = list()
for chunk in [
option_codes[i:i + size]
for i in range(0, len(option_codes), size)
]:
exists_option_codes += [
x[0] for x in OptionContract.objects.filter(
option_code__in=chunk).values_list('option_code')
]
new_option_codes = set(option_codes) - set(exists_option_codes)
contracts = list()
for option_code in set(option_codes):
if option_code in new_option_codes:
try:
contract_dict = [
c for c, _ in option_data
if c['option_code'] == option_code
][0]
except IndexError:
print option_code
raise Exception()
contract = OptionContract()
contract.symbol = symbol
contract.source = 'tos_thinkback'
contract.data = contract_dict
contracts.append(contract)
# insert option contract
OptionContract.objects.bulk_create(contracts)
# for view only
inserted_file['contracts'] = len(contracts)
option_contracts = list()
for chunk in [
option_codes[i:i + size]
for i in range(0, len(option_codes), size)
]:
option_contracts += [
option_contract
for option_contract in OptionContract.objects.filter(
option_code__in=chunk)
]
options = list()
for contract_dict, option_dict in option_data:
try:
option_contract = [
option_contract for option_contract in option_contracts
if option_contract.option_code ==
contract_dict['option_code']
][0]
except IndexError:
raise IndexError('Contract not inserted...')
option = Option()
option.option_contract = option_contract
option.data = option_dict
options.append(option)
# insert options
Option.objects.bulk_create(options)
# for view only
inserted_file['options'] = len(options)
# add into inserted
insert_files.append(inserted_file)
# missing files between dates
missing_files = list()
if Stock.objects.count() > 2 and Stock.objects.filter(
symbol=symbol).exists():
bdays = bdate_range(start=Stock.objects.filter(
symbol=symbol).order_by('date').first().date,
end=Stock.objects.filter(
symbol=symbol).order_by('date').last().date,
freq='B')
for bday in bdays:
try:
Stock.objects.get(
Q(symbol=symbol) & Q(source='tos_thinkback')
& Q(date=bday.strftime('%Y-%m-%d')))
except ObjectDoesNotExist:
if is_not_holiday(bday.strftime('%Y-%m-%d')) and \
is_not_offdays(bday.strftime('%m/%d/%y')):
missing_files.append(
dict(filename='%s-StockAndOptionQuoteFor%s.csv' %
(bday.strftime('%Y-%m-%d'), symbol),
date=bday.strftime('%m/%d/%y')))
parameters = dict(symbol=symbol,
insert_files=insert_files,
missing_files=missing_files)
return render(request, template, parameters)
def data_daily_import_view(request):
"""
Import all csv files in daily folder
then insert web data for that date
:param request: request
:return: render
"""
template = 'data/daily.html'
insert_files = list()
files = [
path for path in glob(os.path.join(THINKBACK_DIR, '_daily', '*.csv'))
]
for f in files:
contracts = 0
options = 0
# get filename and dir
filename = os.path.basename(f)
print 'running file: %s...' % filename
date, symbol = map(lambda x: x.upper(),
filename[:-4].split('-StockAndOptionQuoteFor'))
# file into dict
stock_data, option_data = OpenThinkBack(date=date,
data=open(f).read()).format()
# save stock
stock = Stock()
stock.symbol = symbol
stock.source = 'tos_thinkback'
stock.data = stock_data
stock.save()
# save contract and option
for contract_dict, option_dict in option_data:
try:
contract = OptionContract.objects.get(
option_code=contract_dict['option_code'])
except ObjectDoesNotExist:
contract = OptionContract()
contract.symbol = symbol
contract.source = 'tos_thinkback'
contract.data = contract_dict
contract.save()
contracts += 1
option = Option()
option.option_contract = contract
option.data = option_dict
option.save()
options += 1
# move file into folder
year = filename[:4]
year_dir = os.path.join(THINKBACK_DIR, symbol, year)
# make dir if not exists
if not os.path.isdir(year_dir):
os.mkdir(year_dir)
os.rename(f, os.path.join(year_dir, os.path.basename(f)))
# save data from web
google_data = get_data_google(
symbols=symbol,
# start='2015-04-01', end='2015-04-10', # test only
start=date,
end=date,
adjust_price=True)
yahoo_data = get_data_yahoo(
symbols=symbol,
# start='2015-04-01', end='2015-04-10', # test only
start=date,
end=date,
adjust_price=True)
for index, data in google_data.iterrows():
if int(data['Volume']) > 0:
google_stock = Stock(symbol=symbol,
date=index.strftime('%Y-%m-%d'),
open=data['Open'],
high=data['High'],
low=data['Low'],
close=data['Close'],
volume=data['Volume'],
source='google')
google_stock.save()
for index, data in yahoo_data.iterrows():
if int(data['Volume']) > 0:
yahoo_stock = Stock(symbol=symbol,
date=index.strftime('%Y-%m-%d'),
open=data['Open'],
high=data['High'],
low=data['Low'],
close=data['Close'],
volume=data['Volume'],
source='yahoo')
yahoo_stock.save()
insert_files.append(
dict(symbol=symbol,
date=date,
path=filename,
stock=1,
contracts=contracts,
options=options))
parameters = dict(insert_files=insert_files)
return render(request, template, parameters)
class TestOpenThinkBack(TestSetUp):
# noinspection PyUnresolvedReferences
def setUp(self):
TestSetUp.setUp(self)
self.symbol = 'WFC'
self.year = '2015'
self.date = '2015-01-30'
self.test_path = list()
for path in glob(os.path.join(THINKBACK_DIR, '*')):
if os.path.isdir(path):
self.test_path.append(path)
self.test_dir = [path for path in self.test_path if self.symbol.lower() in path].pop()
# single test file
self.test_file = r'D:\rivers\data' \
r'/csv\%s\%s\%s-StockAndOptionQuoteFor%s.csv' % \
(self.symbol.lower(), self.year, self.date, self.symbol.upper())
self.open_thinkback = OpenThinkBack(
date=self.date,
data=open(self.test_file).read()
)
def test_get_underlying(self):
"""
Test get underlying stock price in line 5
"""
expected_keys = ['date', 'volume', 'open', 'high', 'low', 'last', 'net_change']
print 'run get_stock...'
stock = self.open_thinkback.get_stock()
self.assertEqual(type(stock), dict)
print 'stock dict:'
pprint(stock)
for key, value in stock.items():
self.assertIn(key, expected_keys)
if key is not 'date':
self.assertEqual(type(value), float)
else:
self.assertEqual(type(value), str)
def test_get_cycles(self):
"""
Test get cycles from each option chain head title
"""
print 'run get_cycles...'
cycles = self.open_thinkback.get_cycles()
self.assertEqual(type(cycles), list)
print 'cycle dict:'
pprint(cycles, width=400)
for cycle in cycles:
self.assertEqual(type(cycle), dict)
self.assertEqual(len(cycle['data']), 5)
self.assertEqual(type(cycle['dte']), int)
self.assertGreaterEqual(cycle['dte'], 0)
self.assertEqual(type(cycle['line']), str)
self.assertGreater(cycle['start'], 10)
self.assertLess(cycle['start'], cycle['stop'])
def test_get_cycle_options(self):
"""
Test get cycle options from option chain
"""
months = [month_name[i + 1][:3].upper() for i in range(12)]
contract_keys = [
'ex_month', 'ex_year', 'right', 'special', 'others',
'strike', 'contract', 'option_code'
]
option_keys = [
'date', 'dte',
'last', 'mark', 'bid', 'ask', 'delta', 'gamma', 'theta', 'vega',
'theo_price', 'impl_vol', 'prob_itm', 'prob_otm', 'prob_touch', 'volume',
'open_int', 'intrinsic', 'extrinsic'
]
print 'run get_cycles...'
cycles = self.open_thinkback.get_cycles()
for cycle in cycles[:1]:
print 'cycle: %s' % cycle['data']
print 'run get_cycle_option...'
options = self.open_thinkback.get_cycle_options(cycle)
for contract, option in options:
print 'current contract, option code: %s' % contract['option_code']
pprint(contract, width=400)
self.assertEqual(type(contract), dict)
self.assertEqual(sorted(contract.keys()), sorted(contract_keys))
self.assertEqual(type(contract['others']), str)
self.assertEqual(type(contract['ex_month']), str)
self.assertIn(contract['ex_month'][:3], months)
if len(contract['ex_month']) == 4:
self.assertGreater(int(contract['ex_month'][3]), 0)
self.assertLessEqual(int(contract['ex_month'][3]), 12)
self.assertEqual(type(contract['ex_year']), int)
self.assertGreater(contract['ex_year'], 0)
self.assertLessEqual(contract['ex_year'], 99)
self.assertEqual(type(contract['right']), str) # not int, str
self.assertEqual(type(contract['contract']), str)
self.assertIn(contract['contract'], ['CALL', 'PUT'])
self.assertEqual(type(contract['special']), str)
self.assertIn(contract['special'], ['Weeklys', 'Standard', 'Mini'])
self.assertEqual(type(contract['strike']), float)
self.assertGreater(contract['strike'], 0)
self.assertEqual(type(contract['option_code']), str)
print 'current option:'
pprint(option, width=400)
for key in option.keys():
self.assertIn(key, option_keys)
if key == 'date':
self.assertEqual(type(option['date']), str)
self.assertTrue(datetime.strptime(option['date'], '%Y-%m-%d'))
elif key == 'dte':
self.assertEqual(type(option['dte']), int)
else:
self.assertEqual(type(option[key]), float)
print '.' * 80
print '\n' + '*' * 100 + '\n'
def test_format(self):
"""
Test format a raw lines data into dict
"""
stock, option = self.open_thinkback.format()
print 'stock: %s' % stock
print 'option: '
pprint(option, width=400)
def test_all(self):
"""
Test all inside a symbol folder (max 2 year, 3 files)
:return:
"""
for year_folder in glob(os.path.join(self.test_dir, '*'))[:2]:
thinkback_files = glob(os.path.join(year_folder, '*.*'))
for test_file in thinkback_files[:3]:
date, symbol = os.path.basename(test_file)[:-4].split('-StockAndOptionQuoteFor')
raw_data = open(test_file).read()
open_thinkback = OpenThinkBack(date=date, data=raw_data)
stock, options = open_thinkback.format()
print 'stock: %s' % stock
print 'option count: %d' % len(options)
print '.' * 80
print '*' * 100
class TestOpenThinkBack(TestSetUp):
# noinspection PyUnresolvedReferences
def setUp(self):
TestSetUp.setUp(self)
self.symbol = 'WFC'
self.year = '2015'
self.date = '2015-01-30'
self.test_path = list()
for path in glob(os.path.join(THINKBACK_DIR, '*')):
if os.path.isdir(path):
self.test_path.append(path)
self.test_dir = [
path for path in self.test_path if self.symbol.lower() in path
].pop()
# single test file
self.test_file = r'D:\rivers\data' \
r'/csv\%s\%s\%s-StockAndOptionQuoteFor%s.csv' % \
(self.symbol.lower(), self.year, self.date, self.symbol.upper())
self.open_thinkback = OpenThinkBack(date=self.date,
data=open(self.test_file).read())
def test_get_underlying(self):
"""
Test get underlying stock price in line 5
"""
expected_keys = [
'date', 'volume', 'open', 'high', 'low', 'last', 'net_change'
]
print 'run get_stock...'
stock = self.open_thinkback.get_stock()
self.assertEqual(type(stock), dict)
print 'stock dict:'
pprint(stock)
for key, value in stock.items():
self.assertIn(key, expected_keys)
if key is not 'date':
self.assertEqual(type(value), float)
else:
self.assertEqual(type(value), str)
def test_get_cycles(self):
"""
Test get cycles from each option chain head title
"""
print 'run get_cycles...'
cycles = self.open_thinkback.get_cycles()
self.assertEqual(type(cycles), list)
print 'cycle dict:'
pprint(cycles, width=400)
for cycle in cycles:
self.assertEqual(type(cycle), dict)
self.assertEqual(len(cycle['data']), 5)
self.assertEqual(type(cycle['dte']), int)
self.assertGreaterEqual(cycle['dte'], 0)
self.assertEqual(type(cycle['line']), str)
self.assertGreater(cycle['start'], 10)
self.assertLess(cycle['start'], cycle['stop'])
def test_get_cycle_options(self):
"""
Test get cycle options from option chain
"""
months = [month_name[i + 1][:3].upper() for i in range(12)]
contract_keys = [
'ex_month', 'ex_year', 'right', 'special', 'others', 'strike',
'contract', 'option_code'
]
option_keys = [
'date', 'dte', 'last', 'mark', 'bid', 'ask', 'delta', 'gamma',
'theta', 'vega', 'theo_price', 'impl_vol', 'prob_itm', 'prob_otm',
'prob_touch', 'volume', 'open_int', 'intrinsic', 'extrinsic'
]
print 'run get_cycles...'
cycles = self.open_thinkback.get_cycles()
for cycle in cycles[:1]:
print 'cycle: %s' % cycle['data']
print 'run get_cycle_option...'
options = self.open_thinkback.get_cycle_options(cycle)
for contract, option in options:
print 'current contract, option code: %s' % contract[
'option_code']
pprint(contract, width=400)
self.assertEqual(type(contract), dict)
self.assertEqual(sorted(contract.keys()),
sorted(contract_keys))
self.assertEqual(type(contract['others']), str)
self.assertEqual(type(contract['ex_month']), str)
self.assertIn(contract['ex_month'][:3], months)
if len(contract['ex_month']) == 4:
self.assertGreater(int(contract['ex_month'][3]), 0)
self.assertLessEqual(int(contract['ex_month'][3]), 12)
self.assertEqual(type(contract['ex_year']), int)
self.assertGreater(contract['ex_year'], 0)
self.assertLessEqual(contract['ex_year'], 99)
self.assertEqual(type(contract['right']), str) # not int, str
self.assertEqual(type(contract['contract']), str)
self.assertIn(contract['contract'], ['CALL', 'PUT'])
self.assertEqual(type(contract['special']), str)
self.assertIn(contract['special'],
['Weeklys', 'Standard', 'Mini'])
self.assertEqual(type(contract['strike']), float)
self.assertGreater(contract['strike'], 0)
self.assertEqual(type(contract['option_code']), str)
print 'current option:'
pprint(option, width=400)
for key in option.keys():
self.assertIn(key, option_keys)
if key == 'date':
self.assertEqual(type(option['date']), str)
self.assertTrue(
datetime.strptime(option['date'], '%Y-%m-%d'))
elif key == 'dte':
self.assertEqual(type(option['dte']), int)
else:
self.assertEqual(type(option[key]), float)
print '.' * 80
print '\n' + '*' * 100 + '\n'
def test_format(self):
"""
Test format a raw lines data into dict
"""
stock, option = self.open_thinkback.format()
print 'stock: %s' % stock
print 'option: '
pprint(option, width=400)
def test_all(self):
"""
Test all inside a symbol folder (max 2 year, 3 files)
:return:
"""
for year_folder in glob(os.path.join(self.test_dir, '*'))[:2]:
thinkback_files = glob(os.path.join(year_folder, '*.*'))
for test_file in thinkback_files[:3]:
date, symbol = os.path.basename(test_file)[:-4].split(
'-StockAndOptionQuoteFor')
raw_data = open(test_file).read()
open_thinkback = OpenThinkBack(date=date, data=raw_data)
stock, options = open_thinkback.format()
print 'stock: %s' % stock
print 'option count: %d' % len(options)
print '.' * 80
print '*' * 100