def handle(self, *args, **options):
print("start aync data to stock: >>>>>>>>>>>>>>>>>>>>>>")
t0 = time.time()
data = get_all_stock_info()
if data:
for code, val in data.items():
kwargs = {
'datetime': val.get('datetime') or '%s %s' % (val.get('date', ''), val.get('time', '')),
'now': val['now'],
'low': val['low'],
'high': val['high'],
'turnover': val['turnover'],
'volume': val['volume'],
'buy': val['buy'],
'sell': val['sell'],
'open': val['open'],
'close': val['close'],
'ask1': val.get('ask1', 0.00),
'ask1_volume': val.get('ask1_volume', 0),
'ask2': val.get('ask2', 0.00),
'ask2_volume': val.get('ask2_volume', 0),
'ask3': val.get('ask3', 0.00),
'ask3_volume': val.get('ask3_volume', 0),
'ask4': val.get('ask4', 0.00),
'ask4_volume': val.get('ask4_volume', 0),
'ask5': val.get('ask5', 0.00),
'ask5_volume': val.get('ask5_volume', 0),
'bind1': val.get('bind1', 0.00),
'bind1_volume': val.get('bind1_volume', 0),
'bind2': val.get('bind2', 0.00),
'bind2_volume': val.get('bind2_volume', 0),
'bind3': val.get('bind3', 0.00),
'bind3_volume': val.get('bind3_volume', 0),
'bind4': val.get('bind4', 0.00),
'bind4_volume': val.get('bind4_volume', 0),
'bind5': val.get('bind5', 0.00),
'bind5_volume': val.get('bind5_volume', 0),
}
try:
stock = Stock.objects.get(code=code[2:])
except Stock.DoesNotExist:
kwargs['code'] = code[2:]
kwargs['name'] = val['name']
kwargs['enum_stock_type'] = code[:2]
stock = Stock()
for k, v in kwargs.items():
setattr(stock, k, v)
stock.save()
print("cost: %.2f <<<<<<<<<<<<<<<<<<<<<<<<" % (time.time()-t0))
def create_stock(symbol, index, data, source):
"""
Create a stock object
:param symbol: str
:param index: datetime
:param data: dict
:param source: str
:return: Stock
"""
return Stock(symbol=symbol,
date=index.strftime('%Y-%m-%d'),
open=data['Open'],
high=data['High'],
low=data['Low'],
close=data['Close'],
volume=data['Volume'],
source=source)
def data_tos_thinkback_import_view(request, symbol):
"""
Import a symbol folder into db
:param request: request
:param symbol: str
:return: render
"""
template = 'data/run_csv.html'
symbol = symbol.upper()
""":type: str"""
insert_files = list()
# move files into year folder
no_year_files = glob(os.path.join(THINKBACK_DIR, symbol, '*.csv'))
years = sorted(list(set([os.path.basename(f)[:4] for f in no_year_files])))
for year in years:
year_dir = os.path.join(THINKBACK_DIR, symbol, year)
# make dir if not exists
if not os.path.isdir(year_dir):
os.mkdir(year_dir)
# move all year files into dir
for no_year_file in no_year_files:
#print os.path.basename(no_year_file)[:4], year
filename = os.path.basename(no_year_file)
if filename[:4] == year:
#print 'move ', no_year_file, ' -> ', year_dir
os.rename(no_year_file, os.path.join(year_dir, filename))
# get all files in year folder
files = []
for year in glob(os.path.join(THINKBACK_DIR, symbol, '*')):
for csv in glob(os.path.join(year, '*.csv')):
files.append(csv)
# stocks = list()
for csv in files:
inserted_file = dict(path='', stock=None, contracts=0, options=0)
# get date and symbol
date, symbol = os.path.basename(csv)[:-4].split(
'-StockAndOptionQuoteFor')
# insert stock if not exists in db
query = Q(symbol=symbol) & Q(date=date) & Q(source='tos_thinkback')
if not Stock.objects.filter(query).exists() and date not in SKIP_DATES:
# output to console
print 'running %s file...' % os.path.basename(csv)
stock_data, option_data = OpenThinkBack(
date=date, data=open(csv).read()).format()
# for view only
inserted_file['path'] = os.path.basename(csv)
stock = Stock()
stock.symbol = symbol
stock.source = 'tos_thinkback'
stock.data = stock_data
stock.save()
# stocks.append(stock)
# for view only
inserted_file['stock'] = stock
option_codes = [
contract['option_code'] for contract, _ in option_data
]
size = 100
exists_option_codes = list()
for chunk in [
option_codes[i:i + size]
for i in range(0, len(option_codes), size)
]:
exists_option_codes += [
x[0] for x in OptionContract.objects.filter(
option_code__in=chunk).values_list('option_code')
]
new_option_codes = set(option_codes) - set(exists_option_codes)
contracts = list()
for option_code in set(option_codes):
if option_code in new_option_codes:
try:
contract_dict = [
c for c, _ in option_data
if c['option_code'] == option_code
][0]
except IndexError:
print option_code
raise Exception()
contract = OptionContract()
contract.symbol = symbol
contract.source = 'tos_thinkback'
contract.data = contract_dict
contracts.append(contract)
# insert option contract
OptionContract.objects.bulk_create(contracts)
# for view only
inserted_file['contracts'] = len(contracts)
option_contracts = list()
for chunk in [
option_codes[i:i + size]
for i in range(0, len(option_codes), size)
]:
option_contracts += [
option_contract
for option_contract in OptionContract.objects.filter(
option_code__in=chunk)
]
options = list()
for contract_dict, option_dict in option_data:
try:
option_contract = [
option_contract for option_contract in option_contracts
if option_contract.option_code ==
contract_dict['option_code']
][0]
except IndexError:
raise IndexError('Contract not inserted...')
option = Option()
option.option_contract = option_contract
option.data = option_dict
options.append(option)
# insert options
Option.objects.bulk_create(options)
# for view only
inserted_file['options'] = len(options)
# add into inserted
insert_files.append(inserted_file)
# missing files between dates
missing_files = list()
if Stock.objects.count() > 2 and Stock.objects.filter(
symbol=symbol).exists():
bdays = bdate_range(start=Stock.objects.filter(
symbol=symbol).order_by('date').first().date,
end=Stock.objects.filter(
symbol=symbol).order_by('date').last().date,
freq='B')
for bday in bdays:
try:
Stock.objects.get(
Q(symbol=symbol) & Q(source='tos_thinkback')
& Q(date=bday.strftime('%Y-%m-%d')))
except ObjectDoesNotExist:
if is_not_holiday(bday.strftime('%Y-%m-%d')) and \
is_not_offdays(bday.strftime('%m/%d/%y')):
missing_files.append(
dict(filename='%s-StockAndOptionQuoteFor%s.csv' %
(bday.strftime('%Y-%m-%d'), symbol),
date=bday.strftime('%m/%d/%y')))
parameters = dict(symbol=symbol,
insert_files=insert_files,
missing_files=missing_files)
return render(request, template, parameters)
def data_daily_import_view(request):
"""
Import all csv files in daily folder
then insert web data for that date
:param request: request
:return: render
"""
template = 'data/daily.html'
insert_files = list()
files = [
path for path in glob(os.path.join(THINKBACK_DIR, '_daily', '*.csv'))
]
for f in files:
contracts = 0
options = 0
# get filename and dir
filename = os.path.basename(f)
print 'running file: %s...' % filename
date, symbol = map(lambda x: x.upper(),
filename[:-4].split('-StockAndOptionQuoteFor'))
# file into dict
stock_data, option_data = OpenThinkBack(date=date,
data=open(f).read()).format()
# save stock
stock = Stock()
stock.symbol = symbol
stock.source = 'tos_thinkback'
stock.data = stock_data
stock.save()
# save contract and option
for contract_dict, option_dict in option_data:
try:
contract = OptionContract.objects.get(
option_code=contract_dict['option_code'])
except ObjectDoesNotExist:
contract = OptionContract()
contract.symbol = symbol
contract.source = 'tos_thinkback'
contract.data = contract_dict
contract.save()
contracts += 1
option = Option()
option.option_contract = contract
option.data = option_dict
option.save()
options += 1
# move file into folder
year = filename[:4]
year_dir = os.path.join(THINKBACK_DIR, symbol, year)
# make dir if not exists
if not os.path.isdir(year_dir):
os.mkdir(year_dir)
os.rename(f, os.path.join(year_dir, os.path.basename(f)))
# save data from web
google_data = get_data_google(
symbols=symbol,
# start='2015-04-01', end='2015-04-10', # test only
start=date,
end=date,
adjust_price=True)
yahoo_data = get_data_yahoo(
symbols=symbol,
# start='2015-04-01', end='2015-04-10', # test only
start=date,
end=date,
adjust_price=True)
for index, data in google_data.iterrows():
if int(data['Volume']) > 0:
google_stock = Stock(symbol=symbol,
date=index.strftime('%Y-%m-%d'),
open=data['Open'],
high=data['High'],
low=data['Low'],
close=data['Close'],
volume=data['Volume'],
source='google')
google_stock.save()
for index, data in yahoo_data.iterrows():
if int(data['Volume']) > 0:
yahoo_stock = Stock(symbol=symbol,
date=index.strftime('%Y-%m-%d'),
open=data['Open'],
high=data['High'],
low=data['Low'],
close=data['Close'],
volume=data['Volume'],
source='yahoo')
yahoo_stock.save()
insert_files.append(
dict(symbol=symbol,
date=date,
path=filename,
stock=1,
contracts=contracts,
options=options))
parameters = dict(insert_files=insert_files)
return render(request, template, parameters)
def data_tos_thinkback_import_view(request, symbol):
"""
Import a symbol folder into db
:param request: request
:param symbol: str
:return: render
"""
template = 'data/run_csv.html'
symbol = symbol.upper()
""":type: str"""
insert_files = list()
# move files into year folder
no_year_files = glob(os.path.join(THINKBACK_DIR, symbol, '*.csv'))
years = sorted(list(set([
os.path.basename(f)[:4] for f in no_year_files
])))
for year in years:
year_dir = os.path.join(THINKBACK_DIR, symbol, year)
# make dir if not exists
if not os.path.isdir(year_dir):
os.mkdir(year_dir)
# move all year files into dir
for no_year_file in no_year_files:
#print os.path.basename(no_year_file)[:4], year
filename = os.path.basename(no_year_file)
if filename[:4] == year:
#print 'move ', no_year_file, ' -> ', year_dir
os.rename(no_year_file, os.path.join(year_dir, filename))
# get all files in year folder
files = []
for year in glob(os.path.join(THINKBACK_DIR, symbol, '*')):
for csv in glob(os.path.join(year, '*.csv')):
files.append(csv)
# stocks = list()
for csv in files:
inserted_file = dict(
path='',
stock=None,
contracts=0,
options=0
)
# get date and symbol
date, symbol = os.path.basename(csv)[:-4].split('-StockAndOptionQuoteFor')
# insert stock if not exists in db
query = Q(symbol=symbol) & Q(date=date) & Q(source='tos_thinkback')
if not Stock.objects.filter(query).exists() and date not in SKIP_DATES:
# output to console
print 'running %s file...' % os.path.basename(csv)
stock_data, option_data = OpenThinkBack(date=date, data=open(csv).read()).format()
# for view only
inserted_file['path'] = os.path.basename(csv)
stock = Stock()
stock.symbol = symbol
stock.source = 'tos_thinkback'
stock.data = stock_data
stock.save()
# stocks.append(stock)
# for view only
inserted_file['stock'] = stock
option_codes = [contract['option_code'] for contract, _ in option_data]
size = 100
exists_option_codes = list()
for chunk in [option_codes[i:i + size] for i in range(0, len(option_codes), size)]:
exists_option_codes += [x[0] for x in OptionContract.objects.filter(
option_code__in=chunk).values_list('option_code')]
new_option_codes = set(option_codes) - set(exists_option_codes)
contracts = list()
for option_code in set(option_codes):
if option_code in new_option_codes:
try:
contract_dict = [
c for c, _ in option_data if c['option_code'] == option_code
][0]
except IndexError:
print option_code
raise Exception()
contract = OptionContract()
contract.symbol = symbol
contract.source = 'tos_thinkback'
contract.data = contract_dict
contracts.append(contract)
# insert option contract
OptionContract.objects.bulk_create(contracts)
# for view only
inserted_file['contracts'] = len(contracts)
option_contracts = list()
for chunk in [option_codes[i:i + size] for i in range(0, len(option_codes), size)]:
option_contracts += [
option_contract for option_contract in
OptionContract.objects.filter(option_code__in=chunk)
]
options = list()
for contract_dict, option_dict in option_data:
try:
option_contract = [
option_contract for option_contract in option_contracts
if option_contract.option_code == contract_dict['option_code']
][0]
except IndexError:
raise IndexError('Contract not inserted...')
option = Option()
option.option_contract = option_contract
option.data = option_dict
options.append(option)
# insert options
Option.objects.bulk_create(options)
# for view only
inserted_file['options'] = len(options)
# add into inserted
insert_files.append(inserted_file)
# missing files between dates
missing_files = list()
if Stock.objects.count() > 2 and Stock.objects.filter(symbol=symbol).exists():
bdays = bdate_range(
start=Stock.objects.filter(symbol=symbol).order_by('date').first().date,
end=Stock.objects.filter(symbol=symbol).order_by('date').last().date,
freq='B'
)
for bday in bdays:
try:
Stock.objects.get(
Q(symbol=symbol) & Q(source='tos_thinkback') &
Q(date=bday.strftime('%Y-%m-%d'))
)
except ObjectDoesNotExist:
if is_not_holiday(bday.strftime('%Y-%m-%d')) and \
is_not_offdays(bday.strftime('%m/%d/%y')):
missing_files.append(
dict(
filename='%s-StockAndOptionQuoteFor%s.csv' % (
bday.strftime('%Y-%m-%d'), symbol
),
date=bday.strftime('%m/%d/%y')
)
)
parameters = dict(
symbol=symbol,
insert_files=insert_files,
missing_files=missing_files
)
return render(request, template, parameters)
def data_daily_import_view(request):
"""
Import all csv files in daily folder
then insert web data for that date
:param request: request
:return: render
"""
template = 'data/daily.html'
insert_files = list()
files = [
path for path in
glob(os.path.join(THINKBACK_DIR, '_daily', '*.csv'))
]
for f in files:
contracts = 0
options = 0
# get filename and dir
filename = os.path.basename(f)
print 'running file: %s...' % filename
date, symbol = map(
lambda x: x.upper(),
filename[:-4].split('-StockAndOptionQuoteFor')
)
# file into dict
stock_data, option_data = OpenThinkBack(date=date, data=open(f).read()).format()
# save stock
stock = Stock()
stock.symbol = symbol
stock.source = 'tos_thinkback'
stock.data = stock_data
stock.save()
# save contract and option
for contract_dict, option_dict in option_data:
try:
contract = OptionContract.objects.get(option_code=contract_dict['option_code'])
except ObjectDoesNotExist:
contract = OptionContract()
contract.symbol = symbol
contract.source = 'tos_thinkback'
contract.data = contract_dict
contract.save()
contracts += 1
option = Option()
option.option_contract = contract
option.data = option_dict
option.save()
options += 1
# move file into folder
year = filename[:4]
year_dir = os.path.join(THINKBACK_DIR, symbol, year)
# make dir if not exists
if not os.path.isdir(year_dir):
os.mkdir(year_dir)
os.rename(f, os.path.join(year_dir, os.path.basename(f)))
# save data from web
google_data = get_data_google(
symbols=symbol,
# start='2015-04-01', end='2015-04-10', # test only
start=date, end=date,
adjust_price=True
)
yahoo_data = get_data_yahoo(
symbols=symbol,
# start='2015-04-01', end='2015-04-10', # test only
start=date, end=date,
adjust_price=True
)
for index, data in google_data.iterrows():
if int(data['Volume']) > 0:
google_stock = Stock(
symbol=symbol,
date=index.strftime('%Y-%m-%d'),
open=data['Open'],
high=data['High'],
low=data['Low'],
close=data['Close'],
volume=data['Volume'],
source='google'
)
google_stock.save()
for index, data in yahoo_data.iterrows():
if int(data['Volume']) > 0:
yahoo_stock = Stock(
symbol=symbol,
date=index.strftime('%Y-%m-%d'),
open=data['Open'],
high=data['High'],
low=data['Low'],
close=data['Close'],
volume=data['Volume'],
source='yahoo'
)
yahoo_stock.save()
insert_files.append(
dict(
symbol=symbol,
date=date,
path=filename,
stock=1,
contracts=contracts,
options=options
)
)
parameters = dict(
insert_files=insert_files
)
return render(request, template, parameters)
