def Download_Lastest_Data(name, start, Baseline, Crossline):
end = datetime.now()
DB = web.DataReader(name + str('.KS'), "yahoo", start, end)
if (Baseline != 'Adj Close'):
DB[Baseline] = pd.stats.moments.rolling_mean(DB['Adj Close'], int(Baseline[2:]) )
DB[Crossline] = pd.stats.moments.rolling_mean(DB['Adj Close'], int(Crossline[2:]) )
DB['DIFF'] = (DB[Baseline] - DB[Crossline]) # 'Adj Close' 'MA30'
# plt.plot(DB.index, DB['Adj Close'], 'r')
# plt.plot(DB.index, DB['MA240'], 'b')
# plt.plot(DB.index, DB['D240'], 'g')
# plt.axhline(y=0.0, xmin=0, xmax=1, hold=None)
# plt.show()
DB.to_csv(name + str('.csv'))
def test_fred_multi(self):
raise nose.SkipTest('buggy as of 2/18/14; maybe a data revision?')
names = ['CPIAUCSL', 'CPALTT01USQ661S', 'CPILFESL']
start = datetime(2010, 1, 1)
end = datetime(2013, 1, 27)
received = web.DataReader(names, "fred", start, end).head(1)
expected = DataFrame([[217.478, 0.99701529, 220.544]],
columns=names,
index=[pd.tslib.Timestamp('2010-01-01 00:00:00')])
expected.index.rename('DATE', inplace=True)
assert_frame_equal(received, expected, check_less_precise=True)
def test_fred(self):
# Throws an exception when DataReader can't get a 200 response from
# FRED.
start = datetime(2010, 1, 1)
end = datetime(2013, 1, 27)
received = web.DataReader("GDP", "fred", start, end)['GDP'].tail(1)[0]
self.assertEqual(int(received), 16535)
self.assertRaises(Exception, web.DataReader, "NON EXISTENT SERIES",
'fred', start, end)
def download_SPX(date=None):
'''
@summary: download SPX index
@param date: the date to create
@return: None
'''
if date == None:
date = dt.date.today()
start = dt.datetime(1990, 1, 1)
end = date
f = web.DataReader("^GSPC", 'yahoo', start, end).fillna('NaN')
f.to_csv(make_dir(date) + '/SPX.csv', date_format='%Y%m%d')
def get_historical_closes(ticker, start_date, end_date):
# get the data for the tickers. This will be a panel
p = web.DataReader(ticker, "yahoo", start_date, end_date)
# convert the panel to a DataFrame and selection only Adj Close
# while making all index levels columns
d = p.to_frame()['Adj Close'].reset_index()
# remove the columns
d.rename(columns={'minor': 'Ticker', 'Adj Close': 'Close'}, inplace=True)
# pivot each ticker to a column
pivoted = d.pivot(index='Date', columns='Ticker')
# and drop the one level on the columns
pivoted.columns = pivoted.columns.droplevel(0)
return pivoted
def build_plots_for_training(tickers):
"""
:param tickers:
:return:
>>> tickers = ['F','GOOG']
"""
prices = web.DataReader(tickers, 'yahoo', dt.datetime(1990, 1, 1),
dt.datetime(2014, 1, 1))
for ticker in tickers:
build_smoothed_snapshots(prices.ix[:, :, ticker]['Adj Close'],
ticker,
forward=True,
save_folder='random')
def superImpose():
serdate='2015-1-1'
sdate='2015-01-02'
l=['^VIX', 'SPLK', 'SPY', 'DATA']
for i in l:
DT = web.DataReader(name=i, data_source='yahoo',
start=serdate)
DT['CloseN'] = DT['Close']/DT['Close'][sdate]
#DT['CloseN'].plot(figsize=(8, 5), grid=True)
print DT['CloseN'].tail()
def __call__(self, stock, year=None):
import pandas.io.data as pdd
if year is None: year = self.default_year
try:
ret = pdd.DataReader(stock,
data_source='yahoo',
start='{}/1/1'.format(year))
except IOError:
return None
if not self.strict: return ret
first = ret.index.tolist()[0]
if year != first.year: return None
if 1 != first.month: return None
return ret
def home(request):
context = {}
context['requestMethod'] = request.META['REQUEST_METHOD']
if request.method == 'GET' :
if request.GET.__contains__('hidden') :
#if 'name' in context and 'end_date' in context and 'start_date' in context:
context['name'] = request.GET['name']
context['start_date'] = request.GET['start_date']
context['end_date'] = request.GET['end_date']
print context
requestContext = RequestContext(request, context)
templateIndex = loader.get_template('index.html')
renderedTemplate = templateIndex.render(requestContext)
response = HttpResponse()
response['Age'] = 120
response.write(renderedTemplate)
if 'name' in context:# and 'end_date' in context and 'start_date' in context:
company = context['name'].encode('ascii','ignore')
start_date = context['start_date'].encode('ascii','ignore')
end_date = context['end_date'].encode('ascii','ignore')
a = request.GET.get('name')
start_date = start_date.split('-')
end_date = end_date.split('-')
start = datetime.datetime(int(start_date[0]),int(start_date[1]),int(start_date[2]))
end = datetime.datetime(int(end_date[0]),int(end_date[1]),int(end_date[2]))
f = web.DataReader(company,'yahoo',start,end)
a = f['Close']
b = a.index.tolist()
array = []
for i in range(b.__len__()):
c = b[i]
s = str(c)[:10]
d = a.tolist()
e = round(d[i],2)
array.append(e)
return render_to_response('index.html',{'array':json.dumps(array), 'start_date':json.dumps(start_date), 'end_date':json.dumps(end_date)})
return render_to_response('index.html',{})
def __init__(self, stock,start,end):
'''
Constructor:
Do a few checking the input with Utilities.inputfunctions.
1. Check whether stock, start, end are null. If true, raise EmptyInputException.
2. Check whether start and end date is a valid date form. If false, raise DateInputException.
3. Check whether end date is before the current time. If false, raise EndDateException.
4. Check whether end date is more than start date. If false, raise DateRangeException.
If the dates satisfy 2,3,4, we can parse the start and end date.
5. Check whether stock is a valid stock name between the start date and end date,
which could be found in the yahoo finance data.
If true, parse the stock into its upper form. Otherwise, raise StockNameInputException.
Input:
stock(string): the name of stock, can be read into pandas.io.data.DataReader
start(string): the start time of date range from user input
end(string): the end time of date range from user input
Attributes:
stock(string): the name of stock
starttime(datatime): the start time of date range
endtime(datatime): the end time of date range
dataframe(pandas.dataframe): extract the data from the yahoo finance
close_price(pandas.series): the column 'Adj Close' in the dataframe
volume(pandas.series): the column 'Volume' in the dataframe
'''
if IsEmptyInput(stock,start,end):
raise EmptyInputException()
else:
if IsValidDate(start) and IsValidDate(end):
if IsValidEndDate(end):
if IsValidDateRange(start,end):
pass
else:
raise DateRangeException()
self.starttime = ParseDate(start)
self.endtime = ParseDate(end)
else:
raise EndDateException()
else:
raise DateInputException()
if IsValidStockName(stock,start,end):
self.stock = ParseStockName(stock)
else:
raise StockNameInputException()
self.dataframe = web.DataReader(stock,'yahoo',start,end)
self.close_price = self.dataframe['Adj Close']
self.volume = self.dataframe['Volume']
def test_fred(self):
"""
Throws an exception when DataReader can't get a 200 response from
FRED.
"""
start = datetime(2010, 1, 1)
end = datetime(2013, 01, 27)
try:
self.assertEquals(
web.DataReader("GDP", "fred", start, end)['GDP'].tail(1),
16010.2)
self.assertRaises(
Exception, lambda: web.DataReader("NON EXISTENT SERIES",
'fred', start, end))
except urllib2.URLError:
try:
urllib2.urlopen('http://google.com')
except urllib2.URLError:
raise nose.SkipTest
else:
raise
def test_yahoo(self):
# asserts that yahoo is minimally working and that it throws
# an excecption when DataReader can't get a 200 response from
# yahoo
start = datetime(2010,1,1)
end = datetime(2012,1,24)
try:
self.assertEquals(
pd.DataReader("F", 'yahoo', start, end)['Close'][-1],
12.82)
self.assertRaises(
Exception,
lambda: pd.DataReader("NON EXISTENT TICKER", 'yahoo',
start, end))
except urllib2.URLError:
try:
urllib2.urlopen('http://www.google.com')
except urllib2.URLError:
raise nose.SkipTest
else:
raise
def test_fred(self):
raise nose.SkipTest('buggy as of 2/14/16; maybe a data revision?')
# Throws an exception when DataReader can't get a 200 response from
# FRED.
start = datetime(2010, 1, 1)
end = datetime(2013, 1, 27)
received = web.DataReader("GDP", "fred", start, end)['GDP'].tail(1)[0]
self.assertTrue(int(received) > 10000)
self.assertRaises(Exception, web.DataReader, "NON EXISTENT SERIES",
'fred', start, end)
def build_plots_for_prediction(tickers):
"""
:param tickers:
:return:
>>> tickers = ['F','GOOG']
"""
prices = web.DataReader(tickers, 'yahoo', dt.datetime(2014, 1, 1),
dt.date.today())
for ticker in tickers:
build_smoothed_snapshots(prices.ix[:, :, ticker]['Adj Close'],
ticker,
forward=False,
save_folder='current')
def pull_one(sym, startdate, enddate):
if startdate is None:
start = datetime.datetime(2014,1,1) # choose a default time
else:
start = date(startdate)
if enddate is None:
today = datetime.date.today()
end = datetime.datetime(today.year, today.month, today.day)
else:
end = date(enddate)
security = web.DataReader(sym, 'yahoo', start, end)
beh = security['Adj Close']
beh.name = sym
return beh
def get_history_quotes(ticker):
today = datetime.now().strftime(DATE_FORMAT)
f = web.DataReader([ticker], "yahoo", start='2018-08-01', end=today)
return f
for ticker in symbols_list:
fn = "./quotes/" + ticker + "_day.csv";
if file_exists(fn):
f = get_daily_quote(ticker)
write_to_file(OLD, fn, f)
else:
f = get_history_quotes(ticker)
write_to_file(NEW, fn, f)
def test_google(self):
# asserts that google is minimally working and that it throws
# an exception when DataReader can't get a 200 response from
# google
start = datetime(2010, 1, 1)
end = datetime(2013, 1, 27)
for locale in self.locales:
with tm.set_locale(locale):
panel = web.DataReader("F", 'google', start, end)
self.assertEqual(panel.Close[-1], 13.68)
self.assertRaises(Exception, web.DataReader, "NON EXISTENT TICKER",
'google', start, end)
def results(symbol, trend1, trend2):
data = web.DataReader(symbol, data_source='yahoo')
data['Trend 1'] = pd.rolling_mean(data['Adj Close'], window=int(trend1))
data['Trend 2'] = pd.rolling_mean(data['Adj Close'], window=int(trend2))
layout = Layout(
xaxis=XAxis(showgrid=True, gridcolor='#bdbdbd', gridwidth=2),
yaxis=YAxis(showgrid=True, gridcolor='#bdbdbd', gridwidth=2)
)
fig = Figure(data=df_to_plotly(data[['Adj Close', 'Trend 1', 'Trend 2']]),
layout=layout)
plot = ply.plot(fig, auto_open=False)
table = data.tail().to_html()
return render_template('plotly.html', symbol=symbol,
plot=plot, table=table)
def test_fred(self):
"""
Throws an exception when DataReader can't get a 200 response from
FRED.
"""
start = datetime(2010, 1, 1)
end = datetime(2013, 1, 27)
self.assertEquals(
web.DataReader("GDP", "fred", start, end)['GDP'].tail(1),
15984.1)
self.assertRaises(Exception, web.DataReader, "NON EXISTENT SERIES",
'fred', start, end)
def getcurrent1():
import pandas as pd
import pandas.io.data as web # Package and modules for importing data; this code may change depending on pandas version
import datetime
# We will look at stock prices over the past year, starting at January 1, 2016
start = datetime.datetime(2016, 1, 1)
end = datetime.date.today()
# Let's get Apple stock data; Apple's ticker symbol is AAPL
# First argument is the series we want, second is the source ("yahoo" for Yahoo! Finance), third is the start date, fourth is the end date
apple = web.DataReader("AAPL", "yahoo", start, end)
type(apple)
def FetchDataFromWeb ():
global Config
print "\nFetching data ..."
try:
code = YahooFinanceConfig.select().where(YahooFinanceConfig.processing != True).get()
except:
print "\nReturn false, there are no more records left"
return False
pass
else:
#code.processing = True
lStartDateList = SplitDateIntoString(raw_input("Enter the Start Date [in dd/mm/yyyy format] : "))
lEndDateList = SplitDateIntoString(raw_input("Enter the End Date [in dd/mm/yyyy format] : "))
code.start = unicode(datetime.date(int(lStartDateList[2]), int(lStartDateList[1]), int(lStartDateList[0])))
code.end = unicode(datetime.date(int(lEndDateList[2]), int(lEndDateList[1]), int(lEndDateList[0])))
#code.save()
delete_quary = YahooFinanceResult.delete().where(YahooFinanceConfig.processing != True)
delete_quary.execute()
f=web.DataReader(code.ticker, 'yahoo', code.start, code.end)
for date, _value in f.iterrows():
try:
value = {
'open': _value['Open'],
'high': _value['High'],
'low': _value['Low'],
'volume': _value['Volume'],
'close': _value['Close']
}
record = YahooFinanceResult.create(
identifier = hashlib.sha256(code.ticker + str(date) + code.start + code.end).hexdigest(),
ticker = code.ticker,
date = (dateutil.parser.parse(str(date))),
inserted = int(time.time() * 1000),
open = _value['Open'],
high = _value['High'],
low = _value['Low'],
volume = _value['Volume'],
close = _value['Close']
)
print code.ticker + "[" + str(date) + '] => ' + json.dumps(value)
except Exception as E:
error = fail.Error.create(query = code.ticker, timestamp = int(time.time() * 1000), error = str(E))
print str(E)
pass
return code
def create_pairs_dataframe(datadir, symbols):
#Creates a pandas DataFrame containing the closing price
#of a pair of symbols based on CSV files containing a datetime
#stamp and OHLCV data
# Open the individual CSV files and read into pandas DataFrames
print "Importing CSV data..."
# sym1 = pd.io.parsers.read_csv(os.path.join(datadir, '%s.csv' % symbols[0]),
# header=0, index_col=0,
# names=['Datetime','Open','High','Low','Close','Volume','Adj Close'])
# sym2 = pd.io.parsers.read_csv(os.path.join(datadir, '%s.csv' % symbols[1]),
# header=0, index_col=0,
# names=['Datetime','Open','High','Low','Close','Volume','Adj Close'])
# Fetching Data through Yahoo Finance
sym1 = web.DataReader(symbols[0],
data_source='yahoo',
start='2015/6/1',
end=time.strftime("%Y/%m/%d"))
sym2 = web.DataReader(symbols[1],
data_source='yahoo',
start='2015/6/1',
end=time.strftime("%Y/%m/%d"))
#sym1 = web.DataReader(symbols[0], data_source='yahoo',start ='2009/1/1',end='2012/01/01')
#sym2 = web.DataReader(symbols[1], data_source='yahoo',start ='2009//1',end='2012/01/01')
print "Constructing dual matrix for %s and %s..." % symbols
pairs = pd.DataFrame(index=sym1.index)
pairs['%s_close' % symbols[0]] = sym1['Adj Close']
pairs['%s_close' % symbols[1]] = sym2['Adj Close']
pairs = pairs.dropna()
return pairs
def get_yahoo_data_type(self, l_sym, start_date, end_date, data_type):
start_month = int(start_date.split('/')[0])
start_day = int(start_date.split('/')[1])
start_year = int(start_date.split('/')[2])
end_month = int(end_date.split('/')[0])
end_day = int(end_date.split('/')[1])
end_year = int(end_date.split('/')[2])
start = datetime.datetime(start_year, start_month, start_day)
end = datetime.datetime(end_year, end_month, end_day)
df_prices = web.DataReader(l_sym, 'yahoo', start, end)
return df_prices[data_type]
def IsValidStockName(stock_name):
"""
Check whether the input is a valid stock name.
"""
if isinstance(stock_name, str):
#Check whether input of list has a valid form
try:
df = web.DataReader(stock_name, 'yahoo')
except:
return False
else:
return True
else:
return False
def downloadStockData(self,market_type,code,year1,month1,date1,year2,month2,date2):
def makeCode(market_type,code):
if market_type==1:
return "%s.KS" % (code)
return "%s.KQ" % (code)
start = datetime(year1, month1, date1)
end = datetime(year2, month2, date2)
try:
df = web.DataReader(makeCode(market_type,code), "yahoo", start, end)
return df
except:
print "!!! Fatal Error Occurred"
return None
def read_stock(code='all', engine='yahoo', start=None, end=None, sampling='D'):
"""
Read stock raw information from server
:param code: stock names to extract from KOSPI excel file
:param engine: DB server to extract. Ex) 'yahoo', 'google'
:param start: start day. Ex) '2013-1-1'
:param end: end day. Ex) '2016-5-23'
:param sampling: Sampling date interval. Ex) D: day, W: week, y: year
:return: DataFrame of stock information. Valid columns are Open,High,Low,Close,Volume,Adj_Close
"""
from Stock_Class import Stock
import pandas.io.data as web
if start is None:
start = '2015-1-1'
end = '2016-12-31'
if code == 'all':
StockList = pd.read_excel(
r'C:\Users\Administrator\PycharmProjects\Stock\Input\KOSPI Code.xlsx',
sheetname='Code',
dtypes={'Code': str})
code = StockList['Code'].values
name = StockList['Name'].values
Df = pd.DataFrame()
for i, st in enumerate(code):
try:
print(i, '-', st)
if engine == 'google':
st = str(st).replace('.KS', '')
temp = web.DataReader(st, engine, start, end)
temp = temp[temp['Volume'] != 0]
temp['Code'] = str(st)
temp['Name'] = name[i]
Df = zipping(Df, temp)
except:
pass
if sampling != 'D':
Df = Df.resample(sampling)
return Stock(Df)
def results():
symbol = request.form['symbol']
trend1 = request.form['trend1']
trend2 = request.form['trend2']
print symbol, trend1, trend2
data = web.DataReader(symbol, data_source='yahoo')
data['Trend 1'] = pd.rolling_mean(data['Adj Close'], window=int(trend1))
data['Trend 2'] = pd.rolling_mean(data['Adj Close'], window=int(trend2))
data[['Adj Close', 'Trend 1', 'Trend 2']].plot()
output = 'results.png'
plt.savefig('static/' + output)
table = data.tail().to_html()
return render_template('results.html',
symbol=symbol,
output=output,
table=table)
def run_algo (stocks):
count_fails = 0
for symbol in stocks:
if count_fails < 5:
try:
stock_df = web.DataReader(symbol, 'yahoo', start, end)
first_df = apply_algo_helper(stock=stock_df, symbol=symbol, trend_type='resistance', PREV_DF=None)
final_df = apply_algo_helper(stock=stock_df, symbol=symbol, trend_type='support', PREV_DF=first_df)
#print final_df.tail()
except Exception as e:
print "failed on: ", symbol, "reason: ", e
count_fails += 1
#print 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno)
for ticker in TOP_PICKS_order.order(ascending=False)[:5].index:
plot(TOP_PICKS_dict[ticker]['df'], ticker)
def getvalue(stocks, numshares):
now = datetime.datetime.now()
start = datetime.date(2016, 9, 1)
stockdata = web.DataReader(stocks, 'yahoo', start, now)
cleandata = stockdata.ix['Adj Close']
dataFrame = pd.DataFrame(cleandata)
dataFrame.reset_index(inplace=True, drop=False)
stocklist = dataFrame.as_matrix()
for x in range(1, len(stocklist[0])):
stocklist[:, x] *= int(numshares[x - 1])
dates = stocklist[:, 0]
stocklist = np.delete(stocklist, 0, 1)
portfolio = stocklist.sum(axis=1)
plt.plot(dates, portfolio)
plt.show()
def get_col(symbols, col, start, end, index):
if index not in symbols:
symbols.insert(0, index)
dates = pd.date_range(start, end)
df = pd.DataFrame(index=dates)
for symbol in symbols:
df_temp = web.DataReader(symbol, 'yahoo', start, end)[[col]]
df_temp = df_temp.rename(columns={col: symbol})
df = df.join(df_temp)
# drop dates market did not trade
if symbol == index:
df = df.dropna(subset=[index])
# fill missing data
df.fillna(method='ffill', inplace='TRUE')
df.fillna(method='bfill', inplace='TRUE')
return df
