def scrape_stock_price_f(stock_id, begin=begin_date, end=end_date):
# 抓取股票資料
stock_id = stock_id + ".TW"
df_temp = Fetcher(stock_id, begin, end).getHistorical()
#修正格式 ["dates","open","high","low","close","delta","volume"]
df_temp.iloc[:, 5] = df_temp.iloc[:, 4].rolling(2).apply(
lambda x: x.iloc[1] - x.iloc[0])
df_temp.columns = [
"dates", "open", "high", "low", "close", "delta", "volume"
]
'==========================增加欄位===================================='
#製作均線
df_temp["5MA"] = df_temp["close"].rolling(5).mean()
df_temp["10MA"] = df_temp["close"].rolling(10).mean()
df_temp["20MA"] = df_temp["close"].rolling(20).mean()
df_temp["60MA"] = df_temp["close"].rolling(60).mean()
#RSV
rsv =(df_temp['close']-df_temp['low'].rolling(window=9).min())/\
(df_temp['high'].rolling(window=9).max()-\
df_temp['low'].rolling(window=9).min())*100
df_temp["rsv"] = pd.DataFrame(rsv).fillna(0)
#製作KD線
df_temp["k"], df_temp["d"] = KD_line(df_temp)
#MACD
df_temp = pd.concat([df_temp, macd(df_temp)], axis=1)
#布林通道 (標準差的計算有問題)
df_temp = pd.concat([df_temp, bl_tunnel(df_temp)], axis=1)
# 增加股票代號
return df_temp
def DataDownloader():
start, end = [2010, 1, 1], [2019, 4, 1]
# keep the first row if duplicate company name
companies = pd.read_csv("companylist.csv").drop_duplicates(subset='Name',
keep="first")
companies = companies[companies.Sector == "Technology"]
Symbols = [symbol.rstrip() for symbol in companies.Symbol]
Stocks = Fetcher(
Symbols[0], start,
end).getHistorical().set_index("Date").rename_axis(None)[DATA]
Stocks.index = pd.DatetimeIndex(Stocks.index)
Stocks.columns = pd.MultiIndex.from_product([[Symbols[0]], Stocks.columns])
for s in Symbols[1:]:
df = Fetcher(
s, start,
end).getHistorical().set_index("Date").rename_axis(None)[DATA]
df.index = pd.DatetimeIndex(df.index)
df.columns = pd.MultiIndex.from_product([[s], df.columns])
if df.index[0] <= datetime.datetime(2010, 1, 4):
Stocks = pd.merge(
Stocks, df,
on=Stocks.index).set_index("key_0").rename_axis(None)
Stocks.to_csv(os.getcwd() + "/Stocks.csv")
def Y3_range(loop_holdings_range): data_3ylow = Fetcher(loop_holdings_range, three_year_ago, now_year) df_3y_range = data_3ylow.getHistorical() year3_low = 100000 year3_high = 0 for index, row in df_3y_range.iterrows(): if row['Open'] < year3_low: year3_low = row['Open'] if row['High'] < year3_low: year3_low = row['High'] if row['Low'] < year3_low: year3_low = row['Low'] if row['Close'] < year3_low: year3_low = row['Close'] for index, row in df_3y_range.iterrows(): if row['Open'] > year3_high: year3_high = row['Open'] if row['High'] > year3_high: year3_high = row['High'] if row['Low'] > year3_high: year3_high = row['Low'] if row['Close'] > year3_high: year3_high = row['Close'] return(year3_low, year3_high)
def _get_from_yahoo(self, date_time, failed_tries=0):
"""Fetch close price of instrument on given date from yahoo"""
if failed_tries >= config.getint("price_fetch_max_allowed_tries"):
log.warning(
"Refusing to fetch {} {} because of {} previous failed attempt. Returning 0.001"
.format(self.instrument.symbol_yahoo, date_time, failed_tries))
return 0.001
log.debug("{}: Fetching {}".format(self.instrument.symbol_yahoo,
date_time))
try:
data = Fetcher(self.instrument.symbol_yahoo,
[date_time.year, date_time.month, date_time.day])
except UnboundLocalError:
log.warning("Failed to fetch price: {} {}".format(
self.instrument.symbol_yahoo, date_time))
return self.get(date_time - timedelta(days=1),
failed_tries=failed_tries + 1).as_float
try:
price_as_float = data.getDatePrice().values[0][1]
except IndexError:
log.warning("Failed to fetch price: {} {}".format(
self.instrument.symbol_yahoo, date_time))
price_as_float = self.get(date_time - timedelta(days=1),
failed_tries=failed_tries + 1).as_float
self._write_to_database(price_as_float, date_time)
return price_as_float
def yahoo_finance(stok, m=0, y=0):
'Gets the data from Yahoo finance'
today = date.today()
df = pd.DataFrame(data={})
dif = pd.DataFrame(data={})
'Fix for months'
if y == 0 and m > 0:
y = 0
while m > today.month:
y += 1
m2 = today.month - m
m = 12 * y + m2
else:
m = today.month
'Gets the data from yahoo finance'
try:
stock_prices = Fetcher(stok, [today.year - y, m, today.day],
[today.year, today.month, today.day])
dif = stock_prices.getHistorical()
df[stok] = dif['Adj Close'].to_list()
if len(dif) < 55:
print('We need more data for: ' + stock_ticker)
except:
print(
'Oops! that was no valid ticker. Please delete or correct it from the Stock List file: ',
stok)
'Puts the date in a column'
df['Date'] = dif['Date'].to_list()
return df
def getHistory(stockCode):
stockCode = str(stockCode).zfill(4)
os.makedirs(f'C:\\Users\\Fung\\Downloads\\Financial Reports {stockCode} {compNameDict[stockCode]}', exist_ok=True)
dst = f'C:\\Users\\Fung\\Downloads\\Financial Reports {stockCode} {compNameDict[stockCode]}\\historical price {stockCode}.xlsx' #result destination
now = datetime.datetime.now()
year= int(now.year)
month = int(now.month)
date = int(now.day)
stockData = Fetcher(stockCode+'.hk', [1990,4,1], [year,month,date])
df = stockData.getHistorical()
df['Date'] = pd.to_datetime(df['Date'], format='%Y-%m-%d') #change date column from string to date
df.Date=df.Date.dt.strftime('%Y') #change date format to Year for pivot table
pt= pd.pivot_table(df, values='Close', index=['Date'], #Get Max and low stock price group by each year
aggfunc={'Close':[min, max]})
sortedPT = pt.sort_values('Date',axis=0,ascending=False)
writer = ExcelWriter(dst)
sortedPT.T.to_excel(writer,'Sheet1')
writer.save()
#reformat result to desire format
wb = openpyxl.load_workbook(dst)
ws = wb.active
lastCol = ws.max_column
for i in range(lastCol,2,-1):
ws.insert_cols(i,2)
wb.save(dst)
os.startfile(f'C:\\Users\\Fung\\Downloads\\Financial Reports {stockCode} {compNameDict[stockCode]}') #open report folder
def histdata():
a = datetime.datetime.now()
a = a.strftime('%Y-%m-%d')
namasaham = request.form['namasaham']
tgl1 = request.form['tanggal1']
tgl2 = request.form['tanggal2']
def convert_date(date):
date = datetime.datetime.strptime(date, '%Y-%m-%d')
tanggal = date.strftime('%d')
if tanggal[0] == str(0):
tanggal1 = int(tanggal[1])
else:
tanggal1 = int(tanggal)
bulan = date.strftime('%m')
if bulan[0] == str(0):
bulan1 = int(bulan[1])
else:
bulan1 = int(bulan)
tahun = date.strftime('%Y')
tahun1 = int(tahun)
tanggal_lengkap = str(tahun + '-' + bulan + '-' + tanggal)
return tanggal1, bulan1, tahun1, tanggal_lengkap
tgl1_1, bulan1_1, tahun1_1, tgl_lengkap1 = convert_date(tgl1)
tgl2_1, bulan2_1, tahun2_1, tgl_lengkap2 = convert_date(tgl2)
#import data stock price
real_saham = Fetcher(namasaham + ".JK", [tahun1_1, bulan1_1, tgl1_1],
[tahun2_1, bulan2_1, tgl2_1],
interval="1d")
real_saham = real_saham.getHistorical()
real_saham = real_saham.dropna()
#plot data
plt.plot(real_saham['Date'], real_saham['Open'], 'blue')
plt.title('Plot Stock Price of {}'.format(namasaham))
plt.xlabel('Date')
plt.ylabel('Price (Rp)')
plt.xticks(real_saham['Date'], rotation=90)
plt.tight_layout()
#filename
b = datetime.datetime.now()
namafile = (str(b).split('.'))[-1]
addressplot = './storage/{}.png'.format(namafile)
urlplot = '/fileupload/{}.png'.format(namafile)
plt.savefig(addressplot)
plot = urlplot
plt.close()
return render_template('histdata.html',
tables=real_saham.to_html(),
a=a,
tgl1=tgl_lengkap1,
tgl2=tgl_lengkap2,
namasaham=namasaham,
plot=plot)
def download_yahoo_sp(ticker):
today = datetime.today()
data = Fetcher(ticker, [2000, 1, 1], [today.year, today.month, today.day])
price_dict = data.getPriceDict()
new_filename = (destination_folder + ticker + ".txt")
with open(new_filename, 'w') as f:
json.dump(price_dict, f)
return price_dict
def get_asx(self, tick, start_date, end_date):
''' get prices for date range '''
tick_code = '%s.ax' % tick
ticker = Fetcher(tick_code, start_date, end_date)
time_series = ticker.getHistorical()
time_series['Tick'] = tick
time_series.rename(columns=lambda x: x.strip(), inplace=True)
return time_series
def today_open(loop_holdings_today): data_today = Fetcher(loop_holdings_today, now_year) df_today = data_today.getHistorical() x = 0 for index, row in df_today.iterrows(): if row['Open'] > x: x = row['Open'] return(x)
def __init__(self, ticker):
from yahoo_historical import Fetcher
import datetime
now = datetime.datetime.now()
raw = Fetcher(ticker, [2018,1,1], [now.year,now.month,now.day])
self.data = raw.getHistorical()
self.name = ticker
self.features = []
#The data -- raw and unpruned. Needed for passing data into functions
self.r_opens = self.data['Open']
self.r_highs = self.data['High']
self.r_lows = self.data['Low']
self.r_closes = self.data['Close']
self.r_adj_close = self.data['Adj Close']
self.r_volumes = self.data['Volume']
self.m_opens = self.data['Open'][2:-1]
self.m_highs = self.data['High'][2:-1]
self.m_lows = self.data['Low'][2:-1]
self.m_closes = self.data['Close'][2:-1]
self.m_adj_close = self.data['Adj Close'][2:-1]
self.m_volumes = self.data['Volume'][2:-1]
self.pc_open_close = self.percentChange(self.r_opens, self.r_closes)
self.pc_high_low = self.percentChange(self.r_lows, self.r_highs)
self.rv_opens = self.relVelocity(self.r_opens)
self.rv_highs = self.relVelocity(self.r_highs)
self.rv_lows = self.relVelocity(self.r_lows)
self.rv_closes = self.relVelocity(self.r_closes)
self.rv_adj_closes = self.relVelocity(self.r_adj_close)
self.rv_volumes = self.relVelocity(self.r_volumes)
self.rp_opens = self.relPos(self.r_opens)
self.rp_highs = self.relPos(self.r_highs)
self.rp_lows = self.relPos(self.r_lows)
self.rp_closes = self.relPos(self.r_closes)
self.rp_adj_closes = self.relPos(self.r_adj_close)
self.rp_volumes = self.relPos(self.r_volumes)
self.features.append(self.m_adj_close)
self.features.append(self.m_closes)
self.features.append(self.m_highs)
self.features.append(self.m_lows)
self.features.append(self.m_opens)
self.features.append(self.m_volumes)
self.y = self.percentChangeY(self.r_closes, self.r_closes)#r_opens
def get_daily_returns(symbol, from_date, to_date):
# from and to date format in list [yyyy,mm,dd]
data = Fetcher(symbol, from_date, to_date)
close = data.getHistorical()["Close"].tolist()
returns = [None]
for close2, close1 in zip(close[1:], close):
returns.append(close2 / close1 - 1)
return returns
def validate_ticker(ticker):
"""Function that validates ticker from yahoo_historical api"""
now = arrow.now()
now = get_date_array_for_fetcher(now)
try:
fetcher = Fetcher(ticker, now, now)
fetcher.getHistorical()
except KeyError:
return False
return True
def fill_quote_history(stock):
"""
Given a stock, it fills in the last 10 years of historical data
"""
now = arrow.now()
ten_back = now.replace(years=-10)
now = get_date_array_for_fetcher(now)
ten_back = get_date_array_for_fetcher(ten_back)
fetcher = Fetcher(stock.ticker, ten_back, now)
history = fetcher.getHistorical()
save_stock_quote_from_fetcher(history, stock.id)
def get_ticker_update(symbs, ndays):
# start = datetime(2013,11, 13)
# end = datetime(2017, 5, 24)
start = datetime.today() - timedelta(days=ndays)
start1 = [
int(start.strftime('%Y')),
int(start.strftime('%m')),
int(start.strftime('%d'))
]
end = datetime.today()
end1 = [
int(end.strftime('%Y')),
int(end.strftime('%m')),
int(end.strftime('%d'))
]
result = {}
n = 0
k = 0
failed = []
t0 = time()
CRED = '\033[91m'
CEND = '\033[0m'
for symb in symbs:
try:
data = Fetcher(symb, start1, end1)
df = data.getHistorical()
#df.reset_index(level=0,inplace=True)
# df1={}
# df['Adj_Close']=df['Adj Close']
# del df['Adj Close']
# df1['data']=df.to_dict('list')
# df1['t']=df1['data']['Date']
# df1['symbol']=symb
result.update({symb: df})
n = n + 1
print('TK' + str(n) + ' :: ' + symb + ' loaded sucessfully')
except:
failed.append(symb)
n = n + 1
k = k + 1
print(CRED + 'TK' + str(n) + ' :: ' + symb + ' download failed',
'red' + CEND)
continue
# sio.savemat('result.mat',{'tsPY':result})
print('All done Total time ' + str(int(time() - t0)) + ' Seconds')
print('Total ticker tried: ' + str(n))
print('Sucessfully loaded: ' + str(n - k))
print('Failed loaded: ' + str(k))
print(*failed, sep=',')
return (result)
#ts=get_ticker_update(['C','A'],100)
def find_price(SYB):
now = datetime.datetime.now()
time_now = [now.year, now.month, now.day]
data = Fetcher(SYB, time_now)
data_get = data.getHistorical()
#print (data_get)
open = data_get['Open']
open_price = float(open[0])
#print (SYB, open_price)
return open_price
def update_graph(selected_dropdown_value):
global stockpricedf # Needed to modify global copy of stockpricedf
ticker_fb = Fetcher(selected_dropdown_value, [yr, mo, dy])
#ticker_fb = Fetcher('fb', [yr,mo,dy])
stockpricedf = ticker_fb.get_historical()
stockpricedf['Date'] = pd.to_datetime(stockpricedf['Date'])
stockpricedf['Close'] = stockpricedf['Close']
return {
'data': [{
'x': stockpricedf.Date,
'y': stockpricedf.Close,
'line': {
'color': 'green'
}
}]
}
def calc_avg(ticker: str, t_minus_1: datetime, back_days: int) -> Avg:
start_date = t_minus_1 - timedelta(days=back_days)
end_date = t_minus_1 + timedelta(days=1)
d1 = [start_date.year, start_date.month, start_date.day]
d2 = [end_date.year, end_date.month, end_date.day]
data = Fetcher(ticker, d1, d2).get_historical()
return do_calc_avg(data)
def run_Fetcher(stock_name, start, end, cycle):
#print(stock_name)
AAPL = Fetcher(stock_name, start, end, cycle)
date_close = AAPL.getDatePrice()
close = date_close.ix[:, [1]]
summ = 0
ar = np.empty
for i in range(len(close)):
summ += close.ix[i][0]
ar = np.append(ar, close.ix[i][0])
ar = ar[1:]
return ar
def plot(self, *args):
if args:
ticker = args[0]
else:
ticker = self.ticker
if ticker == None:
raise InvalidTickerException(str(ticker))
try:
data = Fetcher(ticker, [2007, 1, 1]).getDatePrice()
except KeyError:
raise InvalidTickerException(str(ticker))
except IndexError:
raise InvalidTickerException(str(ticker))
except UnboundLocalError:
raise InvalidTickerException(str(ticker))
dates = []
for date in data["Date"]:
date = date.split("-")
date[0], date[1] = date[1], date[0]
date[1], date[2] = date[2], date[1]
dates.append("/".join(date))
dates = [dt.datetime.strptime(d, '%m/%d/%Y').date() for d in dates]
self.figure.clear() # reset the figure
self.figure.suptitle(ticker)
ax = self.figure.add_subplot(111)
ax.plot(dates, data["Adj Close"])
self.canvas.draw()
def get_fetcher():
try:
fetch = Fetcher(self.symbols.iloc[0]['Symbol'],
self.start_yahoo, self.end_yahoo)
return fetch
except KeyError:
get_fetcher()
def fill():
"""Function that fills stock data for missing days"""
stock_id_field = 'stock_id'
stock_ticker = 'stock__ticker'
date = 'date'
now = arrow.now()
data = DailyStockQuote.objects.values(
stock_ticker, stock_id_field).annotate(date=Max(date))
for stock in data:
last_date = arrow.get(stock[date]).replace(days=+1)
last_date = get_date_array_for_fetcher(last_date)
now_fetcher = get_date_array_for_fetcher(now)
ticker = stock[stock_ticker]
stock_id = stock[stock_id_field]
fetcher = Fetcher(ticker, last_date, now_fetcher)
history = fetcher.getHistorical()
save_stock_quote_from_fetcher(history, stock_id)
def scrapeASX_splits(xCode, xYear):
xDateStart = [xYear, 1, 1]
xDateEnd = [xYear, 12, 31]
try:
ASX_splits = Fetcher(xCode + ".AX", xDateStart, xDateEnd).getSplits()
ASX_splits['Code'] = xCode
except Exception as e:
ASX_splits = pd.DataFrame()
return (ASX_splits)
def run_Fetcher(stock_name, start, end, cycle): AAPL = Fetcher(stock_name, start , end , cycle) date_close = AAPL.getHistorical() #print(date_close) close = date_close.ix[:,[1,2,3]] #print(close) ar = np.empty for i in range(len(close)): typical = (close.ix[i][0] + close.ix[i][1] + close.ix[i][2]) / 3.0 ar = np.append(ar, typical) ar = ar[1:] return ar
def get_daily_price_returns(symbol, from_date, to_date, get_returns=True):
# from and to date format in list [yyyy,mm,dd]
df = Fetcher(symbol, from_date, to_date).getHistorical()
if get_returns:
df["returns"] = get_daily_returns(symbol, from_date, to_date)
df = df.iloc[1:, :]
return df
def post():
K.clear_session()
namasaham = request.form['namasaham']
model = joblib.load('model' + namasaham)
#split tanggal,bulan,tahun
date = request.form['tanggal']
date = datetime.datetime.strptime(date, '%Y-%m-%d')
tanggal = date.strftime('%d')
if tanggal[0] == str(0):
tanggal1 = int(tanggal[1])
else:
tanggal1 = int(tanggal)
print(tanggal1)
bulan = date.strftime('%m')
if bulan[0] == str(0):
bulan1 = int(bulan[1])
else:
bulan1 = int(bulan)
tahun = date.strftime('%Y')
tahun1 = int(tahun)
tanggal_predict = str(tahun + '-' + bulan + '-' + tanggal)
#import data stock price
real_saham = Fetcher(namasaham + ".JK", [2018, 1, 1],
[tahun1, bulan1, tanggal1],
interval="1d")
real_saham = real_saham.getHistorical()
real_saham = real_saham.iloc[:, 0:2]
real_saham = real_saham.dropna()
#set index and drop data in predict_date
real_saham = real_saham.set_index("Date")
if tanggal_predict in real_saham.index.values:
real_saham = real_saham.drop(tanggal_predict, axis=0)
real_saham = real_saham.tail(10)
#transform
from sklearn.preprocessing import MinMaxScaler
sc = MinMaxScaler()
saham = sc.fit_transform(real_saham)
window = 3
predictX = dataset_predict.createDataset(saham, window)
predictX = predictX.reshape(len(saham) - window + 1, window, 1)
predictY = model.predict(predictX)
#denormalisasi
predictY = sc.inverse_transform(predictY)
predict_next_day = "Rp. {}".format(int(predictY[-1][0]))
price_previous_day = "Rp. {}".format(int(real_saham['Open'][-1]))
previous_date = real_saham.index[-1]
return render_template('predict.html',
predict=predict_next_day,
price_previous=price_previous_day,
tanggal_predict=tanggal_predict,
previous_date=previous_date,
namasaham=namasaham)
def scrapeASX_price(xCode, xYear):
xDateStart = [xYear, 1, 1]
xDateEnd = [xYear, 12, 31]
try:
ASX_price = Fetcher(xCode + ".AX", xDateStart,
xDateEnd).get_historical()
ASX_price['Code'] = xCode
except Exception as e:
ASX_price = pd.DataFrame()
return (ASX_price)
def loadCsv(symbols: list,
output_dir: str,
start_date=[2015, 5, 24],
end_date=[2020, 10, 9]):
import time
for symbol in symbols:
print("generate csv to symbol ", symbol, "...")
time.sleep(2)
data = Fetcher(symbol, start_date, end_date)
try:
with open(output_dir + '/' + symbol + '.csv', 'w') as csvfile:
dataf = pd.DataFrame(data.getHistorical())
if dataf.empty:
raise Exception('dataframe is empty')
dataf.to_csv(path_or_buf=csvfile, mode='w')
print("success generate csv to symbol ", symbol)
except:
print("FAILD!!!! generate csv to symbol ", symbol)
def yahoo_api(tkr, reset):
poor_data = False
if not reset:
return pd.read_csv('stocks.csv'), poor_data
data = Fetcher(tkr, [2000, 1, 1])
df = data.getHistorical()
df = df.drop(['Close'], axis=1)
df = df.rename(columns={
'High': 'Max',
'Low': 'Min',
'Adj Close': 'Close',
'Volume': 'Vol'
})
df.iloc[:] = df.iloc[::-1].values
df.iloc[:, 0] = list(map(lambda x: x.replace('-', '.'), df.iloc[:, 0]))
df['Date'] = df['Date'].map(lambda x: '.'.join(x.split('.')[::-1]))
df['last_close'] = df['Close'].shift(-1)
df['Var'] = list(
map(lambda x, y: (x / y) - 1, df['Close'], df['last_close']))
df[['Open', 'Max', 'Min', 'Close',
'Vol']] = df[['Open', 'Max', 'Min', 'Close',
'Vol']].applymap(lambda x: x if x != 0 else None)
if df.isnull().sum().max() / len(df.index) > 0.3 or len(df.index) < 50:
poor_data = True
df[['Open', 'Max', 'Min', 'Close',
'Vol']] = df[['Open', 'Max', 'Min', 'Close',
'Vol']].fillna(method='bfill')
df['last_close'] = df['Close'].shift(-1)
df['Var'] = list(
map(lambda x, y: (x / y) - 1, df['Close'], df['last_close']))
df = df[['Date', 'Open', 'Close', 'Min', 'Max', 'Vol', 'Var']]
#if df[['Vol']].sum(axis=0)[0] == 0: df = df.drop(columns=['Vol'])
df = df.dropna()
df.to_csv('stocks.csv')
return df, poor_data
def data_fetcher(etflist, start, end):
"""
Takes in a list of etf symbols
returns pandas dataframe of price details and dividends
"""
price_df = pd.DataFrame()
errors = []
for e in etflist:
try:
data = Fetcher(e, start, end)
pricedata = data.getHistorical()
if 'Date' in data.getHistorical().columns.tolist():
pricedata['ticker'] = e
price_df = price_df.append(pricedata)
except KeyError:
errors.append(e)
else:
pass
return price_df, errors
