def scrape_stock_price_f(stock_id, begin=begin_date, end=end_date): # 抓取股票資料 stock_id = stock_id + ".TW" df_temp = Fetcher(stock_id, begin, end).getHistorical() #修正格式 ["dates","open","high","low","close","delta","volume"] df_temp.iloc[:, 5] = df_temp.iloc[:, 4].rolling(2).apply( lambda x: x.iloc[1] - x.iloc[0]) df_temp.columns = [ "dates", "open", "high", "low", "close", "delta", "volume" ] '==========================增加欄位====================================' #製作均線 df_temp["5MA"] = df_temp["close"].rolling(5).mean() df_temp["10MA"] = df_temp["close"].rolling(10).mean() df_temp["20MA"] = df_temp["close"].rolling(20).mean() df_temp["60MA"] = df_temp["close"].rolling(60).mean() #RSV rsv =(df_temp['close']-df_temp['low'].rolling(window=9).min())/\ (df_temp['high'].rolling(window=9).max()-\ df_temp['low'].rolling(window=9).min())*100 df_temp["rsv"] = pd.DataFrame(rsv).fillna(0) #製作KD線 df_temp["k"], df_temp["d"] = KD_line(df_temp) #MACD df_temp = pd.concat([df_temp, macd(df_temp)], axis=1) #布林通道 (標準差的計算有問題) df_temp = pd.concat([df_temp, bl_tunnel(df_temp)], axis=1) # 增加股票代號 return df_temp
def DataDownloader(): start, end = [2010, 1, 1], [2019, 4, 1] # keep the first row if duplicate company name companies = pd.read_csv("companylist.csv").drop_duplicates(subset='Name', keep="first") companies = companies[companies.Sector == "Technology"] Symbols = [symbol.rstrip() for symbol in companies.Symbol] Stocks = Fetcher( Symbols[0], start, end).getHistorical().set_index("Date").rename_axis(None)[DATA] Stocks.index = pd.DatetimeIndex(Stocks.index) Stocks.columns = pd.MultiIndex.from_product([[Symbols[0]], Stocks.columns]) for s in Symbols[1:]: df = Fetcher( s, start, end).getHistorical().set_index("Date").rename_axis(None)[DATA] df.index = pd.DatetimeIndex(df.index) df.columns = pd.MultiIndex.from_product([[s], df.columns]) if df.index[0] <= datetime.datetime(2010, 1, 4): Stocks = pd.merge( Stocks, df, on=Stocks.index).set_index("key_0").rename_axis(None) Stocks.to_csv(os.getcwd() + "/Stocks.csv")
def Y3_range(loop_holdings_range): data_3ylow = Fetcher(loop_holdings_range, three_year_ago, now_year) df_3y_range = data_3ylow.getHistorical() year3_low = 100000 year3_high = 0 for index, row in df_3y_range.iterrows(): if row['Open'] < year3_low: year3_low = row['Open'] if row['High'] < year3_low: year3_low = row['High'] if row['Low'] < year3_low: year3_low = row['Low'] if row['Close'] < year3_low: year3_low = row['Close'] for index, row in df_3y_range.iterrows(): if row['Open'] > year3_high: year3_high = row['Open'] if row['High'] > year3_high: year3_high = row['High'] if row['Low'] > year3_high: year3_high = row['Low'] if row['Close'] > year3_high: year3_high = row['Close'] return(year3_low, year3_high)
def _get_from_yahoo(self, date_time, failed_tries=0): """Fetch close price of instrument on given date from yahoo""" if failed_tries >= config.getint("price_fetch_max_allowed_tries"): log.warning( "Refusing to fetch {} {} because of {} previous failed attempt. Returning 0.001" .format(self.instrument.symbol_yahoo, date_time, failed_tries)) return 0.001 log.debug("{}: Fetching {}".format(self.instrument.symbol_yahoo, date_time)) try: data = Fetcher(self.instrument.symbol_yahoo, [date_time.year, date_time.month, date_time.day]) except UnboundLocalError: log.warning("Failed to fetch price: {} {}".format( self.instrument.symbol_yahoo, date_time)) return self.get(date_time - timedelta(days=1), failed_tries=failed_tries + 1).as_float try: price_as_float = data.getDatePrice().values[0][1] except IndexError: log.warning("Failed to fetch price: {} {}".format( self.instrument.symbol_yahoo, date_time)) price_as_float = self.get(date_time - timedelta(days=1), failed_tries=failed_tries + 1).as_float self._write_to_database(price_as_float, date_time) return price_as_float
def yahoo_finance(stok, m=0, y=0): 'Gets the data from Yahoo finance' today = date.today() df = pd.DataFrame(data={}) dif = pd.DataFrame(data={}) 'Fix for months' if y == 0 and m > 0: y = 0 while m > today.month: y += 1 m2 = today.month - m m = 12 * y + m2 else: m = today.month 'Gets the data from yahoo finance' try: stock_prices = Fetcher(stok, [today.year - y, m, today.day], [today.year, today.month, today.day]) dif = stock_prices.getHistorical() df[stok] = dif['Adj Close'].to_list() if len(dif) < 55: print('We need more data for: ' + stock_ticker) except: print( 'Oops! that was no valid ticker. Please delete or correct it from the Stock List file: ', stok) 'Puts the date in a column' df['Date'] = dif['Date'].to_list() return df
def getHistory(stockCode): stockCode = str(stockCode).zfill(4) os.makedirs(f'C:\\Users\\Fung\\Downloads\\Financial Reports {stockCode} {compNameDict[stockCode]}', exist_ok=True) dst = f'C:\\Users\\Fung\\Downloads\\Financial Reports {stockCode} {compNameDict[stockCode]}\\historical price {stockCode}.xlsx' #result destination now = datetime.datetime.now() year= int(now.year) month = int(now.month) date = int(now.day) stockData = Fetcher(stockCode+'.hk', [1990,4,1], [year,month,date]) df = stockData.getHistorical() df['Date'] = pd.to_datetime(df['Date'], format='%Y-%m-%d') #change date column from string to date df.Date=df.Date.dt.strftime('%Y') #change date format to Year for pivot table pt= pd.pivot_table(df, values='Close', index=['Date'], #Get Max and low stock price group by each year aggfunc={'Close':[min, max]}) sortedPT = pt.sort_values('Date',axis=0,ascending=False) writer = ExcelWriter(dst) sortedPT.T.to_excel(writer,'Sheet1') writer.save() #reformat result to desire format wb = openpyxl.load_workbook(dst) ws = wb.active lastCol = ws.max_column for i in range(lastCol,2,-1): ws.insert_cols(i,2) wb.save(dst) os.startfile(f'C:\\Users\\Fung\\Downloads\\Financial Reports {stockCode} {compNameDict[stockCode]}') #open report folder
def histdata(): a = datetime.datetime.now() a = a.strftime('%Y-%m-%d') namasaham = request.form['namasaham'] tgl1 = request.form['tanggal1'] tgl2 = request.form['tanggal2'] def convert_date(date): date = datetime.datetime.strptime(date, '%Y-%m-%d') tanggal = date.strftime('%d') if tanggal[0] == str(0): tanggal1 = int(tanggal[1]) else: tanggal1 = int(tanggal) bulan = date.strftime('%m') if bulan[0] == str(0): bulan1 = int(bulan[1]) else: bulan1 = int(bulan) tahun = date.strftime('%Y') tahun1 = int(tahun) tanggal_lengkap = str(tahun + '-' + bulan + '-' + tanggal) return tanggal1, bulan1, tahun1, tanggal_lengkap tgl1_1, bulan1_1, tahun1_1, tgl_lengkap1 = convert_date(tgl1) tgl2_1, bulan2_1, tahun2_1, tgl_lengkap2 = convert_date(tgl2) #import data stock price real_saham = Fetcher(namasaham + ".JK", [tahun1_1, bulan1_1, tgl1_1], [tahun2_1, bulan2_1, tgl2_1], interval="1d") real_saham = real_saham.getHistorical() real_saham = real_saham.dropna() #plot data plt.plot(real_saham['Date'], real_saham['Open'], 'blue') plt.title('Plot Stock Price of {}'.format(namasaham)) plt.xlabel('Date') plt.ylabel('Price (Rp)') plt.xticks(real_saham['Date'], rotation=90) plt.tight_layout() #filename b = datetime.datetime.now() namafile = (str(b).split('.'))[-1] addressplot = './storage/{}.png'.format(namafile) urlplot = '/fileupload/{}.png'.format(namafile) plt.savefig(addressplot) plot = urlplot plt.close() return render_template('histdata.html', tables=real_saham.to_html(), a=a, tgl1=tgl_lengkap1, tgl2=tgl_lengkap2, namasaham=namasaham, plot=plot)
def download_yahoo_sp(ticker): today = datetime.today() data = Fetcher(ticker, [2000, 1, 1], [today.year, today.month, today.day]) price_dict = data.getPriceDict() new_filename = (destination_folder + ticker + ".txt") with open(new_filename, 'w') as f: json.dump(price_dict, f) return price_dict
def get_asx(self, tick, start_date, end_date): ''' get prices for date range ''' tick_code = '%s.ax' % tick ticker = Fetcher(tick_code, start_date, end_date) time_series = ticker.getHistorical() time_series['Tick'] = tick time_series.rename(columns=lambda x: x.strip(), inplace=True) return time_series
def today_open(loop_holdings_today): data_today = Fetcher(loop_holdings_today, now_year) df_today = data_today.getHistorical() x = 0 for index, row in df_today.iterrows(): if row['Open'] > x: x = row['Open'] return(x)
def __init__(self, ticker): from yahoo_historical import Fetcher import datetime now = datetime.datetime.now() raw = Fetcher(ticker, [2018,1,1], [now.year,now.month,now.day]) self.data = raw.getHistorical() self.name = ticker self.features = [] #The data -- raw and unpruned. Needed for passing data into functions self.r_opens = self.data['Open'] self.r_highs = self.data['High'] self.r_lows = self.data['Low'] self.r_closes = self.data['Close'] self.r_adj_close = self.data['Adj Close'] self.r_volumes = self.data['Volume'] self.m_opens = self.data['Open'][2:-1] self.m_highs = self.data['High'][2:-1] self.m_lows = self.data['Low'][2:-1] self.m_closes = self.data['Close'][2:-1] self.m_adj_close = self.data['Adj Close'][2:-1] self.m_volumes = self.data['Volume'][2:-1] self.pc_open_close = self.percentChange(self.r_opens, self.r_closes) self.pc_high_low = self.percentChange(self.r_lows, self.r_highs) self.rv_opens = self.relVelocity(self.r_opens) self.rv_highs = self.relVelocity(self.r_highs) self.rv_lows = self.relVelocity(self.r_lows) self.rv_closes = self.relVelocity(self.r_closes) self.rv_adj_closes = self.relVelocity(self.r_adj_close) self.rv_volumes = self.relVelocity(self.r_volumes) self.rp_opens = self.relPos(self.r_opens) self.rp_highs = self.relPos(self.r_highs) self.rp_lows = self.relPos(self.r_lows) self.rp_closes = self.relPos(self.r_closes) self.rp_adj_closes = self.relPos(self.r_adj_close) self.rp_volumes = self.relPos(self.r_volumes) self.features.append(self.m_adj_close) self.features.append(self.m_closes) self.features.append(self.m_highs) self.features.append(self.m_lows) self.features.append(self.m_opens) self.features.append(self.m_volumes) self.y = self.percentChangeY(self.r_closes, self.r_closes)#r_opens
def get_daily_returns(symbol, from_date, to_date): # from and to date format in list [yyyy,mm,dd] data = Fetcher(symbol, from_date, to_date) close = data.getHistorical()["Close"].tolist() returns = [None] for close2, close1 in zip(close[1:], close): returns.append(close2 / close1 - 1) return returns
def validate_ticker(ticker): """Function that validates ticker from yahoo_historical api""" now = arrow.now() now = get_date_array_for_fetcher(now) try: fetcher = Fetcher(ticker, now, now) fetcher.getHistorical() except KeyError: return False return True
def fill_quote_history(stock): """ Given a stock, it fills in the last 10 years of historical data """ now = arrow.now() ten_back = now.replace(years=-10) now = get_date_array_for_fetcher(now) ten_back = get_date_array_for_fetcher(ten_back) fetcher = Fetcher(stock.ticker, ten_back, now) history = fetcher.getHistorical() save_stock_quote_from_fetcher(history, stock.id)
def get_ticker_update(symbs, ndays): # start = datetime(2013,11, 13) # end = datetime(2017, 5, 24) start = datetime.today() - timedelta(days=ndays) start1 = [ int(start.strftime('%Y')), int(start.strftime('%m')), int(start.strftime('%d')) ] end = datetime.today() end1 = [ int(end.strftime('%Y')), int(end.strftime('%m')), int(end.strftime('%d')) ] result = {} n = 0 k = 0 failed = [] t0 = time() CRED = '\033[91m' CEND = '\033[0m' for symb in symbs: try: data = Fetcher(symb, start1, end1) df = data.getHistorical() #df.reset_index(level=0,inplace=True) # df1={} # df['Adj_Close']=df['Adj Close'] # del df['Adj Close'] # df1['data']=df.to_dict('list') # df1['t']=df1['data']['Date'] # df1['symbol']=symb result.update({symb: df}) n = n + 1 print('TK' + str(n) + ' :: ' + symb + ' loaded sucessfully') except: failed.append(symb) n = n + 1 k = k + 1 print(CRED + 'TK' + str(n) + ' :: ' + symb + ' download failed', 'red' + CEND) continue # sio.savemat('result.mat',{'tsPY':result}) print('All done Total time ' + str(int(time() - t0)) + ' Seconds') print('Total ticker tried: ' + str(n)) print('Sucessfully loaded: ' + str(n - k)) print('Failed loaded: ' + str(k)) print(*failed, sep=',') return (result) #ts=get_ticker_update(['C','A'],100)
def find_price(SYB): now = datetime.datetime.now() time_now = [now.year, now.month, now.day] data = Fetcher(SYB, time_now) data_get = data.getHistorical() #print (data_get) open = data_get['Open'] open_price = float(open[0]) #print (SYB, open_price) return open_price
def update_graph(selected_dropdown_value): global stockpricedf # Needed to modify global copy of stockpricedf ticker_fb = Fetcher(selected_dropdown_value, [yr, mo, dy]) #ticker_fb = Fetcher('fb', [yr,mo,dy]) stockpricedf = ticker_fb.get_historical() stockpricedf['Date'] = pd.to_datetime(stockpricedf['Date']) stockpricedf['Close'] = stockpricedf['Close'] return { 'data': [{ 'x': stockpricedf.Date, 'y': stockpricedf.Close, 'line': { 'color': 'green' } }] }
def calc_avg(ticker: str, t_minus_1: datetime, back_days: int) -> Avg: start_date = t_minus_1 - timedelta(days=back_days) end_date = t_minus_1 + timedelta(days=1) d1 = [start_date.year, start_date.month, start_date.day] d2 = [end_date.year, end_date.month, end_date.day] data = Fetcher(ticker, d1, d2).get_historical() return do_calc_avg(data)
def run_Fetcher(stock_name, start, end, cycle): #print(stock_name) AAPL = Fetcher(stock_name, start, end, cycle) date_close = AAPL.getDatePrice() close = date_close.ix[:, [1]] summ = 0 ar = np.empty for i in range(len(close)): summ += close.ix[i][0] ar = np.append(ar, close.ix[i][0]) ar = ar[1:] return ar
def plot(self, *args): if args: ticker = args[0] else: ticker = self.ticker if ticker == None: raise InvalidTickerException(str(ticker)) try: data = Fetcher(ticker, [2007, 1, 1]).getDatePrice() except KeyError: raise InvalidTickerException(str(ticker)) except IndexError: raise InvalidTickerException(str(ticker)) except UnboundLocalError: raise InvalidTickerException(str(ticker)) dates = [] for date in data["Date"]: date = date.split("-") date[0], date[1] = date[1], date[0] date[1], date[2] = date[2], date[1] dates.append("/".join(date)) dates = [dt.datetime.strptime(d, '%m/%d/%Y').date() for d in dates] self.figure.clear() # reset the figure self.figure.suptitle(ticker) ax = self.figure.add_subplot(111) ax.plot(dates, data["Adj Close"]) self.canvas.draw()
def get_fetcher(): try: fetch = Fetcher(self.symbols.iloc[0]['Symbol'], self.start_yahoo, self.end_yahoo) return fetch except KeyError: get_fetcher()
def fill(): """Function that fills stock data for missing days""" stock_id_field = 'stock_id' stock_ticker = 'stock__ticker' date = 'date' now = arrow.now() data = DailyStockQuote.objects.values( stock_ticker, stock_id_field).annotate(date=Max(date)) for stock in data: last_date = arrow.get(stock[date]).replace(days=+1) last_date = get_date_array_for_fetcher(last_date) now_fetcher = get_date_array_for_fetcher(now) ticker = stock[stock_ticker] stock_id = stock[stock_id_field] fetcher = Fetcher(ticker, last_date, now_fetcher) history = fetcher.getHistorical() save_stock_quote_from_fetcher(history, stock_id)
def scrapeASX_splits(xCode, xYear): xDateStart = [xYear, 1, 1] xDateEnd = [xYear, 12, 31] try: ASX_splits = Fetcher(xCode + ".AX", xDateStart, xDateEnd).getSplits() ASX_splits['Code'] = xCode except Exception as e: ASX_splits = pd.DataFrame() return (ASX_splits)
def run_Fetcher(stock_name, start, end, cycle): AAPL = Fetcher(stock_name, start , end , cycle) date_close = AAPL.getHistorical() #print(date_close) close = date_close.ix[:,[1,2,3]] #print(close) ar = np.empty for i in range(len(close)): typical = (close.ix[i][0] + close.ix[i][1] + close.ix[i][2]) / 3.0 ar = np.append(ar, typical) ar = ar[1:] return ar
def get_daily_price_returns(symbol, from_date, to_date, get_returns=True): # from and to date format in list [yyyy,mm,dd] df = Fetcher(symbol, from_date, to_date).getHistorical() if get_returns: df["returns"] = get_daily_returns(symbol, from_date, to_date) df = df.iloc[1:, :] return df
def post(): K.clear_session() namasaham = request.form['namasaham'] model = joblib.load('model' + namasaham) #split tanggal,bulan,tahun date = request.form['tanggal'] date = datetime.datetime.strptime(date, '%Y-%m-%d') tanggal = date.strftime('%d') if tanggal[0] == str(0): tanggal1 = int(tanggal[1]) else: tanggal1 = int(tanggal) print(tanggal1) bulan = date.strftime('%m') if bulan[0] == str(0): bulan1 = int(bulan[1]) else: bulan1 = int(bulan) tahun = date.strftime('%Y') tahun1 = int(tahun) tanggal_predict = str(tahun + '-' + bulan + '-' + tanggal) #import data stock price real_saham = Fetcher(namasaham + ".JK", [2018, 1, 1], [tahun1, bulan1, tanggal1], interval="1d") real_saham = real_saham.getHistorical() real_saham = real_saham.iloc[:, 0:2] real_saham = real_saham.dropna() #set index and drop data in predict_date real_saham = real_saham.set_index("Date") if tanggal_predict in real_saham.index.values: real_saham = real_saham.drop(tanggal_predict, axis=0) real_saham = real_saham.tail(10) #transform from sklearn.preprocessing import MinMaxScaler sc = MinMaxScaler() saham = sc.fit_transform(real_saham) window = 3 predictX = dataset_predict.createDataset(saham, window) predictX = predictX.reshape(len(saham) - window + 1, window, 1) predictY = model.predict(predictX) #denormalisasi predictY = sc.inverse_transform(predictY) predict_next_day = "Rp. {}".format(int(predictY[-1][0])) price_previous_day = "Rp. {}".format(int(real_saham['Open'][-1])) previous_date = real_saham.index[-1] return render_template('predict.html', predict=predict_next_day, price_previous=price_previous_day, tanggal_predict=tanggal_predict, previous_date=previous_date, namasaham=namasaham)
def scrapeASX_price(xCode, xYear): xDateStart = [xYear, 1, 1] xDateEnd = [xYear, 12, 31] try: ASX_price = Fetcher(xCode + ".AX", xDateStart, xDateEnd).get_historical() ASX_price['Code'] = xCode except Exception as e: ASX_price = pd.DataFrame() return (ASX_price)
def loadCsv(symbols: list, output_dir: str, start_date=[2015, 5, 24], end_date=[2020, 10, 9]): import time for symbol in symbols: print("generate csv to symbol ", symbol, "...") time.sleep(2) data = Fetcher(symbol, start_date, end_date) try: with open(output_dir + '/' + symbol + '.csv', 'w') as csvfile: dataf = pd.DataFrame(data.getHistorical()) if dataf.empty: raise Exception('dataframe is empty') dataf.to_csv(path_or_buf=csvfile, mode='w') print("success generate csv to symbol ", symbol) except: print("FAILD!!!! generate csv to symbol ", symbol)
def yahoo_api(tkr, reset): poor_data = False if not reset: return pd.read_csv('stocks.csv'), poor_data data = Fetcher(tkr, [2000, 1, 1]) df = data.getHistorical() df = df.drop(['Close'], axis=1) df = df.rename(columns={ 'High': 'Max', 'Low': 'Min', 'Adj Close': 'Close', 'Volume': 'Vol' }) df.iloc[:] = df.iloc[::-1].values df.iloc[:, 0] = list(map(lambda x: x.replace('-', '.'), df.iloc[:, 0])) df['Date'] = df['Date'].map(lambda x: '.'.join(x.split('.')[::-1])) df['last_close'] = df['Close'].shift(-1) df['Var'] = list( map(lambda x, y: (x / y) - 1, df['Close'], df['last_close'])) df[['Open', 'Max', 'Min', 'Close', 'Vol']] = df[['Open', 'Max', 'Min', 'Close', 'Vol']].applymap(lambda x: x if x != 0 else None) if df.isnull().sum().max() / len(df.index) > 0.3 or len(df.index) < 50: poor_data = True df[['Open', 'Max', 'Min', 'Close', 'Vol']] = df[['Open', 'Max', 'Min', 'Close', 'Vol']].fillna(method='bfill') df['last_close'] = df['Close'].shift(-1) df['Var'] = list( map(lambda x, y: (x / y) - 1, df['Close'], df['last_close'])) df = df[['Date', 'Open', 'Close', 'Min', 'Max', 'Vol', 'Var']] #if df[['Vol']].sum(axis=0)[0] == 0: df = df.drop(columns=['Vol']) df = df.dropna() df.to_csv('stocks.csv') return df, poor_data
def data_fetcher(etflist, start, end): """ Takes in a list of etf symbols returns pandas dataframe of price details and dividends """ price_df = pd.DataFrame() errors = [] for e in etflist: try: data = Fetcher(e, start, end) pricedata = data.getHistorical() if 'Date' in data.getHistorical().columns.tolist(): pricedata['ticker'] = e price_df = price_df.append(pricedata) except KeyError: errors.append(e) else: pass return price_df, errors