def get_equities(): nasdaq100 = pd.read_hdf(data_path / 'data.h5', '1min_trades') equities_con = sqlite3.connect(equities_db_path.as_posix()) equities = read_sqlite('equity_symbol_mappings', equities_con) all_tickers = nasdaq100.index.get_level_values('ticker').unique() tickers_with_meta = np.sort( all_tickers.intersection(pd.Index(equities.symbol))) nasdaq_info = (get_nasdaq_symbols().reset_index().rename( columns=lambda x: x.lower().replace(' ', '_')). loc[:, ['symbol', 'security_name']].rename( columns={'security_name': 'asset_name'})) nasdaq_tickers = pd.DataFrame({ 'symbol': tickers_with_meta }).merge(nasdaq_info, how='left') nasdaq_sids = (equities.loc[equities.symbol.isin(nasdaq_tickers.symbol), ['symbol', 'sid']]) nasdaq_tickers = (nasdaq_tickers.merge( nasdaq_sids, how='left').reset_index().rename(columns={ 'sid': 'quandl_sid', 'index': 'sid' })) nasdaq_tickers.to_hdf('algoseek.h5', 'equities')
def get_nasdaq_tickers(): tickers = nasdaq.get_nasdaq_symbols() with open('all_stock_tickers.pickle', 'wb') as f: pickle.dump(tickers.index.tolist(), f) return tickers.index.tolist()
def get_all_symbols(to_list=True): '''Get all NASDAQ sybols''' from pandas_datareader.nasdaq_trader import get_nasdaq_symbols symbols = get_nasdaq_symbols() symbols = symbols.reset_index() symbols = symbols['Symbol'] if to_list: return symbols.tolist() return symbols
def get_symbols(): fpath = os.path.join(get_data_path('stock_data'), 'symbols.qdom') if os.path.exists(fpath): with open(fpath, 'rb') as f: symbols = pickle.load(f) else: symbols = get_nasdaq_symbols() symbols.reset_index(inplace=True) with open(fpath, 'wb') as f: pickle.dump(symbols, f, pickle.HIGHEST_PROTOCOL) return symbols
def stock_listings(self): ''' returns the stock listings variable ''' if self._stock_listings == None: listings = get_nasdaq_symbols() self._stock_listings = OrderedDict() for i in range (len(listings)): self._stock_listings[listings.iloc[i]['NASDAQ Symbol']] = listings.iloc[i]['Security Name'] return self._stock_listings
def getStockQuotes(): quotes = nt.get_nasdaq_symbols() num_processes = multiprocessing.cpu_count() - 1 chunk_size = int(quotes.shape[0] / num_processes) chunks = [ quotes.ix[quotes.index[i:i + chunk_size]] for i in range(0, quotes.shape[0], chunk_size) ] pool = multiprocessing.Pool(processes=num_processes) pool.map(func, chunks)
def use_current_data(): global sy symbols = get_nasdaq_symbols() data = {'stocks': []} with open('first_data.json') as test_data: json_results = json.load(test_data) results = [] for d in json_results: results.append(d) # print(json.dumps(json_results['Data Results'])) # NASDAQ Symbol for s in json_results['Data Results']: try: # Security Name symbol = s['symbol'][0] name = symbols.loc[symbol]['Security Name'] data['stocks'].append({ 'name': name, 'symbol': symbol, 'info': s['data'] }) print('Getting data for ' + str(symbol) + " - " + str(symbols.loc[symbol]['Security Name'])) except: continue # print(json.dumps(data)) for stock in data['stocks']: values = [] dates = [] for val in stock['info']: values.append(val['value']) for dat in stock['info']: dates.append(dat['date']) description = stats.describe(values) stock['stats'] = [] stock['stats'].append({ 'nobs': description[0], 'min': description[1][0], 'max': description[1][1], 'mean': description[2], 'variance': description[3], 'skewness': description[4], 'kurtosis': description[5] }) print() print(stock['name']) print(description) print() with open('stock_data.json', 'w') as stock_data: json.dump(data, stock_data, indent=4, sort_keys=True, default=str)
def init(cls, **kwargs) : target = 'filename' filename = kwargs.get(target, None) target = 'retry_count' retry_count = kwargs.get(target,3) target = 'timeout' timeout = kwargs.get(target,30) results = get_nasdaq_symbols(retry_count, timeout) if filename is not None : results.to_csv(filename) ret = cls(results) return ret
def __init__(self, strategy="momentum returns", start=datetime.now() - relativedelta(days=2)): self.strategies = { "momen_returns": self.momen_returns, "momen_low_high": self.momen_low_high } self.markets = get_nasdaq_symbols( ).index #holds all nasdaq traded stocks self.portfolio_strategy = self.check_strategy(strategy) if self.portfolio_strategy == False: raise ValueError("No such strategy exists") self.portfolio = None self.start = start self.purchase_dates = [] #used for testing strategies
def search_stock_symbols(stock): fn = 'symbols.csv' if not os.path.exists(fn): symbols = get_nasdaq_symbols() symbols.to_csv(fn, index='Symbol') else: symbols = pd.read_csv(fn, index_col='Symbol') if stock is None: return symbols stock = stock.upper() hard_search = symbols[symbols['NASDAQ Symbol'] == stock] if len(hard_search) == 1: return 1, symbols[symbols['NASDAQ Symbol'] == stock]['Security Name'][stock] else: found = symbols[symbols['NASDAQ Symbol'].str.contains(stock)] if found.empty: return 0, None else: return len(found), found
def set_json_from_datareader(self, api_key: str): df: DataFrame if self.source == "yfinance": yf.pdr_override() df = pdr.get_data_yahoo( self.symbol, start=self.start.strftime("%Y-%m-%d"), end=self.end.strftime("%Y-%m-%d"), ) elif self.source == "wb": df = wb.download(indicator=self.symbol, start=self.start, end=self.end) elif self.source == "nasdaq": df = get_nasdaq_symbols() # type:ignore elif self.source == "famafrench": df = web.DataReader( self.symbol, self.source, start=self.start, end=self.end, api_key=api_key, )[0] elif self.source == "tiingo": df = web.DataReader( self.symbol, self.source, start=self.start, end=self.end, api_key=api_key, ).reset_index(level="symbol", drop=True) else: df = web.DataReader( self.symbol, self.source, start=self.start, end=self.end, api_key=api_key, ) # print(df) self.price_json = df.to_json(orient="split") # print(self.price_graph) return None
def get_symbols( remove_invalid=True, remove_not_fetched=True, not_fetched_list='drive/My Drive/probabilistic_model/yahoo_not_fetched_sec_code.csv' ): # 使用pandas_datareader下载股票列表 try: symbols = get_nasdaq_symbols() symbols = symbols.loc[symbols['Test Issue'] == False, ] # 直接从纳斯达克网站下载股票列表 except Exception as e: symbols = pd.read_table( 'ftp://ftp.nasdaqtrader.com/symboldirectory/nasdaqtraded.txt', sep='|', index_col='Symbol').drop(np.NaN) symbols = symbols.loc[symbols['Test Issue'] == 'N', ] sec_list = symbols.index.tolist() # 删除无效代码 if remove_invalid: original_len = len(sec_list) sec_list = [x for x in sec_list if '$' not in x] sec_list = [x for x in sec_list if '.' not in x] # 删除yahoo无法匹配的代码 if remove_not_fetched: original_len = len(sec_list) yahoo_not_fetched_list = [] try: yahoo_not_fetched_list = pd.read_csv( not_fetched_list).sec_code.tolist() except Exception as e: print(e) sec_list = [x for x in sec_list if x not in yahoo_not_fetched_list] return symbols.loc[sec_list, ]
def DataReader(name, data_source=None, start=None, end=None, retry_count=3, pause=0.001, session=None, access_key=None): """ Imports data from a number of online sources. Currently supports Yahoo! Finance, Google Finance, St. Louis FED (FRED), Kenneth French's data library, and the SEC's EDGAR Index. Parameters ---------- name : str or list of strs the name of the dataset. Some data sources (yahoo, google, fred) will accept a list of names. data_source: {str, None} the data source ("yahoo", "yahoo-actions", "yahoo-dividends", "google", "fred", "ff", or "edgar-index") start : {datetime, None} left boundary for range (defaults to 1/1/2010) end : {datetime, None} right boundary for range (defaults to today) retry_count : {int, 3} Number of times to retry query request. pause : {numeric, 0.001} Time, in seconds, to pause between consecutive queries of chunks. If single value given for symbol, represents the pause between retries. session : Session, default None requests.sessions.Session instance to be used access_key : (str, None) Optional parameter to specify an API key for certain data sources. Examples ---------- # Data from Yahoo! Finance gs = DataReader("GS", "yahoo") # Corporate Actions (Dividend and Split Data) # with ex-dates from Yahoo! Finance gs = DataReader("GS", "yahoo-actions") # Data from Google Finance aapl = DataReader("AAPL", "google") # Price and volume data from IEX tops = DataReader(["GS", "AAPL"], "iex-tops") # Top of book executions from IEX gs = DataReader("GS", "iex-last") # Real-time depth of book data from IEX gs = DataReader("GS", "iex-book") # Data from FRED vix = DataReader("VIXCLS", "fred") # Data from Fama/French ff = DataReader("F-F_Research_Data_Factors", "famafrench") ff = DataReader("F-F_Research_Data_Factors_weekly", "famafrench") ff = DataReader("6_Portfolios_2x3", "famafrench") ff = DataReader("F-F_ST_Reversal_Factor", "famafrench") # Data from EDGAR index ed = DataReader("full", "edgar-index") ed2 = DataReader("daily", "edgar-index") """ if data_source == "yahoo": return YahooDailyReader(symbols=name, start=start, end=end, adjust_price=False, chunksize=25, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "yahoo-actions": return YahooActionReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "yahoo-dividends": return YahooDivReader(symbols=name, start=start, end=end, adjust_price=False, chunksize=25, retry_count=retry_count, pause=pause, session=session, interval='d').read() elif data_source == "google": return GoogleDailyReader(symbols=name, start=start, end=end, chunksize=25, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "iex-tops": return IEXTops(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "iex-last": return IEXLasts(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "iex-book": return IEXDeep(symbols=name, service="book", start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "enigma": return EnigmaReader(datapath=name, api_key=access_key).read() elif data_source == "fred": return FredReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "famafrench": return FamaFrenchReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "oecd": return OECDReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "eurostat": return EurostatReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "edgar-index": return EdgarIndexReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "oanda": return get_oanda_currency_historical_rates(start, end, quote_currency="USD", base_currency=name, reversed=True, session=session) elif data_source == 'nasdaq': if name != 'symbols': raise ValueError("Only the string 'symbols' is supported for " "Nasdaq, not %r" % (name, )) return get_nasdaq_symbols(retry_count=retry_count, pause=pause) else: msg = "data_source=%r is not implemented" % data_source raise NotImplementedError(msg)
def DataReader(name, data_source=None, start=None, end=None, retry_count=3, pause=0.001, session=None, access_key=None): """ Imports data from a number of online sources. Currently supports Google Finance, St. Louis FED (FRED), and Kenneth French's data library, among others. Parameters ---------- name : str or list of strs the name of the dataset. Some data sources (google, fred) will accept a list of names. data_source: {str, None} the data source ("google", "fred", "ff") start : {datetime, None} left boundary for range (defaults to 1/1/2010) end : {datetime, None} right boundary for range (defaults to today) retry_count : {int, 3} Number of times to retry query request. pause : {numeric, 0.001} Time, in seconds, to pause between consecutive queries of chunks. If single value given for symbol, represents the pause between retries. session : Session, default None requests.sessions.Session instance to be used access_key : (str, None) Optional parameter to specify an API key for certain data sources. Examples ---------- # Data from Google Finance aapl = DataReader("AAPL", "google") # Price and volume data from IEX tops = DataReader(["GS", "AAPL"], "iex-tops") # Top of book executions from IEX gs = DataReader("GS", "iex-last") # Real-time depth of book data from IEX gs = DataReader("GS", "iex-book") # Data from FRED vix = DataReader("VIXCLS", "fred") # Data from Fama/French ff = DataReader("F-F_Research_Data_Factors", "famafrench") ff = DataReader("F-F_Research_Data_Factors_weekly", "famafrench") ff = DataReader("6_Portfolios_2x3", "famafrench") ff = DataReader("F-F_ST_Reversal_Factor", "famafrench") """ if data_source == "yahoo": raise ImmediateDeprecationError(DEP_ERROR_MSG.format('Yahoo Daily')) return YahooDailyReader(symbols=name, start=start, end=end, adjust_price=False, chunksize=25, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "yahoo-actions": raise ImmediateDeprecationError(DEP_ERROR_MSG.format('Yahoo Actions')) return YahooActionReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "yahoo-dividends": comp = 'Yahoo Dividends' raise ImmediateDeprecationError(DEP_ERROR_MSG.format(comp)) return YahooDivReader(symbols=name, start=start, end=end, adjust_price=False, chunksize=25, retry_count=retry_count, pause=pause, session=session, interval='d').read() elif data_source == "google": return GoogleDailyReader(symbols=name, start=start, end=end, chunksize=25, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "iex": return IEXDailyReader(symbols=name, start=start, end=end, chunksize=25, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "iex-tops": return IEXTops(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "iex-last": return IEXLasts(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "bankofcanada": return BankOfCanadaReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "stooq": return StooqDailyReader(symbols=name, chunksize=25, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "iex-book": return IEXDeep(symbols=name, service="book", start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "enigma": return EnigmaReader(dataset_id=name, api_key=access_key).read() elif data_source == "fred": return FredReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "famafrench": return FamaFrenchReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "oecd": return OECDReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "eurostat": return EurostatReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "edgar-index": raise ImmediateDeprecationError(DEP_ERROR_MSG.format('EDGAR')) return EdgarIndexReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == 'nasdaq': if name != 'symbols': raise ValueError("Only the string 'symbols' is supported for " "Nasdaq, not %r" % (name, )) return get_nasdaq_symbols(retry_count=retry_count, pause=pause) elif data_source == "quandl": return QuandlReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "moex": return MoexReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == "morningstar": return MorningstarDailyReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, interval="d").read() elif data_source == 'robinhood': return RobinhoodHistoricalReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() elif data_source == 'tiingo': return TiingoDailyReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, api_key=access_key).read() else: msg = "data_source=%r is not implemented" % data_source raise NotImplementedError(msg)
def Process(self): self.Result.setText("Please wait while the Models are being Trained....") time.sleep(3) START_DATE = '2020-01-01' END_DATE = str(datetime.now().strftime('%Y-%m-%d')) user_input = self.edit_sname.text() user_input = str(user_input).upper() try: symbols = get_nasdaq_symbols() stocks = data.DataReader(user_input, 'yahoo', START_DATE, END_DATE) except RemoteDataError: self.label_error.setText("No data found, Try Again") QApplication.processEvents() stock_data = pd.DataFrame(stocks) sym_data = pd.DataFrame(symbols[['Security Name']]) if os.path.isfile('data.csv'): print('File already exists') os.remove('data.csv') file = pd.DataFrame(stock_data) file.to_csv('data.csv') else: file = pd.DataFrame(stock_data) file.to_csv('data.csv') symbols_data = sym_data.index.tolist() if user_input in symbols_data: self.label_error.setText(f"{(sym_data.loc[user_input])}") QApplication.processEvents() else: self.label_error.setText("Stock Name not found!") self.close() QApplication.processEvents() df_read = pd.read_csv('data.csv') dates = [] prices = [] #Get the all data except for the last row df = df_read.head(len(df_read)-1) df_dates = df.loc[:,'Date'] df_open = df.loc[:,'Open'] #create independant data set X as dates for date in df_dates: dates.append([int(date.split('-')[2])]) #create the independant y data set as prices for open_price in df_open: prices.append(float(open_price)) self.Result.clear() self.Result.setText("Make sure to save the chart window or close it for the final results") QApplication.processEvents() def predict_prices(dates, prices, x): #create 3 svr models svr_lin = SVR(kernel='linear', C=1e3) svr_poly = SVR(kernel='poly', C=1e3, degree=2) svr_rbf = SVR(kernel='rbf', C=1e3, gamma=0.1) #train the models on the dates and prices svr_lin.fit(dates, prices) svr_poly.fit(dates, prices) svr_rbf.fit(dates, prices) #Linear Regeression Model lin_reg = LinearRegression() #Train the model lin_reg.fit(dates, prices) #plot the models on a graph to see which is best plt.scatter(dates, prices, color='black', label='Data') plt.scatter(dates, svr_rbf.predict(dates), color='red', label='RBF') plt.scatter(dates, svr_lin.predict(dates), color='green', label='Linear') plt.scatter(dates, svr_poly.predict(dates), color='blue', label='Polynomial') plt.scatter(dates, lin_reg.predict(dates), color='orange', label='Linear Reg') plt.xlabel('Days') plt.ylabel('Price') plt.title("Support Vector Regression") plt.legend() plt.show() #return all three models return(f"RBF: {round(svr_rbf.predict(x)[0], 2)} \t\t Linear: {round(svr_lin.predict(x)[0], 2)} \nPolynomial: {round(svr_poly.predict(x)[0], 2)} \t Linear Regression {round(lin_reg.predict(x)[0], 2)}") d = float(self.edit_sday.text()) predicted = predict_prices(dates, prices, [[d]]) self.Result.clear() self.Result.setText(f"{predicted}") QApplication.processEvents()
import math import numpy as np import pandas as pd import pandas_datareader as pdr import matplotlib.pyplot as plt from datetime import datetime from pandas_datareader.nasdaq_trader import get_nasdaq_symbols from sklearn.preprocessing import MinMaxScaler from keras.models import Sequential from keras.layers import Dense, LSTM, Dropout today = datetime.today().strftime('%Y-%m-%d') symbols = get_nasdaq_symbols().index #get data from single stock df = pdr.DataReader('AAPL', data_source='yahoo', start='1998-01-01', end=today) #visualize the closing price history plt.title('Close Price History') plt.xlabel('Date', fontsize=18) plt.ylabel('Close Price USE ($)', fontsize=18) plt.plot(df['Close']) plt.figure(figsize=(20, 10)) plt.show() #Create new dataframe with only the "Close" column data = df.filter(['Close']) #Convert the df to a numpy array dataset = data.values #use 80% of the data to train use the other 20% to test
def DataReader( name, data_source=None, start=None, end=None, retry_count=3, pause=0.1, session=None, api_key=None, ): """ Imports data from a number of online sources. Currently supports Google Finance, St. Louis FED (FRED), and Kenneth French's data library, among others. Parameters ---------- name : str or list of strs the name of the dataset. Some data sources (IEX, fred) will accept a list of names. data_source: {str, None} the data source ("iex", "fred", "ff") start : {datetime, None} left boundary for range (defaults to 1/1/2010) end : {datetime, None} right boundary for range (defaults to today) retry_count : {int, 3} Number of times to retry query request. pause : {numeric, 0.001} Time, in seconds, to pause between consecutive queries of chunks. If single value given for symbol, represents the pause between retries. session : Session, default None requests.sessions.Session instance to be used api_key : (str, None) Optional parameter to specify an API key for certain data sources. Examples ---------- # Data from Google Finance aapl = DataReader("AAPL", "iex") # Price and volume data from IEX tops = DataReader(["GS", "AAPL"], "iex-tops") # Top of book executions from IEX gs = DataReader("GS", "iex-last") # Real-time depth of book data from IEX gs = DataReader("GS", "iex-book") # Data from FRED vix = DataReader("VIXCLS", "fred") # Data from Fama/French ff = DataReader("F-F_Research_Data_Factors", "famafrench") ff = DataReader("F-F_Research_Data_Factors_weekly", "famafrench") ff = DataReader("6_Portfolios_2x3", "famafrench") ff = DataReader("F-F_ST_Reversal_Factor", "famafrench") """ expected_source = [ "yahoo", "iex", "iex-tops", "iex-last", "iex-last", "bankofcanada", "stooq", "iex-book", "enigma", "fred", "famafrench", "oecd", "eurostat", "nasdaq", "quandl", "moex", "robinhood", "tiingo", "yahoo-actions", "yahoo-dividends", "av-forex", "av-daily", "av-daily-adjusted", "av-weekly", "av-weekly-adjusted", "av-monthly", "av-monthly-adjusted", "av-intraday", "econdb", ] if data_source not in expected_source: msg = "data_source=%r is not implemented" % data_source raise NotImplementedError(msg) if data_source == "yahoo": return YahooDailyReader( symbols=name, start=start, end=end, adjust_price=False, chunksize=25, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "iex": return IEXDailyReader( symbols=name, start=start, end=end, chunksize=25, api_key=api_key, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "iex-tops": return IEXTops( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "iex-last": return IEXLasts( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "bankofcanada": return BankOfCanadaReader( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "stooq": return StooqDailyReader( symbols=name, chunksize=25, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "iex-book": return IEXDeep( symbols=name, service="book", start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "enigma": return EnigmaReader(dataset_id=name, api_key=api_key).read() elif data_source == "fred": return FredReader( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "famafrench": return FamaFrenchReader( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "oecd": return OECDReader( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "eurostat": return EurostatReader( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "nasdaq": if name != "symbols": raise ValueError("Only the string 'symbols' is supported for " "Nasdaq, not %r" % (name, )) return get_nasdaq_symbols(retry_count=retry_count, pause=pause) elif data_source == "quandl": return QuandlReader( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, api_key=api_key, ).read() elif data_source == "moex": return MoexReader( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "robinhood": return RobinhoodHistoricalReader( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "tiingo": return TiingoDailyReader( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, api_key=api_key, ).read() elif data_source == "yahoo-actions": return YahooActionReader( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() elif data_source == "yahoo-dividends": return YahooDivReader( symbols=name, start=start, end=end, adjust_price=False, chunksize=25, retry_count=retry_count, pause=pause, session=session, interval="d", ).read() elif data_source == "av-forex": return AVForexReader( symbols=name, retry_count=retry_count, pause=pause, session=session, api_key=api_key, ).read() elif data_source == "av-daily": return AVTimeSeriesReader( symbols=name, function="TIME_SERIES_DAILY", start=start, end=end, retry_count=retry_count, pause=pause, session=session, api_key=api_key, ).read() elif data_source == "av-daily-adjusted": return AVTimeSeriesReader( symbols=name, function="TIME_SERIES_DAILY_ADJUSTED", start=start, end=end, retry_count=retry_count, pause=pause, session=session, api_key=api_key, ).read() elif data_source == "av-weekly": return AVTimeSeriesReader( symbols=name, function="TIME_SERIES_WEEKLY", start=start, end=end, retry_count=retry_count, pause=pause, session=session, api_key=api_key, ).read() elif data_source == "av-weekly-adjusted": return AVTimeSeriesReader( symbols=name, function="TIME_SERIES_WEEKLY_ADJUSTED", start=start, end=end, retry_count=retry_count, pause=pause, session=session, api_key=api_key, ).read() elif data_source == "av-monthly": return AVTimeSeriesReader( symbols=name, function="TIME_SERIES_MONTHLY", start=start, end=end, retry_count=retry_count, pause=pause, session=session, api_key=api_key, ).read() elif data_source == "av-monthly-adjusted": return AVTimeSeriesReader( symbols=name, function="TIME_SERIES_MONTHLY_ADJUSTED", start=start, end=end, retry_count=retry_count, pause=pause, session=session, api_key=api_key, ).read() elif data_source == "av-intraday": return AVTimeSeriesReader( symbols=name, function="TIME_SERIES_INTRADAY", start=start, end=end, retry_count=retry_count, pause=pause, session=session, api_key=api_key, ).read() elif data_source == "econdb": return EcondbReader( symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session, ).read() else: msg = "data_source=%r is not implemented" % data_source raise NotImplementedError(msg)
def fetch_symbols(cls,symbol): symbols = get_nasdaq_symbols() tmpsymbols=symbols.iloc[:,[1,4,7,9]] symbol_data =tmpsymbols.loc[symbol] return symbol_data
def get_stock(msg): ### To make sure dates are up-to-date (possible change of date) global previous_trade_today, today, yesterday today = datetime.strptime( datetime.now(pytz.timezone('US/Eastern')).strftime("%Y%m%d"), "%Y%m%d") yesterday = today - timedelta(1) previous_trade_today = last_trade_day(today) ### Date assignments end msg['Text'] = msg['Text'].upper() if my_glob.symbols == {}: # Load ticker - security name as into dictionary my_glob.symbols = get_nasdaq_symbols().ix.obj["Security Name"].to_dict( ) if msg['Text'] in [ u"換股", u"换股", u"股票", u"换股票", u"換股票" ] and (not (my_glob.params.get("params", msg['FromUserName'], "stock") is None)): # If customer wants to change stock my_glob.params.set("params", msg['FromUserName'], None, "stock") return my_glob.modes_greeting["stock"] if my_glob.params.get( "params", msg['FromUserName'], "stock" ) is None: # If no pre-selected stock, see if this text is meant for choosing one return setstock(msg['Text'], msg['FromUserName']) # Ticker okay and already chosen. Then choose date or change stock tickers = my_glob.params.get("params", msg['FromUserName'], "stock") # Customer will be choosing a new date no matter if he already has chosen a date (another query) or not (first query) return_text = "" if msg['Text'].upper() in today_names: now = datetime.now(pytz.timezone('US/Eastern')) if now.hour < 9 or now.hour == 9 and now.min.minute < 30: return_text = return_text + u"現在未開市,為您返回上一交易日結果\n" search_day = last_trade_day(yesterday) else: search_day = today elif msg['Text'].upper() in yesterday_names: search_day = yesterday elif msg['Text'].upper().replace(" ", "") in beforeyes_names: search_day = yesterday - timedelta(1) elif not ( re.search(u"[年月週周天]$", msg['Text']) is None ): ## If contains 年,月,週,周,天 at the end of the string, then test if it is a length for drawing candle stick. Cannot be a date since a date must ends in 日,號,号, or number. k_period = chinese_days(msg['Text']) if k_period is None: return wrong_date_prompt() + "\n\n" + k_prompt elif k_period == -1: return u"日期長度有誤," + k_prompt else: for ticker in tickers: send_candlestick(ticker, k_period, today, msg['FromUserName']) return u"請輸入日期(紐約時間)查詢其它日期,或其他日期長度繪畫K線圖,或輸入其它股票名或交易代碼,或輸入「換」退出股票模式。" # But he will probably want to directly change the stock ticker: if numeric or numeric+chinese -> date, (any alphabet+$ or pure chinese) == (any alphabet+$ or not contain numbers) -> stock; "3M" is okay # Placed this step here so that u"今天" would not be treated as a stock elif (not re.search('[a-zA-Z$]', msg['Text']) is None) or (re.search( '\d', msg['Text']) is None): return setstock(msg['Text'], msg['FromUserName']) ## Stock else: ## Date search_day = detect_date(msg['Text']) if search_day is None: # If cannot find date return wrong_date_prompt() # If everything is in place holi = is_holiday(search_day) # See if is a holiday if holi != "": return_text = return_text + search_day.strftime("%Y年%-m月%-d日").decode( "utf-8" ) + u"是一個" + holi + u",已為您改為上一交易日\n\n" ## SHOULD ONLY APPEAR TO BE THE FIRST IN RETURN_TEXT THOUGH search_day = last_trade_day(search_day) for ticker in tickers: try: stock = web.DataReader(ticker, "google", search_day, search_day).values[0] except: if search_day == today: return_text = return_text + u"今天尚未有" + ticker + u"的股票收盤信息\n" else: return_text = return_text + u"找不到" + ticker + " (" + my_glob.symbols[ ticker] + u") " + search_day.strftime( "%Y年%-m月%-d日").decode("utf-8") + u"的股票信息\n" continue return_text = return_text + ticker + " (" + my_glob.symbols[ ticker] + u"):\n" + search_day.strftime("%Y年%-m月%-d日").decode( "utf-8") + u"的開市價為$" + str(stock[0]) + u",最高價為$" + str( stock[1]) + u",最低價為$" + str(stock[2]) + u",收市價為$" + str( stock[3]) + u",成交量為" + format_volume( stock[4]) + u"股。\n\n" return return_text + u"\n請輸入日期(紐約時間)查詢其它日期,或時間長度(431天/貳拾壹週/叄拾柒個月/半年/兩年)繪畫K線圖,或輸入其它股票名或交易代碼,或輸入「換」退出股票模式。"
""" import pandas as pd pd.core.common.is_list_like = pd.api.types.is_list_like import pandas_datareader.data as web from pandas_datareader.nasdaq_trader import get_nasdaq_symbols import numpy as np import matplotlib.pyplot as plt from datetime import datetime from sklearn.cluster import KMeans #%% Descargar los datos inicio = datetime(2018, 3, 12) final = datetime(2019, 3, 12) #data = web.YahooDailyReader(symbols='CC', start=inicio, end=final, interval ='d').read() data = get_nasdaq_symbols() #%%Visualizar la serie de datos plt.plot(data['Close']) plt.show() #%% Descompener la serie original en subseries de lingutd nv dat = data['Close'] nv = 5 n_prices = len(dat) dat_new = np.zeros((n_prices - nv, nv)) for k in np.arange(nv): dat_new[:, k] = dat[k:(n_prices - nv) + k] #%% Normalizar los datos tmp = dat_new.transpose()
import pandas_datareader.nasdaq_trader as web df = web.get_nasdaq_symbols() print(df.columns)
def start_data_gather(): global sy symbols = get_nasdaq_symbols() print("Number of companies: "+str(len(symbols))) print('input the year you would like to start from:') start_year = input() print('input the number of threads you would like to use:') num_threads = input() try: start_year = int(start_year) num_threads = int(num_threads) except: print('Invalid input exiting program') exit(1) sy = start_year data = {'stocks': []} pool = ThreadPool(num_threads) results = list(tqdm(pool.imap(get_data, symbols['NASDAQ Symbol']), total=len(symbols['NASDAQ Symbol']))) json_results = {'Data Results': []} for tr in results: json_results['Data Results'].append(tr) print("Storing data please wait...") with open('first_data.json', 'w') as f: json.dump(json_results, f, indent=4, sort_keys=True, default=str) # NASDAQ Symbol for s in json_results['Data Results']: try: # Security Name symbol = s['symbol'][0] name = symbols.loc[symbol]['Security Name'] data['stocks'].append({ 'name': name, 'symbol': symbol, 'info': s['data'] }) print('Getting data for ' + str(symbol) + " - " + str(symbols.loc[symbol]['Security Name'])) except: continue # print(json.dumps(data)) for stock in data['stocks']: values = [] dates = [] for val in stock['info']: values.append(val['value']) for dat in stock['info']: dates.append(dat['date']) description = stats.describe(values) stock['stats'] = [] stock['stats'].append({ 'nobs': description[0], 'min': description[1][0], 'max': description[1][1], 'mean': description[2], 'variance': description[3], 'skewness': description[4], 'kurtosis': description[5] }) print() print(stock['name']) print(description) print() print("Storing data please wait...") with open('stock_data.json', 'w') as stock_data: json.dump(data, stock_data, indent=4, sort_keys=True, default=str)
import requests import json import requests_cache import time import logging import pandas as pd import urllib.request from datetime import date, timedelta, datetime from pyquery import PyQuery from pandas_datareader.nasdaq_trader import get_nasdaq_symbols from .holding import Holding from financescraper import scraper, conversions symbols = get_nasdaq_symbols() expire_after = timedelta(days=5) requests_cache.install_cache('cache_data', expire_after=expire_after) yahoo_scraper = scraper.YahooScraper() usd_converter = conversions.CurrencyConverter('USD') # Scrape name and holdings if any for a given ticker def scrape_ticker(ticker, use_yahoo): holdings = [] data = get_data(ticker, use_yahoo) # invalid ticker if data is None: return holdings if _is_etf(data):
# print(obj3+obj4) # # obj4.index.name = "states" # obj4.name = "population" # print(obj4) import pandas_datareader.data as web import datetime import pandas_datareader.nasdaq_trader as nasdaq start = datetime.datetime(2020, 1, 1) end = datetime.datetime(2020, 4, 17) # google = web.DataReader("GOOGL", "yahoo", start, end) # print(google.head()) syms = nasdaq.get_nasdaq_symbols() print(type(syms)) count = 0 print("****************************************") print("****************************************") print("****************************************") for index, row in syms.iterrows(): count += 1 if count > 500: break print(index) print('____________________________')