def get_equities():
nasdaq100 = pd.read_hdf(data_path / 'data.h5', '1min_trades')
equities_con = sqlite3.connect(equities_db_path.as_posix())
equities = read_sqlite('equity_symbol_mappings', equities_con)
all_tickers = nasdaq100.index.get_level_values('ticker').unique()
tickers_with_meta = np.sort(
all_tickers.intersection(pd.Index(equities.symbol)))
nasdaq_info = (get_nasdaq_symbols().reset_index().rename(
columns=lambda x: x.lower().replace(' ', '_')).
loc[:, ['symbol', 'security_name']].rename(
columns={'security_name': 'asset_name'}))
nasdaq_tickers = pd.DataFrame({
'symbol': tickers_with_meta
}).merge(nasdaq_info, how='left')
nasdaq_sids = (equities.loc[equities.symbol.isin(nasdaq_tickers.symbol),
['symbol', 'sid']])
nasdaq_tickers = (nasdaq_tickers.merge(
nasdaq_sids, how='left').reset_index().rename(columns={
'sid': 'quandl_sid',
'index': 'sid'
}))
nasdaq_tickers.to_hdf('algoseek.h5', 'equities')
def get_nasdaq_tickers():
tickers = nasdaq.get_nasdaq_symbols()
with open('all_stock_tickers.pickle', 'wb') as f:
pickle.dump(tickers.index.tolist(), f)
return tickers.index.tolist()
def get_all_symbols(to_list=True):
'''Get all NASDAQ sybols'''
from pandas_datareader.nasdaq_trader import get_nasdaq_symbols
symbols = get_nasdaq_symbols()
symbols = symbols.reset_index()
symbols = symbols['Symbol']
if to_list:
return symbols.tolist()
return symbols
def get_symbols():
fpath = os.path.join(get_data_path('stock_data'), 'symbols.qdom')
if os.path.exists(fpath):
with open(fpath, 'rb') as f:
symbols = pickle.load(f)
else:
symbols = get_nasdaq_symbols()
symbols.reset_index(inplace=True)
with open(fpath, 'wb') as f:
pickle.dump(symbols, f, pickle.HIGHEST_PROTOCOL)
return symbols
def stock_listings(self): ''' returns the stock listings variable ''' if self._stock_listings == None: listings = get_nasdaq_symbols() self._stock_listings = OrderedDict() for i in range (len(listings)): self._stock_listings[listings.iloc[i]['NASDAQ Symbol']] = listings.iloc[i]['Security Name'] return self._stock_listings
def getStockQuotes():
quotes = nt.get_nasdaq_symbols()
num_processes = multiprocessing.cpu_count() - 1
chunk_size = int(quotes.shape[0] / num_processes)
chunks = [
quotes.ix[quotes.index[i:i + chunk_size]]
for i in range(0, quotes.shape[0], chunk_size)
]
pool = multiprocessing.Pool(processes=num_processes)
pool.map(func, chunks)
def use_current_data():
global sy
symbols = get_nasdaq_symbols()
data = {'stocks': []}
with open('first_data.json') as test_data:
json_results = json.load(test_data)
results = []
for d in json_results:
results.append(d)
# print(json.dumps(json_results['Data Results']))
# NASDAQ Symbol
for s in json_results['Data Results']:
try:
# Security Name
symbol = s['symbol'][0]
name = symbols.loc[symbol]['Security Name']
data['stocks'].append({
'name': name,
'symbol': symbol,
'info': s['data']
})
print('Getting data for ' + str(symbol) + " - " + str(symbols.loc[symbol]['Security Name']))
except:
continue
# print(json.dumps(data))
for stock in data['stocks']:
values = []
dates = []
for val in stock['info']:
values.append(val['value'])
for dat in stock['info']:
dates.append(dat['date'])
description = stats.describe(values)
stock['stats'] = []
stock['stats'].append({
'nobs': description[0],
'min': description[1][0],
'max': description[1][1],
'mean': description[2],
'variance': description[3],
'skewness': description[4],
'kurtosis': description[5]
})
print()
print(stock['name'])
print(description)
print()
with open('stock_data.json', 'w') as stock_data:
json.dump(data, stock_data, indent=4, sort_keys=True, default=str)
def init(cls, **kwargs) :
target = 'filename'
filename = kwargs.get(target, None)
target = 'retry_count'
retry_count = kwargs.get(target,3)
target = 'timeout'
timeout = kwargs.get(target,30)
results = get_nasdaq_symbols(retry_count, timeout)
if filename is not None :
results.to_csv(filename)
ret = cls(results)
return ret
def __init__(self,
strategy="momentum returns",
start=datetime.now() - relativedelta(days=2)):
self.strategies = {
"momen_returns": self.momen_returns,
"momen_low_high": self.momen_low_high
}
self.markets = get_nasdaq_symbols(
).index #holds all nasdaq traded stocks
self.portfolio_strategy = self.check_strategy(strategy)
if self.portfolio_strategy == False:
raise ValueError("No such strategy exists")
self.portfolio = None
self.start = start
self.purchase_dates = [] #used for testing strategies
def search_stock_symbols(stock):
fn = 'symbols.csv'
if not os.path.exists(fn):
symbols = get_nasdaq_symbols()
symbols.to_csv(fn, index='Symbol')
else:
symbols = pd.read_csv(fn, index_col='Symbol')
if stock is None:
return symbols
stock = stock.upper()
hard_search = symbols[symbols['NASDAQ Symbol'] == stock]
if len(hard_search) == 1:
return 1, symbols[symbols['NASDAQ Symbol'] ==
stock]['Security Name'][stock]
else:
found = symbols[symbols['NASDAQ Symbol'].str.contains(stock)]
if found.empty:
return 0, None
else:
return len(found), found
def set_json_from_datareader(self, api_key: str):
df: DataFrame
if self.source == "yfinance":
yf.pdr_override()
df = pdr.get_data_yahoo(
self.symbol,
start=self.start.strftime("%Y-%m-%d"),
end=self.end.strftime("%Y-%m-%d"),
)
elif self.source == "wb":
df = wb.download(indicator=self.symbol, start=self.start, end=self.end)
elif self.source == "nasdaq":
df = get_nasdaq_symbols() # type:ignore
elif self.source == "famafrench":
df = web.DataReader(
self.symbol,
self.source,
start=self.start,
end=self.end,
api_key=api_key,
)[0]
elif self.source == "tiingo":
df = web.DataReader(
self.symbol,
self.source,
start=self.start,
end=self.end,
api_key=api_key,
).reset_index(level="symbol", drop=True)
else:
df = web.DataReader(
self.symbol,
self.source,
start=self.start,
end=self.end,
api_key=api_key,
)
# print(df)
self.price_json = df.to_json(orient="split")
# print(self.price_graph)
return None
def get_symbols(
remove_invalid=True,
remove_not_fetched=True,
not_fetched_list='drive/My Drive/probabilistic_model/yahoo_not_fetched_sec_code.csv'
):
# 使用pandas_datareader下载股票列表
try:
symbols = get_nasdaq_symbols()
symbols = symbols.loc[symbols['Test Issue'] == False, ]
# 直接从纳斯达克网站下载股票列表
except Exception as e:
symbols = pd.read_table(
'ftp://ftp.nasdaqtrader.com/symboldirectory/nasdaqtraded.txt',
sep='|',
index_col='Symbol').drop(np.NaN)
symbols = symbols.loc[symbols['Test Issue'] == 'N', ]
sec_list = symbols.index.tolist()
# 删除无效代码
if remove_invalid:
original_len = len(sec_list)
sec_list = [x for x in sec_list if '$' not in x]
sec_list = [x for x in sec_list if '.' not in x]
# 删除yahoo无法匹配的代码
if remove_not_fetched:
original_len = len(sec_list)
yahoo_not_fetched_list = []
try:
yahoo_not_fetched_list = pd.read_csv(
not_fetched_list).sec_code.tolist()
except Exception as e:
print(e)
sec_list = [x for x in sec_list if x not in yahoo_not_fetched_list]
return symbols.loc[sec_list, ]
def DataReader(name,
data_source=None,
start=None,
end=None,
retry_count=3,
pause=0.001,
session=None,
access_key=None):
"""
Imports data from a number of online sources.
Currently supports Yahoo! Finance, Google Finance, St. Louis FED (FRED),
Kenneth French's data library, and the SEC's EDGAR Index.
Parameters
----------
name : str or list of strs
the name of the dataset. Some data sources (yahoo, google, fred) will
accept a list of names.
data_source: {str, None}
the data source ("yahoo", "yahoo-actions", "yahoo-dividends",
"google", "fred", "ff", or "edgar-index")
start : {datetime, None}
left boundary for range (defaults to 1/1/2010)
end : {datetime, None}
right boundary for range (defaults to today)
retry_count : {int, 3}
Number of times to retry query request.
pause : {numeric, 0.001}
Time, in seconds, to pause between consecutive queries of chunks. If
single value given for symbol, represents the pause between retries.
session : Session, default None
requests.sessions.Session instance to be used
access_key : (str, None)
Optional parameter to specify an API key for certain data sources.
Examples
----------
# Data from Yahoo! Finance
gs = DataReader("GS", "yahoo")
# Corporate Actions (Dividend and Split Data)
# with ex-dates from Yahoo! Finance
gs = DataReader("GS", "yahoo-actions")
# Data from Google Finance
aapl = DataReader("AAPL", "google")
# Price and volume data from IEX
tops = DataReader(["GS", "AAPL"], "iex-tops")
# Top of book executions from IEX
gs = DataReader("GS", "iex-last")
# Real-time depth of book data from IEX
gs = DataReader("GS", "iex-book")
# Data from FRED
vix = DataReader("VIXCLS", "fred")
# Data from Fama/French
ff = DataReader("F-F_Research_Data_Factors", "famafrench")
ff = DataReader("F-F_Research_Data_Factors_weekly", "famafrench")
ff = DataReader("6_Portfolios_2x3", "famafrench")
ff = DataReader("F-F_ST_Reversal_Factor", "famafrench")
# Data from EDGAR index
ed = DataReader("full", "edgar-index")
ed2 = DataReader("daily", "edgar-index")
"""
if data_source == "yahoo":
return YahooDailyReader(symbols=name,
start=start,
end=end,
adjust_price=False,
chunksize=25,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "yahoo-actions":
return YahooActionReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "yahoo-dividends":
return YahooDivReader(symbols=name,
start=start,
end=end,
adjust_price=False,
chunksize=25,
retry_count=retry_count,
pause=pause,
session=session,
interval='d').read()
elif data_source == "google":
return GoogleDailyReader(symbols=name,
start=start,
end=end,
chunksize=25,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "iex-tops":
return IEXTops(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "iex-last":
return IEXLasts(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "iex-book":
return IEXDeep(symbols=name,
service="book",
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "enigma":
return EnigmaReader(datapath=name, api_key=access_key).read()
elif data_source == "fred":
return FredReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "famafrench":
return FamaFrenchReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "oecd":
return OECDReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "eurostat":
return EurostatReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "edgar-index":
return EdgarIndexReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "oanda":
return get_oanda_currency_historical_rates(start,
end,
quote_currency="USD",
base_currency=name,
reversed=True,
session=session)
elif data_source == 'nasdaq':
if name != 'symbols':
raise ValueError("Only the string 'symbols' is supported for "
"Nasdaq, not %r" % (name, ))
return get_nasdaq_symbols(retry_count=retry_count, pause=pause)
else:
msg = "data_source=%r is not implemented" % data_source
raise NotImplementedError(msg)
def DataReader(name,
data_source=None,
start=None,
end=None,
retry_count=3,
pause=0.001,
session=None,
access_key=None):
"""
Imports data from a number of online sources.
Currently supports Google Finance, St. Louis FED (FRED),
and Kenneth French's data library, among others.
Parameters
----------
name : str or list of strs
the name of the dataset. Some data sources (google, fred) will
accept a list of names.
data_source: {str, None}
the data source ("google", "fred", "ff")
start : {datetime, None}
left boundary for range (defaults to 1/1/2010)
end : {datetime, None}
right boundary for range (defaults to today)
retry_count : {int, 3}
Number of times to retry query request.
pause : {numeric, 0.001}
Time, in seconds, to pause between consecutive queries of chunks. If
single value given for symbol, represents the pause between retries.
session : Session, default None
requests.sessions.Session instance to be used
access_key : (str, None)
Optional parameter to specify an API key for certain data sources.
Examples
----------
# Data from Google Finance
aapl = DataReader("AAPL", "google")
# Price and volume data from IEX
tops = DataReader(["GS", "AAPL"], "iex-tops")
# Top of book executions from IEX
gs = DataReader("GS", "iex-last")
# Real-time depth of book data from IEX
gs = DataReader("GS", "iex-book")
# Data from FRED
vix = DataReader("VIXCLS", "fred")
# Data from Fama/French
ff = DataReader("F-F_Research_Data_Factors", "famafrench")
ff = DataReader("F-F_Research_Data_Factors_weekly", "famafrench")
ff = DataReader("6_Portfolios_2x3", "famafrench")
ff = DataReader("F-F_ST_Reversal_Factor", "famafrench")
"""
if data_source == "yahoo":
raise ImmediateDeprecationError(DEP_ERROR_MSG.format('Yahoo Daily'))
return YahooDailyReader(symbols=name,
start=start,
end=end,
adjust_price=False,
chunksize=25,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "yahoo-actions":
raise ImmediateDeprecationError(DEP_ERROR_MSG.format('Yahoo Actions'))
return YahooActionReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "yahoo-dividends":
comp = 'Yahoo Dividends'
raise ImmediateDeprecationError(DEP_ERROR_MSG.format(comp))
return YahooDivReader(symbols=name,
start=start,
end=end,
adjust_price=False,
chunksize=25,
retry_count=retry_count,
pause=pause,
session=session,
interval='d').read()
elif data_source == "google":
return GoogleDailyReader(symbols=name,
start=start,
end=end,
chunksize=25,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "iex":
return IEXDailyReader(symbols=name,
start=start,
end=end,
chunksize=25,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "iex-tops":
return IEXTops(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "iex-last":
return IEXLasts(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "bankofcanada":
return BankOfCanadaReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "stooq":
return StooqDailyReader(symbols=name,
chunksize=25,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "iex-book":
return IEXDeep(symbols=name,
service="book",
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "enigma":
return EnigmaReader(dataset_id=name, api_key=access_key).read()
elif data_source == "fred":
return FredReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "famafrench":
return FamaFrenchReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "oecd":
return OECDReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "eurostat":
return EurostatReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "edgar-index":
raise ImmediateDeprecationError(DEP_ERROR_MSG.format('EDGAR'))
return EdgarIndexReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == 'nasdaq':
if name != 'symbols':
raise ValueError("Only the string 'symbols' is supported for "
"Nasdaq, not %r" % (name, ))
return get_nasdaq_symbols(retry_count=retry_count, pause=pause)
elif data_source == "quandl":
return QuandlReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "moex":
return MoexReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == "morningstar":
return MorningstarDailyReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
interval="d").read()
elif data_source == 'robinhood':
return RobinhoodHistoricalReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session).read()
elif data_source == 'tiingo':
return TiingoDailyReader(symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
api_key=access_key).read()
else:
msg = "data_source=%r is not implemented" % data_source
raise NotImplementedError(msg)
def Process(self):
self.Result.setText("Please wait while the Models are being Trained....")
time.sleep(3)
START_DATE = '2020-01-01'
END_DATE = str(datetime.now().strftime('%Y-%m-%d'))
user_input = self.edit_sname.text()
user_input = str(user_input).upper()
try:
symbols = get_nasdaq_symbols()
stocks = data.DataReader(user_input, 'yahoo', START_DATE, END_DATE)
except RemoteDataError:
self.label_error.setText("No data found, Try Again")
QApplication.processEvents()
stock_data = pd.DataFrame(stocks)
sym_data = pd.DataFrame(symbols[['Security Name']])
if os.path.isfile('data.csv'):
print('File already exists')
os.remove('data.csv')
file = pd.DataFrame(stock_data)
file.to_csv('data.csv')
else:
file = pd.DataFrame(stock_data)
file.to_csv('data.csv')
symbols_data = sym_data.index.tolist()
if user_input in symbols_data:
self.label_error.setText(f"{(sym_data.loc[user_input])}")
QApplication.processEvents()
else:
self.label_error.setText("Stock Name not found!")
self.close()
QApplication.processEvents()
df_read = pd.read_csv('data.csv')
dates = []
prices = []
#Get the all data except for the last row
df = df_read.head(len(df_read)-1)
df_dates = df.loc[:,'Date']
df_open = df.loc[:,'Open']
#create independant data set X as dates
for date in df_dates:
dates.append([int(date.split('-')[2])])
#create the independant y data set as prices
for open_price in df_open:
prices.append(float(open_price))
self.Result.clear()
self.Result.setText("Make sure to save the chart window or close it for the final results")
QApplication.processEvents()
def predict_prices(dates, prices, x):
#create 3 svr models
svr_lin = SVR(kernel='linear', C=1e3)
svr_poly = SVR(kernel='poly', C=1e3, degree=2)
svr_rbf = SVR(kernel='rbf', C=1e3, gamma=0.1)
#train the models on the dates and prices
svr_lin.fit(dates, prices)
svr_poly.fit(dates, prices)
svr_rbf.fit(dates, prices)
#Linear Regeression Model
lin_reg = LinearRegression()
#Train the model
lin_reg.fit(dates, prices)
#plot the models on a graph to see which is best
plt.scatter(dates, prices, color='black', label='Data')
plt.scatter(dates, svr_rbf.predict(dates), color='red', label='RBF')
plt.scatter(dates, svr_lin.predict(dates), color='green', label='Linear')
plt.scatter(dates, svr_poly.predict(dates), color='blue', label='Polynomial')
plt.scatter(dates, lin_reg.predict(dates), color='orange', label='Linear Reg')
plt.xlabel('Days')
plt.ylabel('Price')
plt.title("Support Vector Regression")
plt.legend()
plt.show()
#return all three models
return(f"RBF: {round(svr_rbf.predict(x)[0], 2)} \t\t Linear: {round(svr_lin.predict(x)[0], 2)} \nPolynomial: {round(svr_poly.predict(x)[0], 2)} \t Linear Regression {round(lin_reg.predict(x)[0], 2)}")
d = float(self.edit_sday.text())
predicted = predict_prices(dates, prices, [[d]])
self.Result.clear()
self.Result.setText(f"{predicted}")
QApplication.processEvents()
import math
import numpy as np
import pandas as pd
import pandas_datareader as pdr
import matplotlib.pyplot as plt
from datetime import datetime
from pandas_datareader.nasdaq_trader import get_nasdaq_symbols
from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential
from keras.layers import Dense, LSTM, Dropout
today = datetime.today().strftime('%Y-%m-%d')
symbols = get_nasdaq_symbols().index
#get data from single stock
df = pdr.DataReader('AAPL', data_source='yahoo', start='1998-01-01', end=today)
#visualize the closing price history
plt.title('Close Price History')
plt.xlabel('Date', fontsize=18)
plt.ylabel('Close Price USE ($)', fontsize=18)
plt.plot(df['Close'])
plt.figure(figsize=(20, 10))
plt.show()
#Create new dataframe with only the "Close" column
data = df.filter(['Close'])
#Convert the df to a numpy array
dataset = data.values
#use 80% of the data to train use the other 20% to test
def DataReader(
name,
data_source=None,
start=None,
end=None,
retry_count=3,
pause=0.1,
session=None,
api_key=None,
):
"""
Imports data from a number of online sources.
Currently supports Google Finance, St. Louis FED (FRED),
and Kenneth French's data library, among others.
Parameters
----------
name : str or list of strs
the name of the dataset. Some data sources (IEX, fred) will
accept a list of names.
data_source: {str, None}
the data source ("iex", "fred", "ff")
start : {datetime, None}
left boundary for range (defaults to 1/1/2010)
end : {datetime, None}
right boundary for range (defaults to today)
retry_count : {int, 3}
Number of times to retry query request.
pause : {numeric, 0.001}
Time, in seconds, to pause between consecutive queries of chunks. If
single value given for symbol, represents the pause between retries.
session : Session, default None
requests.sessions.Session instance to be used
api_key : (str, None)
Optional parameter to specify an API key for certain data sources.
Examples
----------
# Data from Google Finance
aapl = DataReader("AAPL", "iex")
# Price and volume data from IEX
tops = DataReader(["GS", "AAPL"], "iex-tops")
# Top of book executions from IEX
gs = DataReader("GS", "iex-last")
# Real-time depth of book data from IEX
gs = DataReader("GS", "iex-book")
# Data from FRED
vix = DataReader("VIXCLS", "fred")
# Data from Fama/French
ff = DataReader("F-F_Research_Data_Factors", "famafrench")
ff = DataReader("F-F_Research_Data_Factors_weekly", "famafrench")
ff = DataReader("6_Portfolios_2x3", "famafrench")
ff = DataReader("F-F_ST_Reversal_Factor", "famafrench")
"""
expected_source = [
"yahoo",
"iex",
"iex-tops",
"iex-last",
"iex-last",
"bankofcanada",
"stooq",
"iex-book",
"enigma",
"fred",
"famafrench",
"oecd",
"eurostat",
"nasdaq",
"quandl",
"moex",
"robinhood",
"tiingo",
"yahoo-actions",
"yahoo-dividends",
"av-forex",
"av-daily",
"av-daily-adjusted",
"av-weekly",
"av-weekly-adjusted",
"av-monthly",
"av-monthly-adjusted",
"av-intraday",
"econdb",
]
if data_source not in expected_source:
msg = "data_source=%r is not implemented" % data_source
raise NotImplementedError(msg)
if data_source == "yahoo":
return YahooDailyReader(
symbols=name,
start=start,
end=end,
adjust_price=False,
chunksize=25,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "iex":
return IEXDailyReader(
symbols=name,
start=start,
end=end,
chunksize=25,
api_key=api_key,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "iex-tops":
return IEXTops(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "iex-last":
return IEXLasts(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "bankofcanada":
return BankOfCanadaReader(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "stooq":
return StooqDailyReader(
symbols=name,
chunksize=25,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "iex-book":
return IEXDeep(
symbols=name,
service="book",
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "enigma":
return EnigmaReader(dataset_id=name, api_key=api_key).read()
elif data_source == "fred":
return FredReader(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "famafrench":
return FamaFrenchReader(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "oecd":
return OECDReader(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "eurostat":
return EurostatReader(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "nasdaq":
if name != "symbols":
raise ValueError("Only the string 'symbols' is supported for "
"Nasdaq, not %r" % (name, ))
return get_nasdaq_symbols(retry_count=retry_count, pause=pause)
elif data_source == "quandl":
return QuandlReader(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
api_key=api_key,
).read()
elif data_source == "moex":
return MoexReader(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "robinhood":
return RobinhoodHistoricalReader(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "tiingo":
return TiingoDailyReader(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
api_key=api_key,
).read()
elif data_source == "yahoo-actions":
return YahooActionReader(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
elif data_source == "yahoo-dividends":
return YahooDivReader(
symbols=name,
start=start,
end=end,
adjust_price=False,
chunksize=25,
retry_count=retry_count,
pause=pause,
session=session,
interval="d",
).read()
elif data_source == "av-forex":
return AVForexReader(
symbols=name,
retry_count=retry_count,
pause=pause,
session=session,
api_key=api_key,
).read()
elif data_source == "av-daily":
return AVTimeSeriesReader(
symbols=name,
function="TIME_SERIES_DAILY",
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
api_key=api_key,
).read()
elif data_source == "av-daily-adjusted":
return AVTimeSeriesReader(
symbols=name,
function="TIME_SERIES_DAILY_ADJUSTED",
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
api_key=api_key,
).read()
elif data_source == "av-weekly":
return AVTimeSeriesReader(
symbols=name,
function="TIME_SERIES_WEEKLY",
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
api_key=api_key,
).read()
elif data_source == "av-weekly-adjusted":
return AVTimeSeriesReader(
symbols=name,
function="TIME_SERIES_WEEKLY_ADJUSTED",
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
api_key=api_key,
).read()
elif data_source == "av-monthly":
return AVTimeSeriesReader(
symbols=name,
function="TIME_SERIES_MONTHLY",
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
api_key=api_key,
).read()
elif data_source == "av-monthly-adjusted":
return AVTimeSeriesReader(
symbols=name,
function="TIME_SERIES_MONTHLY_ADJUSTED",
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
api_key=api_key,
).read()
elif data_source == "av-intraday":
return AVTimeSeriesReader(
symbols=name,
function="TIME_SERIES_INTRADAY",
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
api_key=api_key,
).read()
elif data_source == "econdb":
return EcondbReader(
symbols=name,
start=start,
end=end,
retry_count=retry_count,
pause=pause,
session=session,
).read()
else:
msg = "data_source=%r is not implemented" % data_source
raise NotImplementedError(msg)
def fetch_symbols(cls,symbol):
symbols = get_nasdaq_symbols()
tmpsymbols=symbols.iloc[:,[1,4,7,9]]
symbol_data =tmpsymbols.loc[symbol]
return symbol_data
def get_stock(msg):
### To make sure dates are up-to-date (possible change of date)
global previous_trade_today, today, yesterday
today = datetime.strptime(
datetime.now(pytz.timezone('US/Eastern')).strftime("%Y%m%d"), "%Y%m%d")
yesterday = today - timedelta(1)
previous_trade_today = last_trade_day(today)
### Date assignments end
msg['Text'] = msg['Text'].upper()
if my_glob.symbols == {}: # Load ticker - security name as into dictionary
my_glob.symbols = get_nasdaq_symbols().ix.obj["Security Name"].to_dict(
)
if msg['Text'] in [
u"換股", u"换股", u"股票", u"换股票", u"換股票"
] and (not (my_glob.params.get("params", msg['FromUserName'], "stock") is
None)): # If customer wants to change stock
my_glob.params.set("params", msg['FromUserName'], None, "stock")
return my_glob.modes_greeting["stock"]
if my_glob.params.get(
"params", msg['FromUserName'], "stock"
) is None: # If no pre-selected stock, see if this text is meant for choosing one
return setstock(msg['Text'], msg['FromUserName'])
# Ticker okay and already chosen. Then choose date or change stock
tickers = my_glob.params.get("params", msg['FromUserName'], "stock")
# Customer will be choosing a new date no matter if he already has chosen a date (another query) or not (first query)
return_text = ""
if msg['Text'].upper() in today_names:
now = datetime.now(pytz.timezone('US/Eastern'))
if now.hour < 9 or now.hour == 9 and now.min.minute < 30:
return_text = return_text + u"現在未開市，為您返回上一交易日結果\n"
search_day = last_trade_day(yesterday)
else:
search_day = today
elif msg['Text'].upper() in yesterday_names:
search_day = yesterday
elif msg['Text'].upper().replace(" ", "") in beforeyes_names:
search_day = yesterday - timedelta(1)
elif not (
re.search(u"[年月週周天]$", msg['Text']) is None
): ## If contains 年,月,週,周,天 at the end of the string, then test if it is a length for drawing candle stick. Cannot be a date since a date must ends in 日,號,号, or number.
k_period = chinese_days(msg['Text'])
if k_period is None:
return wrong_date_prompt() + "\n\n" + k_prompt
elif k_period == -1:
return u"日期長度有誤，" + k_prompt
else:
for ticker in tickers:
send_candlestick(ticker, k_period, today, msg['FromUserName'])
return u"請輸入日期（紐約時間）查詢其它日期，或其他日期長度繪畫K線圖，或輸入其它股票名或交易代碼，或輸入「換」退出股票模式。"
# But he will probably want to directly change the stock ticker: if numeric or numeric+chinese -> date, (any alphabet+$ or pure chinese) == (any alphabet+$ or not contain numbers) -> stock; "3M" is okay
# Placed this step here so that u"今天" would not be treated as a stock
elif (not re.search('[a-zA-Z$]', msg['Text']) is None) or (re.search(
'\d', msg['Text']) is None):
return setstock(msg['Text'], msg['FromUserName']) ## Stock
else: ## Date
search_day = detect_date(msg['Text'])
if search_day is None: # If cannot find date
return wrong_date_prompt()
# If everything is in place
holi = is_holiday(search_day) # See if is a holiday
if holi != "":
return_text = return_text + search_day.strftime("%Y年%-m月%-d日").decode(
"utf-8"
) + u"是一個" + holi + u"，已為您改為上一交易日\n\n" ## SHOULD ONLY APPEAR TO BE THE FIRST IN RETURN_TEXT THOUGH
search_day = last_trade_day(search_day)
for ticker in tickers:
try:
stock = web.DataReader(ticker, "google", search_day,
search_day).values[0]
except:
if search_day == today:
return_text = return_text + u"今天尚未有" + ticker + u"的股票收盤信息\n"
else:
return_text = return_text + u"找不到" + ticker + " (" + my_glob.symbols[
ticker] + u") " + search_day.strftime(
"%Y年%-m月%-d日").decode("utf-8") + u"的股票信息\n"
continue
return_text = return_text + ticker + " (" + my_glob.symbols[
ticker] + u"):\n" + search_day.strftime("%Y年%-m月%-d日").decode(
"utf-8") + u"的開市價為$" + str(stock[0]) + u"，最高價為$" + str(
stock[1]) + u"，最低價為$" + str(stock[2]) + u"，收市價為$" + str(
stock[3]) + u"，成交量為" + format_volume(
stock[4]) + u"股。\n\n"
return return_text + u"\n請輸入日期（紐約時間）查詢其它日期，或時間長度（431天/貳拾壹週/叄拾柒個月/半年/兩年）繪畫K線圖，或輸入其它股票名或交易代碼，或輸入「換」退出股票模式。"
"""
import pandas as pd
pd.core.common.is_list_like = pd.api.types.is_list_like
import pandas_datareader.data as web
from pandas_datareader.nasdaq_trader import get_nasdaq_symbols
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime
from sklearn.cluster import KMeans
#%% Descargar los datos
inicio = datetime(2018, 3, 12)
final = datetime(2019, 3, 12)
#data = web.YahooDailyReader(symbols='CC', start=inicio, end=final, interval ='d').read()
data = get_nasdaq_symbols()
#%%Visualizar la serie de datos
plt.plot(data['Close'])
plt.show()
#%% Descompener la serie original en subseries de lingutd nv
dat = data['Close']
nv = 5
n_prices = len(dat)
dat_new = np.zeros((n_prices - nv, nv))
for k in np.arange(nv):
dat_new[:, k] = dat[k:(n_prices - nv) + k]
#%% Normalizar los datos
tmp = dat_new.transpose()
import pandas_datareader.nasdaq_trader as web df = web.get_nasdaq_symbols() print(df.columns)
def start_data_gather():
global sy
symbols = get_nasdaq_symbols()
print("Number of companies: "+str(len(symbols)))
print('input the year you would like to start from:')
start_year = input()
print('input the number of threads you would like to use:')
num_threads = input()
try:
start_year = int(start_year)
num_threads = int(num_threads)
except:
print('Invalid input exiting program')
exit(1)
sy = start_year
data = {'stocks': []}
pool = ThreadPool(num_threads)
results = list(tqdm(pool.imap(get_data, symbols['NASDAQ Symbol']), total=len(symbols['NASDAQ Symbol'])))
json_results = {'Data Results': []}
for tr in results:
json_results['Data Results'].append(tr)
print("Storing data please wait...")
with open('first_data.json', 'w') as f:
json.dump(json_results, f, indent=4, sort_keys=True, default=str)
# NASDAQ Symbol
for s in json_results['Data Results']:
try:
# Security Name
symbol = s['symbol'][0]
name = symbols.loc[symbol]['Security Name']
data['stocks'].append({
'name': name,
'symbol': symbol,
'info': s['data']
})
print('Getting data for ' + str(symbol) + " - " + str(symbols.loc[symbol]['Security Name']))
except:
continue
# print(json.dumps(data))
for stock in data['stocks']:
values = []
dates = []
for val in stock['info']:
values.append(val['value'])
for dat in stock['info']:
dates.append(dat['date'])
description = stats.describe(values)
stock['stats'] = []
stock['stats'].append({
'nobs': description[0],
'min': description[1][0],
'max': description[1][1],
'mean': description[2],
'variance': description[3],
'skewness': description[4],
'kurtosis': description[5]
})
print()
print(stock['name'])
print(description)
print()
print("Storing data please wait...")
with open('stock_data.json', 'w') as stock_data:
json.dump(data, stock_data, indent=4, sort_keys=True, default=str)
import requests
import json
import requests_cache
import time
import logging
import pandas as pd
import urllib.request
from datetime import date, timedelta, datetime
from pyquery import PyQuery
from pandas_datareader.nasdaq_trader import get_nasdaq_symbols
from .holding import Holding
from financescraper import scraper, conversions
symbols = get_nasdaq_symbols()
expire_after = timedelta(days=5)
requests_cache.install_cache('cache_data', expire_after=expire_after)
yahoo_scraper = scraper.YahooScraper()
usd_converter = conversions.CurrencyConverter('USD')
# Scrape name and holdings if any for a given ticker
def scrape_ticker(ticker, use_yahoo):
holdings = []
data = get_data(ticker, use_yahoo)
# invalid ticker
if data is None:
return holdings
if _is_etf(data):
# print(obj3+obj4)
#
# obj4.index.name = "states"
# obj4.name = "population"
# print(obj4)
import pandas_datareader.data as web
import datetime
import pandas_datareader.nasdaq_trader as nasdaq
start = datetime.datetime(2020, 1, 1)
end = datetime.datetime(2020, 4, 17)
# google = web.DataReader("GOOGL", "yahoo", start, end)
# print(google.head())
syms = nasdaq.get_nasdaq_symbols()
print(type(syms))
count = 0
print("****************************************")
print("****************************************")
print("****************************************")
for index, row in syms.iterrows():
count += 1
if count > 500:
break
print(index)
print('____________________________')