def fetchFromPandasDatareader(self, ticker, fname):
try:
quotes = pandas_datareader.get_data_tiingo(ticker, api_key=self.tiingo_api_key)
return quotes
except:
print('Connection with', fname, 'failed.')
return[]
def fetch_data(symbols=assets):
"""Fetches historical data for given symbols from Tiingo"""
api_key = utils.get_environment_var("TIINGO_API_KEY")
symbols = [symbol.upper() for symbol in symbols]
done = 0
failed = []
for symbol in symbols:
try:
symbol_data = pdr.get_data_tiingo(symbol, api_key=api_key)
except ConnectionError as ce:
msg = "Unable to connect to api.tiingo.com when fetching symbol {}".format(
symbol)
logger.error(msg, exc_info=True)
raise ce
except TypeError:
# pandas_datareader raises TypeError when fetching invalid symbol
msg = "Attempted to fetch invalid symbol {}".format(symbol)
logger.error(msg, exc_info=True)
except Exception:
msg = "Error fetching symbol {}".format(symbol)
logger.error(msg, exc_info=True)
else:
_save_data(symbol, symbol_data.reset_index())
done += 1
retry_failure(failed, done)
send_report(done, failed, __name__)
def predict(self, days = 1, lazy=False):
model = self.__train()
df = None
if lazy:
df = pd.read_csv(f'/home/{getpass.getuser()}/Desktop/FF-Backend/backend/webapp/templates/webapp/ml_data/datasets/{self.__ticker}.csv')
else:
df = pdr.get_data_tiingo(self.__ticker, api_key=key)
df_price = df.reset_index()['open']
df_price = self.__scale.transform(np.array(df_price).reshape(-1,1))
time_step = self.__time_step
l = len(df_price) - time_step # 60 => time_step
op_X = []
op = []
op_X.append(df_price[l:])
op_X = np.array(op_X).reshape(1,-1,1)
op_predict = model.predict(op_X)
op_Y = self.__scale.inverse_transform(op_predict)
op.append(op_Y)
for day in range(days-1):
op_X = np.append(op_X, self.__scale.transform(np.array(op_Y).reshape(-1,1))) # append newly predicted price
op_X = op_X.reshape(1,-1, 1)
op_X = np.delete(op_X, [0]).reshape(1,-1,1) # remove zeroth element
op_predict = model.predict(op_X)
op_Y = self.__scale.inverse_transform(op_predict)
op.append(op_Y)
print(op)
return op[-1][0]
def scrape(self, args):
symbol = args[0]
last_date = args[1]
if not self.limit_reached:
if last_date is None or last_date < self.last_business_date:
try:
t0 = datetime.now()
if last_date is None:
min_date = self.min_date
else:
min_date = last_date + BDay(1)
data = pdr.get_data_tiingo(symbol,
min_date,
self.last_business_date,
api_key=self.api_key)
# adjClose adjHigh adjLow adjOpen adjVolume close divCash high low open splitFactor volume
cols = [
'symbol', 'date', 'open', 'close', 'adjClose',
'divCash', 'volume', 'splitFactor'
]
data = data.reset_index()[cols]
data['date'] = data['date'].dt.date
data.rename(columns={'splitFactor': 'split'}, inplace=True)
data.columns = [col.lower() for col in data.columns]
# Upload data to db
self.sql_manager.upload_df('prices', data)
t1 = datetime.now()
print('Download successful for {0} ({1:.2f} sec)'.format(
symbol, (t1 - t0).total_seconds()))
except Exception as e:
print('Download failed for {}'.format(symbol))
print(e)
def update_graph(n_clicks, stock_ticker, start_date, end_date):
start = datetime.strptime(start_date[:10], '%Y-%m-%d')
end = datetime.strptime(end_date[:10], '%Y-%m-%d')
# to make tz aware
start = pytz.utc.localize(start)
end = pytz.utc.localize(end)
traces = []
for tic in stock_ticker:
df = pdr.get_data_tiingo(
tic, api_key='3e46e9afa3a7716e0bdefb6a7369b1ff902f583e'
).reset_index().set_index(['date'])
df_date_range = df.loc[start:end]
traces.append({
'x': df_date_range.index,
'y': df_date_range['close'],
'name': tic,
})
figure = {
'data': traces,
'layout': {
'title': ', '.join(stock_ticker) + ' Closing Prices'
}
}
return figure
def main():
# Argument parser for simple settings changes
parser = argparse.ArgumentParser()
parser.add_argument('query', help='Search query', nargs='*')
parser.add_argument('-s', '--start_date', nargs=3, type=int,
help='Start date for gathering stock data')
parser.add_argument('-e', '--end_date', nargs=3, type=int,
help='End date for gathering stock data')
parser.add_argument('-c', '--config_file',
help='Path to non-default config file')
parser.add_argument('-kapi', '--kapi',
help='Use a different apu key then the default in config')
parser.add_argument('-p', '--pickle_file',
help='use a different pickle file')
args = parser.parse_args()
api_key, pickle_path = settings(args)
if args.start_date is None:
start_date = None
else:
start_date = datetime.datetime(*map(int, args.start_date))
if args.end_date is None:
end_date = None
else:
end_date = datetime.datetime(*map(int, args.end_date))
reader = get_data_tiingo(args.query, api_key=api_key, start=start_date, end=end_date)
with open(pickle_path, 'wb') as file:
pickle.dump(reader, file)
def predict():
import pandas_datareader as pdr
key = "613e326124e4bc913a40eac66f133572b2f62811"
df = pdr.get_data_tiingo('AAPL', api_key=key)
df.to_csv('AAPL.csv')
import pandas as pd
df = pd.read_csv('AAPL.csv')
df1 = df.reset_index(drop=True)['close']
import numpy as np
from sklearn.model_selection import train_test_split
train_data, test_data = train_test_split(df1,
test_size=0.35,
random_state=42)
x_input = test_data[340:].values.reshape(1, -1)
x_input.shape
temp_input = list(x_input)
temp_input = temp_input[0].tolist()
lst_output = []
n_steps = 100
i = 0
inputFromUi = 30
while (i < inputFromUi):
if (len(temp_input) > 100):
#print(temp_input)
x_input = np.array(temp_input[1:])
# print("{} day input {}".format(i,x_input))
x_input = x_input.reshape(1, -1)
x_input = x_input.reshape((1, n_steps, 1))
#print(x_input)
yhat = model.predict(x_input, verbose=0)
print("{} day output {}".format(i, yhat))
temp_input.extend(yhat[0].tolist())
temp_input = temp_input[1:]
#print(temp_input)
lst_output.extend(yhat.tolist())
i = i + 1
else:
x_input = x_input.reshape((1, n_steps, 1))
yhat = model.predict(x_input, verbose=0)
# print(yhat[0])
temp_input.extend(yhat[0].tolist())
# print(len(temp_input))
lst_output.extend(yhat.tolist())
i = i + 1
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler(feature_range=(0, 1))
predictionInversed = modelScaler.inverse_transform(lst_output)
return render_template(
'index.html',
predicted_text="1st day output [{}] and {}th day output {}".format(
lst_output[0], i, yhat))
def get_stock_change(symbol, start, end):
_apiKey = "af08d3cdca987b9b5f0101ca0a63984ce6ab03d0"
stock_data = pdata.get_data_tiingo(symbols=symbol.upper(),
start=start,
end=end,
api_key=_apiKey).as_matrix()[:, 0]
return [
symbol.upper(),
((stock_data[-1] - stock_data[-2]) / stock_data[-2]) * 100
]
def chk_tiingo():
print('entering chk_tiingo()')
df = None
try:
df = pdr.get_data_tiingo('GOOG', api_key=TIINGO_API_KEY)
except Exception as e:
print("Error found:", e)
print(
df
) #print(df.loc['Exchange Rate','USD/JPY']) #print(df.loc['Exchange Rate','USD/INR'])
def __get_data(self):
ticker = self.__ticker
path = f'{os.path.dirname(__file__)}/templates/webapp/ml_data/datasets/{ticker}.csv'
if (not os.path.isfile(path)):
print("Made an API call")
df = pdr.get_data_tiingo(ticker, api_key=key)
df.to_csv(path)
print("Done reading")
df = pd.read_csv(path)
df_price = df.reset_index()['open']
return df_price
def get_stock_data(tickers: Iterable[str],
tiingo_api_key: str) -> pd.DataFrame:
asset_data = pdr.get_data_tiingo(symbols=tickers,
api_key=tiingo_api_key).reset_index()
# format the price data in the manner that PyPortfolioOpt expects it which is where the columns
# are the tickers and their respective prices indexed by date. Use adjClose to account for
# splits, dividends, etc.
formatted_df = asset_data.pivot(index="date",
columns="symbol",
values="adjClose")
return formatted_df
def __get_data(self):
ticker = self.__ticker
path = f'/home/{getpass.getuser()}/Desktop/FF-Backend/backend/webapp/templates/webapp/ml_data/datasets/{ticker}.csv'
if(not os.path.isfile(path)):
print("Made an API call")
df = pdr.get_data_tiingo(ticker, api_key=key)
df.to_csv(path);
print("Done reading")
df = pd.read_csv(path)
df_price = df.reset_index()['open']
return df_price
def read(self):
df_orig = pdr.get_data_tiingo(self.symbols, api_key=self.api_key)
df_orig.to_csv('downloads/df_orig_tiingo.csv')
exit(0)
column_names = [
'Symbol', 'Date', 'Price', 'Open', 'High', 'Low', 'Vol'
]
df_orig.columns = column_names
df_result = df_orig[['Symbol', 'Date', 'Price']]
df_result.columns = ['symbol', 'date', 'close']
df_result.to_csv('downloads/df_converter.csv')
return df_result
return df
def collect_data(company):
df = pdr.get_data_tiingo(company, api_key=key)
df.to_csv(company + '.csv')
df = pd.read_csv(company + '.csv')
df1 = df.reset_index()['close']
# df1 = df1.to_numpy()
size = len(df1)
df1 = scaler.fit_transform(np.array(df1).reshape(-1, 1))
X_input = df1[size - 60:].reshape(1, -1, 1)
# print(X_input.shape)
# print("After Scaling")
print(X_input.shape)
print(X_input[0, :5, 0])
return X_input
def update_graph1(n_clicks, value, start_date, end_date):
start = dt.strptime(start_date[:10], '%Y-%m-%d')
end = dt.strptime(end_date[:10], '%Y-%m-%d')
traces = []
for x in value:
df = pdr.get_data_tiingo(x, start, end, api_key="60d8eaf8353902d2582307d4f9d12ac11ffce7d4")
df2 = df.reset_index()
df2['date'] = df2['date'].dt.strftime('%Y-%m-%d')
df2 = df2.set_index('date')
traces.append({'x': df2.index, 'y': df2['close'], 'name': x})
fig = {
'data': traces,
'layout': {'title': "Stock"}
}
return fig
def get_data(s):
data = None
try:
data = get_data_tiingo(s.symbol, api_key=getenv('TIINGO_API_KEY'))
except NameError:
client = c()
try:
data = client.get_dataframe(s.symbol, startDate='1980-01-01')
print(colored.green(s.symbol))
except HTTPError as err:
print(colored.red(err))
except RestClientError as err:
print(colored.red(err))
except KeyError as err:
print(colored.red(err))
return data
def create_from_tiingo(cls, instrument, *args, **kwargs):
""" Загрузка данных через pandas_datareader is https://www.tiingo.com
Преимущество данного источника, в частности, в том, что он предоставляет
все 4 скорректированные (adjusted) цены (а не только цену закрытия)
"""
df = pdr.get_data_tiingo(instrument,
api_key=os.environ.get('TIINGO_API_KEY'),
*args,
**kwargs)
return cls.create_(
instrument,
df.index.get_level_values('date').values, df, **{
'o': 'adjOpen',
'h': 'adjHigh',
'l': 'adjLow',
'c': 'adjClose',
'v': 'adjVolume'
})
def get_tiingo(symbol='pnqi'):
os.environ['TIINGO_API_KEY'] = '4f0ef79d635f50b78281480f44ff160d4ef68064'
TIINGO_API_KEY = os.getenv('TIINGO_API_KEY')
# df = pdr.get_data_tiingo(symbol, api_key=os.getenv('TIINGO_API_KEY'), start = '1/2/2004', end = '9/23/2020')
# pnqi, ixn, vgt
df = pdr.get_data_tiingo('ixn',
api_key=os.getenv('TIINGO_API_KEY'),
start='1/2/2004',
end='9/23/2020')
print(df.columns)
df.reset_index(drop=False, inplace=True)
df.columns = [
'symbol', 'Date', 'close', 'high', 'low', 'open', 'volume', 'adjClose',
'adjHigh', 'adjLow', 'adjOpen', 'adjVolume', 'divCash', 'splitFactor'
]
df.set_index('Date', inplace=True)
print('after:', df.columns)
print(df.head(5))
return df
def getCryptoData(coinTicker):
# setting default crypto-fiat currency pair to EUR
againstCurrency = "EUR"
# specifying an early start date for the selected coin so the API will retrieve the earliest start
# date available for historic prices if it doesn't match the specified start date
start = dt.datetime(2015, 1, 1)
# end date is the current date
end = dt.datetime.now()
data = web.get_data_tiingo("{}{}".format(coinTicker, againstCurrency),
start,
end,
api_key=TIINGOKEY)
# dropping any N/A values in case
data = data.dropna()
# removing multi-index via resetting the index to just the date column
data = data.reset_index()
data = data.set_index("date")
return data
def retry_failure(failed, done):
api_key = utils.get_environment_var("TIINGO_API_KEY")
for symbol in failed:
try:
symbol_data = pdr.get_data_tiingo(symbol, api_key=api_key)
except ConnectionError as ce:
msg = "Unable to connect to api.tiingo.com when fetching symbol {}".format(
symbol)
logger.error(msg, exc_info=True)
raise ce
except TypeError:
# pandas_datareader raises TypeError when fetching invalid symbol
failed.append(symbol)
msg = "Attempted to fetch invalid symbol {}".format(symbol)
logger.error(msg, exc_info=True)
except Exception:
msg = "Error fetching symbol {}".format(symbol)
logger.error(msg, exc_info=True)
else:
_save_data(symbol, symbol_data)
done += 1
failed.remove(symbol)
def process(symbol='SPY', source=None, mock=None, **kwargs):
"""Get a stock quote history.
::parents:: mock
::params:: symbol, source
::alerts:: source: AV, source: Tiingo, ex-dividend, split, reverse split
"""
## Get the data from whichever source
if mock is not None:
source = 'mock'
elif not source:
source = 'Tiingo'
if source == 'AV':
data = pdr.get_data_alphavantage(symbol,
api_key=Config.APIKEY_AV,
start='1900')
data.index = pd.to_datetime(data.index)
elif source == 'Tiingo':
data = pdr.get_data_tiingo(symbol,
api_key=Config.APIKEY_TIINGO,
start='1900')
data = data.droplevel(
'symbol') # Multiple stock symbols are possible
data.index = data.index.date
elif source == 'mock':
data = mock
## Create alerts
alerts = [f'source: {source}']
div = data.loc[data.index.max(), 'divCash']
if div > 0:
alerts.append('ex-dividend')
splitf = data.loc[data.index.max(), 'splitFactor']
if splitf > 1:
alerts.append('split')
elif splitf < 1:
alerts.append('reverse split')
return (data, alerts)
def get_tingo_weekly(symbol):
try:
df = pdr.get_data_tiingo(symbol, api_key=TINGO_API_KEY)
except JSONDecodeError:
df = None
return df
def alpha6():
with open("config/key.txt", "r") as file:
key = file.readline()
df = pdr.get_data_tiingo(
'AAPL', api_key='ae6f21b55e3e69fc7ec16d0fcb64bce08a3eba3f')
df.to_csv('data/AAPL.csv')
df = pd.read_csv('data/AAPL.csv')
df1 = df.reset_index()['close']
#plt.plot(df1)
# plt.show()
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler(feature_range=(0, 1))
df1 = scaler.fit_transform(np.array(df1).reshape(-1, 1))
training_size = int(len(df1) * 0.65)
test_size = len(df1) - training_size
train_data, test_data = df1[0:training_size, :], df1[
training_size:len(df1), :1]
def create_dataset(dataset, time_step=1):
dataX, dataY = [], []
for i in range(len(dataset) - time_step - 1):
a = dataset[i:(i + time_step), 0] ###i=0, 0,1,2,3-----99 100
dataX.append(a)
dataY.append(dataset[i + time_step, 0])
return np.array(dataX), np.array(dataY)
time_step = 100
X_train, y_train = create_dataset(train_data, time_step)
X_test, ytest = create_dataset(test_data, time_step)
X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], 1)
X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], 1)
model = Sequential()
model.add(LSTM(50, return_sequences=True, input_shape=(100, 1)))
model.add(LSTM(50, return_sequences=True))
model.add(LSTM(50))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(X_train,
y_train,
validation_data=(X_test, ytest),
epochs=2,
batch_size=64,
verbose=1)
train_predict = model.predict(X_train)
test_predict = model.predict(X_test)
train_predict = scaler.inverse_transform(train_predict)
test_predict = scaler.inverse_transform(test_predict)
print(math.sqrt(mean_squared_error(y_train, train_predict)))
print(math.sqrt(mean_squared_error(ytest, test_predict)))
look_back = 100
trainPredictPlot = np.empty_like(df1)
trainPredictPlot[:, :] = np.nan
trainPredictPlot[look_back:len(train_predict) +
look_back, :] = train_predict
# shift test predictions for plotting
testPredictPlot = np.empty_like(df1)
testPredictPlot[:, :] = np.nan
testPredictPlot[len(train_predict) + (look_back * 2) + 1:len(df1) -
1, :] = test_predict
# plot baseline and predictions
# plt.plot(scaler.inverse_transform(df1))
# plt.plot(trainPredictPlot)
#plt.plot(testPredictPlot)
# plt.show()
len(test_data)
x_input = test_data[340:].reshape(1, -1)
temp_input = list(x_input)
temp_input = temp_input[0].tolist()
lst_output = []
n_steps = 100
i = 0
while (i < 30):
if (len(temp_input) > 100):
x_input = np.array(temp_input[1:])
#print("{} day input {}".format(i, x_input))
x_input = x_input.reshape(1, -1)
x_input = x_input.reshape((1, n_steps, 1))
#print(x_input)
yhat = model.predict(x_input, verbose=0)
#print("{} day output {}".format(i, yhat))
temp_input.extend(yhat[0].tolist())
temp_input = temp_input[1:]
# print(temp_input)
lst_output.extend(yhat.tolist())
i = i + 1
else:
x_input = x_input.reshape((1, n_steps, 1))
yhat = model.predict(x_input, verbose=0)
temp_input.extend(yhat[0].tolist())
lst_output.extend(yhat.tolist())
i = i + 1
day_new = np.arange(1, 101)
day_pred = np.arange(101, 131)
plt.plot(day_new, scaler.inverse_transform(df1[1158:]))
plt.plot(day_pred, scaler.inverse_transform(lst_output))
plt.savefig('img/Модель 6')
plt.show()
df3 = df1.tolist()
df3.extend(lst_output)
plt.plot(df3[1200:])
plt.savefig('img/Модель 6(1)')
df3 = scaler.inverse_transform(df3).tolist()
plt.plot(df3)
plt.savefig('img/Модель 6(2)')
i = i + 1
print(lst_output)
# In[13]:
new = np.arange(1, 21)
pred = np.arange(20, 40)
# In[14]:
plt.plot(new, data["Sales"][-20:])
plt.plot(pred, lst_output)
# In[15]:
df = pdr.get_data_tiingo('AAPL',
api_key='2616ae4c299d778e6ad7cc4a129f2a495322a5e8')
# In[16]:
df.to_csv('AAPL.csv')
# In[17]:
df1 = df.reset_index()['close']
# In[18]:
df1.shape
# In[19]:
#!/usr/bin/env python
# coding: utf-8
# In[4]:
import pandas_datareader as pdr
# In[5]:
df = pdr.get_data_tiingo('GOOG',
api_key='600965445e480ded65188f8485444e6643e71e2a')
# In[6]:
df.to_csv('GOOG.csv')
# In[7]:
import pandas as pd
# In[9]:
df = pd.read_csv('GOOG.csv')
# In[10]:
df
# In[11]:
df1 = df.reset_index()['close']
import os
import pandas as pd
import pandas_datareader as pdr
# query = 'WIKI/AAPL'
# data = pdr.DataReader('^DJI', 'stooq')
# # Meta Data
# https://api.tiingo.com/tiingo/daily/<ticker>
#
# # Latest Price
# https://api.tiingo.com/tiingo/daily/<ticker>/prices
#
# # Historical Prices
# https://api.tiingo.com/tiingo/daily/<ticker>/prices?startDate=2012-1-1&endDate=2016-1-1
TIINGO_API_KEY = os.getenv("TIINGO_API_KEY")
# https://api.tiingo.com/tiingo/daily/ETY/prices?token=5bbb9e7c4c5aab5c84531c9b8bc04c3b56848e21
data = pdr.get_data_tiingo("PDI", api_key=TIINGO_API_KEY)
x = data[data["divCash"] != 0]
# tiingo_tickers = pd.read_csv("https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip")
#
#
#
# data = pdr.get_data_yahoo("ETY", get_actions=True) # adjust_price=True
#
#
# data = pdr.data.DataReader("ETY", "iex")
import pandas as pd
import datetime
import pandas_datareader.data as web
from pandas import Series, DataFrame
import os
import pandas_datareader as pdr
SPY = pdr.get_data_tiingo('SPY', api_key='63e3bed28936dbb6416aeb18440310f60884fcf6')
SPY = SPY.reset_index(level=[0,1])
SPY.index = SPY['date']
SPY_adj = SPY.iloc[:,7:11]
SPY_adj.columns = ['Close','High','Low','Open']
SPY_Open_adj = SPY_adj.Open
SPY_Open_adj
SPY_Open_adj.shift(-2)
SPY_Open_adj.shift(-1)
ret1 = SPY_Open_adj.shift(-2) / SPY_Open_adj.shift(-1)
########
start = datetime.datetime(1999, 1, 1)
end = datetime.datetime(2019, 12, 31)
df = web.DataReader("^TWII", 'yahoo', start, end)
df.head()
df.info()
from feature_list import chose_list
fd_path = 'D:/Time_Series_Research/new_data/DJI/DJI_2001.csv'
df = pd.read_csv(fd_path, sep=',', header=0)
fd = "DJI_2001.csv"
# In[7]:
### Data Collection
### insert the API key generated from the TIINGO platform (keeping it hidden here)
import pandas_datareader as pdr
key=" "
# In[8]:
# TTM = Tata Motors stock data
df = pdr.get_data_tiingo('TTM', api_key=key)
# In[9]:
df.to_csv('TTM.csv')
# In[10]:
import pandas as pd
# In[12]:
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1YPE7cupT5P6H07WMjq7WKNH8LCUVXt5C
### Stock Market Prediction And Forecasting Using Stacked LSTM
"""
### Keras and Tensorflow >2.0
### Data Collection
import pandas_datareader as pdr
key = "613e326124e4bc913a40eac66f133572b2f62811"
df = pdr.get_data_tiingo('AAPL', api_key=key)
df.to_csv('AAPL.csv')
import pandas as pd
df = pd.read_csv('AAPL.csv')
df.head()
df.tail()
df1 = df.reset_index()['close']
df1
print("Entry found")
else:
print("No entry found")
if minDate == date.min:
minDate = datetime.date()
maxDate = datetime.date()
else:
maxDate = datetime.date()
print(minDate)
print(maxDate)
if minDate != date.min:
#2020.05.17, cey, Query database first
df = pdr.get_data_tiingo(symbol,
minDate,
maxDate,
api_key=os.getenv('TIINGO_API_KEY'))
for row in df.index:
symbol = row[0]
date = row[1].date()
cur = conn.execute(
'SELECT COUNT(*) FROM stocks WHERE symbol=:symbol and date=:date',
{
"symbol": symbol,
"date": date
})
if (cur.fetchone()[0] > 0):
print("Entry found")
else: