class yahooFinancialsProvider(providerBase):
yahooFinancials: YahooFinancials
valor = "yyyfff"
def __init__(self, ticker):
self.yahooFinancials = YahooFinancials(ticker)
def getOHLCV(self, ticker, start_date, end_date, interval: str) -> dict:
ohlcv_data = {}
json_obj = self.yahooFinancials.get_historical_price_data(start_date,end_date,interval)
ohlcv = json_obj[ticker]['prices']
temp = pd.DataFrame(ohlcv)[["formatted_date", "open", "high", "low", "adjclose", "volume"]]
temp.set_index("formatted_date", inplace=True)
temp.dropna(inplace=True)
ohlcv_data[ticker] = temp
return ohlcv_data
def getClosePrice(self, ticker, start_date, end_date, interval: str) -> dict:
close_prices = pd.DataFrame()
json_obj = self.yahooFinancials.get_historical_price_data(start_date, end_date, interval)
ohlv = json_obj[ticker]['prices']
temp = pd.DataFrame(ohlv)[["formatted_date", "adjclose"]]
temp.set_index("formatted_date", inplace=True)
temp.dropna(inplace=True)
close_prices[ticker] = temp["adjclose"]
return close_prices
def get_new_data(database, collection):
if database == 'stock':
start_date = str(
(dt.datetime.now(timezone('EST')) - dt.timedelta(days=1)))[:10]
end_date = str(
(dt.datetime.now(timezone('EST')) - dt.timedelta(days=0)))[:10]
ticker = STOCK_INDICES[collection]
elif database == 'forex' or database == 'crypto':
start_date = str(
(dt.datetime.now(timezone('EST')) - dt.timedelta(days=1)))[:10]
end_date = start_date
if database == 'forex':
ticker = FOREX_LIST[collection]
elif database == 'crypto':
ticker = CRYPTO_LIST[collection]
yf = YahooFinancials(ticker)
try:
prices = yf.get_historical_price_data(start_date, end_date,
'daily')[ticker]['prices'][0]
except IndexError:
older_date = str(
(dt.datetime.now(timezone('EST')) - dt.timedelta(days=2)))[:10]
prices = yf.get_historical_price_data(older_date, older_date,
'daily')[ticker]['prices'][0]
result = {
'high': prices['high'],
'low': prices['low'],
'open': prices['open'],
'close': prices['close'],
'formatted_date': prices['formatted_date']
}
return result
def check_rates_yahoo(guid):
"""Check if rates are available on yahoo"""
user = User.get_by_guid_or_404(guid)
_set_task_progress(0)
start = datetime.utcnow()
end = start
# CHF -> USD
yahoo_currencies = 'CHF=X'
yahoo_financials_currencies = YahooFinancials(yahoo_currencies)
daily_currency_prices = yahoo_financials_currencies.get_historical_price_data(start.strftime('%Y-%m-%d'), end.strftime('%Y-%m-%d'), 'daily')
USD_inCHF = daily_currency_prices['CHF=X']['prices'][0]['adjclose']
existing_currencies = Currency.query.all()
# reset source flag
for c in existing_currencies:
c.source = None
log_add('INFORMATION', 'scheduler.task', 'check_rates_yahoo', 'flags reset', user)
db.session.commit()
# search an dupdate on yahoo
i = 0
n = len(existing_currencies)
for c in existing_currencies:
trace = None
try:
yahoo_currency = '{}=X'.format(c.code)
yahoo_financials_currencies = YahooFinancials(yahoo_currency)
daily_currency_prices = yahoo_financials_currencies.get_historical_price_data(start.strftime('%Y-%m-%d'), end.strftime('%Y-%m-%d'), 'daily')
trace = str(daily_currency_prices)
if len(trace)>4000:
trace = trace[0:4000]
exchange_rate = USD_inCHF/daily_currency_prices[yahoo_currency]['prices'][0]['adjclose']
c.inCHF = exchange_rate
c.source = 'yahoo'
c.db_updated_at = start
c.db_updated_by = 'check_rates_yahoo'
message = 'Currency {} updated successfully from yahoo with rate {}.'.format(c.code, c.inCHF)
log_add('INFORMATION', 'scheduler.task', 'check_rates_yahoo', message, user, trace=trace)
db.session.commit()
except:
message = 'Currency {} could not be updated from yahoo'.format(c.code)
log_add('WARNING', 'scheduler.task', 'check_rates_yahoo', message, user, trace=(trace if trace is not None else str(sys.exc_info())))
i += 1
_set_task_progress(100*i//n)
_set_task_progress(100)
def update_rates_yahoo(guid):
"""Update rates from yahoo"""
user = User.get_by_guid_or_404(guid)
_set_task_progress(0)
start = datetime.utcnow()
end = start
# CHF -> USD
yahoo_currencies = 'CHF=X'
yahoo_financials_currencies = YahooFinancials(yahoo_currencies)
daily_currency_prices = yahoo_financials_currencies.get_historical_price_data(start.strftime('%Y-%m-%d'), end.strftime('%Y-%m-%d'), 'daily')
USD_inCHF = daily_currency_prices['CHF=X']['prices'][0]['adjclose']
existing_currencies = Currency.query.filter(Currency.source=='yahoo').all()
yahoo_currencies = ['{}=X'.format(c.code) for c in existing_currencies]
n = len(existing_currencies)
# def chunks(lst, n):
# """Yield successive n-sized chunks from lst."""
# for i in range(0, len(lst), n):
# yield lst[i:i + n]
_set_task_progress(1)
trace = None
try:
yahoo_financials_currencies = YahooFinancials(yahoo_currencies)
daily_currency_prices = yahoo_financials_currencies.get_historical_price_data(start.strftime('%Y-%m-%d'), end.strftime('%Y-%m-%d'), 'daily')
trace = str(daily_currency_prices)
if len(trace)>4000:
trace = trace[0:4000]
exchange_rates = {v['currency']: USD_inCHF/daily_currency_prices[k]['prices'][0]['adjclose']
for k, v in daily_currency_prices.items()
if 'currency' in v}
for c in existing_currencies:
if c.code in exchange_rates:
c.inCHF = exchange_rates[c.code]
c.db_updated_at = start
c.db_updated_by = 'update_rates_yahoo'
message = '{} currencies updated successfully from yahoo.'.format(n)
log_add('INFORMATION', 'scheduler.task', 'get_rates_yahoo', message, user, trace=trace)
db.session.commit()
except:
message = '{} currencies could not be updated from yahoo.'.format(n)
log_add('WARNING', 'scheduler.task', 'get_rates_yahoo', message, user, trace=(trace if trace is not None else str(sys.exc_info())))
_set_task_progress(100)
def get_price(ticker):
yf = YahooFinancials(ticker)
today = str(dt.datetime.now(timezone('EST')))[:10]
yesterday = str((dt.datetime.now(timezone('EST')) - dt.timedelta(days=1)))[:10]
try:
prices = yf.get_historical_price_data(yesterday, today, 'daily')
price = prices[ticker]['prices'][0]['close']
except KeyError:
print(ticker)
today = str((dt.datetime.now(timezone('EST')) - dt.timedelta(days=1)))[:10]
yesterday = str((dt.datetime.now(timezone('EST')) - dt.timedelta(days=2)))[:10]
prices = yf.get_historical_price_data(yesterday, today, 'daily')
price = prices[ticker]['prices'][0]['close']
return price
def get_start_date(ticker, start_date, days_prior):
'''
Calculate the modified starting of the backtest to account for the warm-up period.
Parameters
------------
ticker : str
Ticker of the asset we want to use in the backtest
start_date : str
The start date we want to modify
days_prior : int
The required number of trading days prior to the first day of the backtest
Returns
-----------
new_start_date : str
The adjusted starting date for the backtest
'''
start_date_dt = datetime.strptime(start_date, '%Y-%m-%d')
prior_to_start_date_dt = start_date_dt - relativedelta(days=2 * days_prior)
prior_to_start_date = prior_to_start_date_dt.strftime('%Y-%m-%d')
yahoo_financials = YahooFinancials(ticker)
df = yahoo_financials.get_historical_price_data(prior_to_start_date,
start_date, 'daily')
df = pd.DataFrame(df[ticker]['prices'])['formatted_date']
if df.iloc[-1] == start_date:
days_prior += 1
new_start_date = df.iloc[-days_prior]
return new_start_date
def Get_MarketReturn(MR):
YF_MarketReturn = YahooFinancials(MR)
YF_MarketReturn_Hist = YF_MarketReturn.get_historical_price_data(
Start_Date, End_Date, Period)
return YF_MarketReturn_Hist
def get_ticker_data(self, start_date, today):
yahoo_financials = YahooFinancials(
self.tick) #taking rates by company ticker
js_array = yahoo_financials.get_historical_price_data(
start_date, today, 'daily') #getting all history data
price_array = js_array[self.tick]['prices']
return price_array
def add_regime_probs( self, low = True, med = True, high = True ):
start_date = "2000-01-01"
end_date = "2020-06-19"
DF = pd.DataFrame()
date_range = pd.bdate_range(start=start_date,end=end_date)
values = pd.DataFrame({ 'Date': date_range})
DF['Date']= pd.to_datetime(values['Date'])
#Extracting Data from Yahoo Finance and Adding them to Values table using date as key
raw_data = YahooFinancials('^GSPC')
raw_data = raw_data.get_historical_price_data(start_date, end_date, "daily")
df = pd.DataFrame(raw_data['^GSPC']['prices'])[['formatted_date','adjclose']]
df.columns = ['Date1','SPY']
df['Date1']= pd.to_datetime(df['Date1'])
DF = DF.merge(df,how='left',left_on='Date',right_on='Date1')
DF = DF.drop(labels='Date1', axis=1)
DF['Returns'] = DF['SPY'].pct_change( periods = 1 )
DF = DF[1:-1]
rosemary = sm.tsa.MarkovRegression(DF['Returns'], k_regimes=2, trend='nc', switching_variance=True)
rosemary_fitted = rosemary.fit()
ddf = pd.DataFrame()
ddf['Date'] = DF['Date']
if low:
ddf['regime_low'] = rosemary_fitted.smoothed_marginal_probabilities[0]
self.the_list.append( 'regime_low' )
if high:
ddf['regime_high'] = rosemary_fitted.smoothed_marginal_probabilities[1]
self.the_list.append( 'regime_high' )
self.train_data = pd.merge( left = self.train_data, right = ddf, how = 'inner', on = 'Date', suffixes = ( False, False ) )
def get_ma_map(index_symbol, ma_days):
yahoo_stock = YahooFinancials(index_symbol)
json = (yahoo_stock.get_historical_price_data('2006-01-01', '2020-10-30',
'daily'))
map_of_prices = {}
x = 0
for row in json[index_symbol]["prices"]:
price_today = row['close']
date = row['formatted_date']
map_of_prices[x] = [date, price_today]
x = x + 1
date_to_ma_with_index = calculate_ma(50, map_of_prices)
map_date_to_ma = dict()
for x in date_to_ma_with_index:
map_date_to_ma[date_to_ma_with_index[x][0]] = [
date_to_ma_with_index[x][1]
]
return map_date_to_ma
def getPrice(self, ticker, startDate, endDate, dateAscending=True):
# Get the price series for single ticker
# ----Input-----
# ticker: ticker name for the stock
# startDate: the start date of price series, the format is 'YYYY-MM-DD'
# endDate: the end date of price series, the format is 'YYYY-MM-DD'
# dateAscending: whether rank the price series by date ascending, the default value is true
# ----output----
# price series for multiple stocks in pandas DataFrame format and use date as index
startDate = dt.datetime.strptime(startDate, '%Y-%m-%d').date()
endDate = dt.datetime.strptime(endDate, '%Y-%m-%d').date()
if (startDate > endDate):
raise Exception('startDate is later than endDate')
#table = YahooFinancials(ticker)
#yahooPrices = table.get_historical_price_data(startDate.isoformat(), endDate.isoformat(), 'daily')[ticker]['prices']
if (self.approach == 'Yahoo'):
try:
table = YahooFinancials(ticker)
yahooPrices = table.get_historical_price_data(
startDate.isoformat(), endDate.isoformat(),
'daily')[ticker]['prices']
table = pd.DataFrame(yahooPrices)
table = table[['adjclose', 'formatted_date']]
table.columns = [ticker, 'date']
table = table.sort_values(by='date',
axis=0,
ascending=dateAscending)
table = table.set_index('date')
table.index = pd.to_datetime(table.index)
except:
table = pd.DataFrame(columns=[ticker, 'date'])
table = table.set_index('date')
return table
def get_yf(ticker, start_date = '2000-01-01'):
"""
Get YahooFinancials data.
ticker: Ticker name (format = 'XXXX');
start_date: Data collection start date (format = 'YYYY-MM-DD').
"""
# Get today
today_yf = datetime.today().strftime('%Y-%m-%d')
ticker = str(ticker.replace('"',''))
# Adjusting ticker name
if ticker == 'BVSP':
ticker = '^' + ticker
else:
ticker = ticker + '.SA'
# Getting raw data
yf = YahooFinancials(ticker = ticker)
data = yf.get_historical_price_data(start_date = start_date,
end_date = today_yf,
time_interval = 'daily')
# Cleaning data
data = pd.DataFrame(data[ticker]['prices'])
data['formatted_date'] = pd.to_datetime(data['formatted_date'])
data.set_index('formatted_date', inplace = True)
data = data['adjclose']
data = data.fillna(method = 'ffill')
return data
class YFPrice():
ticker = None
yf_obj = None
prices = None
data = {}
def setting(self, ticker):
self.ticker = ticker
self.yf_obj = YahooFinancials(self.ticker)
self.prices = None
self.data = {}
def get_data(self, fromdt, todt, period, srch_term):
self.prices = self.yf_obj.get_historical_price_data(
fromdt, todt, period)
for key in self.prices.keys():
# print(key, prices[key])
if isinstance(self.prices[key], dict):
for inner_key in self.prices[key].keys():
# print(inner_key, self.prices[key][inner_key])
if inner_key == 'prices':
for d in self.prices[key][inner_key]:
self.data[pd.to_datetime(d['formatted_date'],
format='%Y-%m-%d').
to_pydatetime()] = d[srch_term]
return pd.Series(self.data)
def close(self):
self.ticker = None
self.yf_obj = None
self.prices = None
self.data = {}
def _DownloadPricesCSVFromYahooFinance(self, symbols: List[Text]) -> List[Optional[Text]]:
log_symbols('Loading prices from Yahoo', symbols)
asset = YahooFinancials(symbols)
today = datetime.datetime.now().strftime("%Y-%m-%d")
historical_data = asset.get_historical_price_data(
start_date=config.DEFAULT_MIN_DATE, end_date=today,
time_interval='daily')
csv_data = []
for symbol in symbols:
if symbol not in historical_data or historical_data[symbol] is None or 'prices' not in historical_data[symbol]:
csv_data.append(None)
logging.info('No price information for symbol "{}"'.format(symbol))
continue
prices = historical_data[symbol]['prices']
lines = ['Date,Open,Close,High,Low,Volume'] # Header
for e in prices:
if e['volume'] is None:
continue
lines.append(f'{e["formatted_date"]},{e["open"]},{e["close"]},' +
f'{e["high"]},{e["low"]},{e["volume"]}')
if len(lines) <= 1:
csv_data.append(None)
logging.info(
'Price information for symbol "{}" is empty'.format(symbol))
continue
lines.append('') # This will add a last line break.
csv_data.append('\n'.join(lines))
return csv_data
def get_historical_prices(self, start, end, frequency):
"""
Collects historical prices of the equity from Yahoo Finance.
Collects all data from the date of purchase until today
"""
equity_financials = YahooFinancials(self.ticker)
equity_data = equity_financials.get_historical_price_data(
start, end, frequency)[self.ticker]['prices']
df = pd.DataFrame(
columns=['date', 'high', 'low', 'open', 'close', 'adjclose'])
for data in equity_data:
df = df.append(
{
'date': data['formatted_date'],
'high': data['high'],
'low': data['low'],
'open': data['open'],
'close': data['close'],
'adjclose': data['adjclose']
},
ignore_index=True)
df = df.set_index('date')
return df
def _get_events(self, params):
yf = YahooFinancials([params['symbol']])
hist_events = yf.get_historical_price_data(
start_date=params['start_date'].strftime("%Y-%m-%d"),
end_date=params['end_date'].strftime("%Y-%m-%d"),
time_interval=params['frequency'])[params['symbol']]['eventsData']
hist_events = hist_events
res = {}
for req_data_field in [
x for x in params['req_data_fields'] if x != 'formatted_date'
]:
hist = pd.DataFrame.from_dict(hist_events[req_data_field]).T
hist = hist[['amount']]
data_field = find_key_by_value(hist_fields_dct, req_data_field)
hist.columns = [data_field]
hist.index = pd.to_datetime(hist.index)
hist.index.name = 'DATE'
hist = hist.sort_index()
hist[data_field] = pd.to_numeric(hist[data_field])
res[data_field] = hist
return res
def get_price_dividend(ticker, d1, d2, ret_daily=False):
# (i) YahooFinancials to get price data
cn_price = ['formatted_date', 'open']
di_price = dict(zip(cn_price, ['date', 'price']))
stock = YahooFinancials(ticker)
price = stock.get_historical_price_data(d1, d2, 'daily')[ticker]
price = pd.DataFrame(price['prices'])[cn_price]
price = price.rename(columns=di_price).assign(ticker=ticker)
price.date = pd.to_datetime(price.date)
# (ii) yfinance to get dividends
stock = yf.Ticker(ticker)
cn_dividend = ['Date', 'Dividends', 'Stock Splits']
di_dividend = dict(zip(cn_dividend, ['date', 'dividend', 'splits']))
dividend = stock.history(interval='1d', start=d1, end=d2)
dividend = dividend.reset_index()[cn_dividend].rename(columns=di_dividend)
# Merge and get monthly values
cn_gg = ['ticker', 'year', 'month']
df = price.merge(dividend, 'left', 'date').drop(columns='splits')
if ret_daily:
return df
df = add_date_int(df).groupby(cn_gg).apply(
lambda x: pd.Series({
'price': x.price.mean(),
'dividend': x.dividend.sum()
})).reset_index()
df = ym2date(df)[['date'] + cn_gg + ['price', 'dividend']]
return df
def get_YahooFinancials(ticker, d1, d2, dividend=True):
di = {'formatted_date': 'date', 'open': 'price', 'amount': 'dividend'}
# (1) Get price data
cn_price = ['formatted_date', 'open']
stock = YahooFinancials(ticker)
price = stock.get_historical_price_data(d1, d2, 'daily')[ticker]
price = pd.DataFrame(price['prices'])[cn_price]
price = price.rename(columns=di).assign(ticker=ticker)
price.date = pd.to_datetime(price.date)
price = add_date_int(price) #.drop(columns=['day','date'])
price = price.query('day == 1').reset_index(None, True).drop(columns='day')
assert not (price.year + price.month / 100).duplicated().any()
assert price.price.notnull().all()
# (2) Get dividend data
if dividend:
cn_dividend = ['formatted_date', 'amount']
dividend = stock.get_daily_dividend_data(d1, d2)[ticker]
dividend = pd.DataFrame(dividend)[cn_dividend]
dividend.rename(columns=di, inplace=True)
dividend.date = pd.to_datetime(dividend.date)
dividend = add_date_int(dividend)
dividend = dividend.groupby(['year',
'month']).dividend.sum().reset_index()
price = price.merge(dividend, 'left', ['year', 'month']) # merge
return price
def download_yahoo(ticker: str, base_dir: str = default_data_dir):
try:
yf = YahooFinancials(ticker)
data = yf.get_historical_price_data(dt_to_str(start_date), dt_to_str(end_date), 'daily')
except Exception as err:
print(f'Unable to read data for {ticker}: {err}')
return pd.DataFrame({})
if data.get(ticker) is None or data[ticker].get('prices') is None or \
data[ticker].get('timeZone') is None or len(data[ticker]['prices']) == 0:
print(f'Yahoo: no data for {ticker}')
return pd.DataFrame({})
prices = {}
for rec in sorted(data[ticker]['prices'], key=lambda r: r['date']):
date = datetime.strptime(rec['formatted_date'], '%Y-%m-%d')
dic_with_prices(prices, ticker, date, rec['open'], rec['high'], rec['low'], rec['close'], rec['volume'])
if 'dividends' in data[ticker]['eventsData']:
for date, rec in sorted(data[ticker]['eventsData']['dividends'].items(), key=lambda r: r[0]):
date = datetime.strptime(date, '%Y-%m-%d')
dic_with_div(prices, ticker, date, rec['amount'])
if 'splits' in data[ticker]['eventsData']:
for date, rec in sorted(data[ticker]['eventsData']['splits'].items(), key=lambda r: r[0]):
date = datetime.strptime(date, '%Y-%m-%d')
print(f"{ticker} has split {rec['splitRatio']} for {date}")
frame = pd.DataFrame.from_dict(prices, orient='index',
columns=['Open', 'High', 'Low', 'Close', 'Volume', 'Dividend'])
save_csv(base_dir, ticker, frame, 'yahoo')
def collectData():
start_date = "1792-05-17"
end_date = datetime.date.today().__str__()
if not os.path.exists('pharmaceutical_data_new'):
os.mkdir('pharmaceutical_data_new')
print("made directory")
file = list(open("FDA_data20182019.csv"))
print(type(file))
for i in range(1, len(file)):
print(type(file))
line_arr = file[i].strip("\n").split(",")
# if line_arr[1] == "NODATE" or line_arr[2] == "NODATA"or line_arr[2] == "OUTCOME":
# continue
ticker = line_arr[0]
drug_name = line_arr[1]
drug_name = drug_name.replace("/", "-")
print(ticker)
end_date = datetime.datetime.strptime(line_arr[2],
"%m/%d/%Y").strftime("%Y-%m-%d")
if os.path.exists('pharmaceutical_data_new/' + ticker + "_" +
drug_name + "_" + end_date + '.csv'):
continue
# collect historical data for ticker
financials = YahooFinancials(ticker)
data = financials.get_historical_price_data(start_date, end_date,
"daily")
if data[ticker] is None or "prices" not in data[ticker].keys():
continue
prices = data[ticker]["prices"][-91:]
# header of csv
ordering = [
'distance', 'date', 'open', 'close', 'high', 'low', 'volume'
]
counter = -90
# rearrange values with different keys
for j in range(len(prices)):
formatted_dict = {}
for k in ordering:
if k == 'distance':
formatted_dict[k] = counter
elif k == 'date':
formatted_dict[k] = prices[j]['formatted_date']
else:
formatted_dict[k] = prices[j][k]
prices[j] = formatted_dict
counter += 1
keys = prices[0].keys()
# write prices dictionary to csv file
with open('pharmaceutical_data_new/' + ticker + "_" + drug_name + "_" +
end_date + '.csv',
'w',
newline='') as write_file:
dict_writer = csv.DictWriter(write_file, keys)
dict_writer.writeheader()
dict_writer.writerows(prices)
def get_distribution_dict(index_symbol, ma_map, start_date, end_date):
yahoo_stock = YahooFinancials(index_symbol)
json = (yahoo_stock.get_historical_price_data(start_date,
end_date,
'daily',
proxy="135.181.36.161"))
price_yesterday = sys.maxsize
volume_yesterday = sys.maxsize
ma_yesterday = -1
days_counter = 1
days_counter_of_25_days = 0
disturbtion_dict = {}
for row in json[index_symbol]["prices"]:
volume_today = row['volume']
price_today = row['close']
date = row['formatted_date']
is_disturbtion = is_disturbtion_day(volume_yesterday, volume_today,
price_today, price_yesterday)
if (is_disturbtion):
disturbtion_dict[days_counter] = [date, price_today]
days_counter_of_25_days = 0
volume_yesterday = volume_today
price_yesterday = price_today
days_counter = days_counter + 1
days_counter_of_25_days = days_counter_of_25_days + 1
print(disturbtion_dict)
return disturbtion_dict
def get_yahoo_financials(tickers, start_date, time_interval='daily'):
"""
Pulls historical pricing data for stocks, currencies, ETFs, mutual funds, U.S. Treasuries, cryptocurrencies,
commodities, and indexes.
:param tickers: list of tickers
:param start_date: ‘YYYY-MM-DD’ format and is the first day that data will be pulled for
:param time_interval: 'daily','weekly','monthly' and default is daily
:return: a dictionary with each ticker as key and stock data as value
"""
validate_start_date(start_date)
validate_tickers(tickers)
res_dict = {}
end_date = date.today().strftime("%Y-%m-%d")
if time_interval not in ['daily', 'weekly', 'monthly']:
raise ValueError("Interval should be daily, weekly or monthly.")
try:
for ticker in tickers:
financials = YahooFinancials(ticker)
json_obj = financials.get_historical_price_data(start_date, end_date, time_interval)
ohlv = json_obj[ticker]['prices']
res_dict[ticker] = {}
for row in ohlv:
formatted_date = datetime.fromtimestamp(row.pop('date')).strftime("%Y-%m-%d %H:%M:%S")
del row['formatted_date']
res_dict[ticker][formatted_date] = row
except:
raise ValueError("System is temporarily unavailable. Please try again later.")
return res_dict
def process(self):
start_date = self.start_date
end_date = self.end_date
ticker = self.ticker
cash = self.cash
date_range = pd.bdate_range(start=start_date, end=end_date)
values = pd.DataFrame({'Dates': date_range})
values['Dates'] = pd.to_datetime(values['Dates'])
raw_data = YahooFinancials(ticker)
raw_data = raw_data.get_historical_price_data(start_date, end_date,
"daily")
df = pd.DataFrame(raw_data[ticker]['prices'])[[
'formatted_date', 'open', 'high', 'low', 'close', 'volume'
]]
df.columns = ['Dates1', 'open', 'high', 'low', 'close', 'volume']
df['Dates1'] = pd.to_datetime(df['Dates1'])
values = values.merge(df,
how='left',
left_on='Dates',
right_on='Dates1')
values = values.drop(labels='Dates1', axis=1)
values = values.fillna(method="ffill", axis=0)
values = values.fillna(method="bfill", axis=0)
cols = values.columns.drop('Dates')
values[cols] = values[cols].apply(pd.to_numeric,
errors='coerce').round(decimals=3)
values.set_index('Dates', inplace=True)
values = bt.feeds.PandasData(dataname=values)
self.values = values
def read_ticker_to_df(self):
ticker = self.ticker_details['Ticker'].to_list()
names = self.ticker_details['Description'].to_list()
#Extracting Data from Yahoo Finance and Adding them to Values table using date as key
for k, i in enumerate(ticker):
raw_data = YahooFinancials(i)
raw_data = raw_data.get_historical_price_data(
self.start_date, self.end_date, "daily")
df = pd.DataFrame(
raw_data[i]['prices'])[['formatted_date', 'adjclose']]
if k == 0:
self.DF['Date'] = df['formatted_date']
df.columns = ['Date1', i]
df['Date1'] = pd.to_datetime(df['Date1'])
self.DF['Date'] = pd.to_datetime(self.DF['Date'])
self.DF = self.DF.merge(df,
how='left',
left_on='Date',
right_on='Date1')
self.DF = self.DF.drop(labels='Date1', axis=1)
#Renaming columns to represent instrument names rather than their ticker codes for ease of readability
names.insert(0, 'Date')
self.DF.columns = names
print('\n AFTER YAHOOFINANCE {} \n'.format(len(self.DF)))
def get_historical_data(
self,
ticker: str,
start_date: datetime.date,
end_date: datetime.date,
time_interval: Interval,
) -> YFPriceHistory:
if start_date > end_date:
raise InvalidPeriodError
yahoo_financial = YahooFinancials(ticker)
data = yahoo_financial.get_historical_price_data(
start_date=date_to_str(start_date),
end_date=date_to_str(end_date),
time_interval=time_interval.value,
)
data = data[ticker]
if "prices" not in data:
raise InvalidTickerError
prices = data["prices"]
if not prices:
raise NoHistoricalDataError
time_offset = data["timeZone"]["gmtOffset"]
return YFPriceHistory.create(
prices=prices,
time_offset=time_offset,
currency=Currency(data["currency"]),
)
def add_technical_ind(self):
raw_data = YahooFinancials('^GSPC')
start_date = self.DF['Date'].iloc[0]
end_date = self.DF['Date'].iloc[-1]
start_date = datetime.strftime(start_date, "%Y-%m-%d")
end_date = datetime.strftime(end_date, "%Y-%m-%d")
raw_data = raw_data.get_historical_price_data(start_date, end_date,
"daily")
df = pd.DataFrame(raw_data['^GSPC']['prices'])[[
'formatted_date', 'adjclose', 'open', 'high', 'low', 'close',
'volume'
]]
self.df_tech = pd.DataFrame()
self.df_tech['Date'] = df['formatted_date']
self.df_tech['RSI'] = df.ta.rsi()
self.df_tech['RSI_5'] = self.df_tech['RSI'].shift(5)
self.df_tech['RSI_10'] = self.df_tech['RSI'].shift(10)
self.df_tech['RSI_20'] = self.df_tech['RSI'].shift(20)
self.df_tech.set_index('Date')
self.the_list.append('RSI')
self.the_list.append('RSI_5')
self.the_list.append('RSI_10')
self.the_list.append('RSI_20')
def Get_Tickers(T):
YF_Tickers = YahooFinancials(T)
YF_Tickers_Hist = YF_Tickers.get_historical_price_data(
Start_Date, End_Date, Period)
return YF_Tickers_Hist
def make_initial_data(start_date, end_date):
yahoo_financials = YahooFinancials('GC=F')
values = yahoo_financials.get_historical_price_data(
start_date, end_date, "daily")
# Extracting the closing prices from the JSON file
values = pd.DataFrame(
values['GC=F']['prices'])[['formatted_date', 'adjclose']]
values.columns = ['date', 'Gold']
values['date'] = pd.to_datetime(values['date'])
values['year'] = [d.year for d in values.date]
values['month'] = [d.strftime('%b') for d in values.date]
values["month"] = pd.to_datetime(values.month,
format='%b',
errors='coerce').dt.month
years = values['year'].unique()
# reordering the columns
values = values[['date', 'year', 'month', 'Gold']]
values = values.set_index('date')
# remove the records which gold price is null
values = values.dropna()
return values
def ticker_data(ticker):
yahoo_financials = YahooFinancials(ticker)
stock_prices = yahoo_financials.get_historical_price_data( \
start_date, end_date, 'daily' )
stock_df = pd.DataFrame(stock_prices[ticker]['prices'])
stock_df['formatted_date'] = pd.to_datetime(stock_df['formatted_date'])
stock_df = stock_df.drop( ['high', 'low', 'open', 'close', 'volume', 'date'],\
axis = 1 ).set_index( 'formatted_date' )
stock_df['simple_return'] = stock_df['adjclose'].pct_change()
stock_df['simple_return_std'] = stock_df['simple_return'].rolling( window = 21 )\
.agg( ['std'] )
stock_df['simple_return_mean'] = stock_df['simple_return'].rolling( window = 21 )\
.agg( ['mean'] )
mu = stock_df['simple_return_mean']
sigma = stock_df['simple_return_std']
stock_df['upper_limit'] = mu + (3 * sigma)
stock_df['lower_limit'] = mu - (3 * sigma)
condition = (stock_df['simple_return'] > stock_df['upper_limit']) | (
stock_df['simple_return'] < stock_df['lower_limit'])
stock_df['is_outlier'] = np.where(condition, 1, 0)
stock_df['symbol'] = ticker
return stock_df
def download_csv_data(ticker, start_date, end_date, path, freq='daily'):
'''
Function for downloading data from Yahoo Finance and storing the results in a CSV file accepted by `zipline`.
Parameters
------------
ticker : str
The ticker of the assset
start_date : str
The start date for downloading the data
end_date : str
The end date for downloading the data
path : str
The path to store the CSV file
freq : str
The frequency of the data
'''
yahoo_financials = YahooFinancials(ticker)
df = yahoo_financials.get_historical_price_data(start_date, end_date, freq)
df = pd.DataFrame(df[ticker]['prices']).drop(['date'], axis=1) \
.rename(columns={'formatted_date': 'date'}) \
.loc[:, ['date', 'open', 'high', 'low', 'close', 'volume']] \
.set_index('date')
df.index = pd.to_datetime(df.index)
df['dividend'] = 0
df['split'] = 1
# save data to csv for later ingestion
df.to_csv(path, header=True, index=True)
# plot the time series
df.close.plot(
title='{} prices --- {}:{}'.format(ticker, start_date, end_date))