class TestFred(unittest.TestCase):
def setUp(self):
self.fred = Fred()
def testGetSeries(self):
s = self.fred.get_series("SP500", observation_start="9/2/2014", observation_end="9/5/2014")
self.assertEqual(s.ix["9/2/2014"], 2002.28)
self.assertEqual(len(s), 4)
info = self.fred.get_series_info("PAYEMS")
self.assertEqual(info["title"], "All Employees: Total nonfarm")
# invalid series id
self.assertRaises(ValueError, self.fred.get_series, "invalid")
self.assertRaises(ValueError, self.fred.get_series_info, "invalid")
# invalid parameter
try:
self.fred.get_series("SP500", observation_start="invalid-datetime-str")
self.assertTrue(False, "previous line should have thrown a ValueError")
except ValueError:
pass
def testSearch(self):
personal_income_series = self.fred.search_by_release(175, limit=3, order_by="popularity", sort_order="desc")
series_ids = ["PCPI06037", "PCPI06075", "PCPI34039"]
for series_id in series_ids:
self.assertTrue(series_id in personal_income_series.index)
self.assertEqual(personal_income_series.ix[series_id, "observation_start"], datetime(1969, 1, 1))
def tearDown(self):
return
class TestFred(unittest.TestCase):
def setUp(self):
self.fred = Fred()
def testGetSeries(self):
s = self.fred.get_series('SP500', observation_start='9/2/2014', observation_end='9/5/2014')
self.assertEqual(s.ix['9/2/2014'], 2002.28)
self.assertEqual(len(s), 4)
info = self.fred.get_series_info('PAYEMS')
self.assertEqual(info['title'], 'All Employees: Total nonfarm')
# invalid series id
self.assertRaises(ValueError, self.fred.get_series, 'invalid')
self.assertRaises(ValueError, self.fred.get_series_info, 'invalid')
# invalid parameter
try:
self.fred.get_series('SP500', observation_start='invalid-datetime-str')
self.assertTrue(False, 'previous line should have thrown a ValueError')
except ValueError:
pass
def testSearch(self):
personal_income_series = self.fred.search_by_release(175, limit=3, order_by='popularity', sort_order='desc')
series_ids = ['PCPI06037', 'PCPI06075', 'PCPI24510']
for series_id in series_ids:
self.assertTrue(series_id in personal_income_series.index)
self.assertEqual(personal_income_series.ix[series_id, 'observation_start'], datetime(1969, 1, 1))
def tearDown(self):
return
def spx(start, end):
fred = Fred(api_key='3de60b3b483033f57252b59f497000cf')
s = fred.get_series('SP500', observation_start=start, observation_end=end)
return s
def fed_total_asset(start, end):
fred = Fred(api_key='3de60b3b483033f57252b59f497000cf')
s = fred.get_series('WALCL', observation_start=start, observation_end=end)
return s
def fred_3mon_ir_today(end=datetime.now()):
fred = Fred(api_key='3de60b3b483033f57252b59f497000cf')
s = fred.get_series('DTB3', observation_end=end)
return s[-1] / 100
from flask import Flask
from fredapi import Fred
import pandas as pd
import yaml
import datetime
import sys
import os
import settings
# Create FRED object
# https://github.com/mortada/fredapi
fred = Fred(settings.API_KEY)
today = datetime.date.today()
lastyear = today - datetime.timedelta(days=365)
def loadseries(*args):
with open("config.yml", "r") as ymlfile:
cfg = yaml.safe_load(ymlfile)
series = cfg["series"]
for series in cfg["series"]:
print("Fetching series: %s" % series)
i = fred.get_series_info(series)
print(i)
s = fred.get_series(series,
observation_start=lastyear,
observation_end=today)
print(s.tail())
def __init__(self, api_key):
self.fred = Fred(api_key=api_key)
"""
This is a simple project using the FRED API that reads in a list of data points that can be fully customized
"""
from fredapi import Fred
import pandas as pd
import matplotlib.pyplot as plt
fred = Fred(api_key = """key here""")
"""These are the keys that correspond to certain macro economic indicators and data points from the FRED Database"""
series = ['SP500','GDP', 'A067RL1A156NBEA', 'CPIAUCSL', 'A191RL1Q225SBEA', 'DGS10', 'IC4WSA',
'UNRATE', 'DEXUSEU', 'BAMLH0A0HYM2', 'MEHOINUSA672N', 'M2V', 'GFDEGDQ188S',
'FEDFUNDS', 'NAPM', 'DCOILWTICO', 'M2', 'CIVPART', 'PSAVERT', 'USD3MTD156N',
'T10Y2Y', 'HOUST', 'DGS30', 'MORTG', 'DEXCHUS', 'BUSLOANS', 'UEMPMEAN',
'EXPGSCA', 'NETEXP', 'A067RP1A027NBEA', 'FYFSD']
#strips the data down to the title, frequency of reporting, units, and the latest values
for t in series:
data = fred.get_series(t)
info = fred.get_series_info(t)
print info['title']
print info['frequency']
print info['units']
print " "
print "LATEST VALUES:"
print data.tail()
def setUp(self):
self.fred = Fred()
from fredapi import Fred
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
fred = Fred(api_key='10ea8a3688d94a2274fdff5c6579cd57')
data = fred.get_series('DDOI02CNA156NWDB') # SP500
"""
https://fred.stlouisfed.org/
https://fred.stlouisfed.org/series/SP500
https://fred.stlouisfed.org/series/LFWA64TTUSM647S Working Age Population
https://fred.stlouisfed.org/series/MEPAINUSA672N Real Median Personal Income in the United States
Real Disposable Personal Income: Per Capita (A229RX0)
"""
print(type(data))
df = pd.DataFrame(data, columns=['Value'])
df = df.dropna()
print(df.info())
print(df.describe())
print(df.head())
df.index = pd.to_datetime(df.index)
# df = df.resample(rule='M').last()
print(df.head())
df['Value_PCT'] = df['Value'].pct_change()
auto_correlation = df['Value_PCT'].autocorr()
print('The auto_correlation is: ', auto_correlation)
df['Value'].plot()
# and full joins it to our empty data frame
for ticker in tickerList:
tickerRetrieve = yf.Ticker(ticker)
tickerDf = pd.DataFrame(tickerRetrieve.history(period='max')['Close'])
tickerDf.rename(columns={'Close': str(ticker) + '_Close'}, inplace=True)
myData = pd.merge(myData,
tickerDf,
how='outer',
right_index=True,
left_index=True)
# make date into a column from index
myData.reset_index(inplace=True)
# lets get some FRED CPI data to compare our inflation to
fred = Fred(api_key=fredAPIKey)
CPIdata = fred.get_series('CPIAUCSL')
############################
### Data Cleaning ##########
############################
# assure Date is a datetime object and make sure we don't have any future dates
myData['Date'] = myData['Date'].dt.date
today = dt.date.today()
myData['Date'] = myData[myData['Date'] <= today]
assert myData['Date'].max() <= today
myData['Date'] = myData['Date'].astype('datetime64')
# assure all underlyings are floats
for ticker in tickerList:
params = {'$limit': 10000, '$order': 'date DESC'}
response = requests.get(url, headers=headers, params=params)
prices = pd.DataFrame(response.json(), dtype=float)
prices = reverse_dataframe(prices)
# prices = end_of_period_timeseries(prices)
# prices.set_index('date')
prices.index = pd.to_datetime(prices.date)
prices = end_of_period_timeseries(prices)
#--------------------------------------------------------------------------
# Get supplemental data from FRED (Federal Reserve Economic Data).
#--------------------------------------------------------------------------
# Initialize Fred client.
config = dotenv_values('../.env')
fred = Fred(api_key=config.get('FRED_API_KEY'))
# Find the observation time start.
observation_start = prices.index.min()
# Get the Federal funds interest rate.
interest_rate = fred.get_series('FEDFUNDS', observation_start=observation_start)
interest_rate = end_of_period_timeseries(interest_rate)
#--------------------------------------------------------------------------
# Estimate a VAR model.
#--------------------------------------------------------------------------
# Calculate inflation.
lag_price = prices.avg_1oz.shift(1)
inflation = (prices.avg_1oz - lag_price) / lag_price
#--------------------------------------------------------------------------
# Read data if already downloaded.
filename = './data/2021-12-06-MJ-Sales-Activity-by-License-Number-Traceability-Contingency-Reporting-Retail.xlsx'
data = pd.read_excel(filename, skiprows=3)
# Remove null values.
data = data.loc[data['License Number'].notnull()]
# Create a date column.
data['date'] = pd.to_datetime(data['Reporting Period'])
# Get the Washington State population.
config = dotenv_values('../.env')
fred_api_key = config.get('FRED_API_KEY')
fred = Fred(api_key=fred_api_key)
observation_start = data['date'].min().isoformat()
population = fred.get_series('WAPOP', observation_start=observation_start)
population = end_of_period_timeseries(population, 'Y')
population = population.multiply(1000) # thousands of people
new_row = pd.DataFrame([population[-1]], index=[pd.to_datetime('2021-12-31')])
population = pd.concat([population, pd.DataFrame(new_row)], ignore_index=False)
monthly_population = population[0].resample('M').mean().pad()
#--------------------------------------------------------------------------
# Look at monthly sales data over time.
#--------------------------------------------------------------------------
# Plot monthly sales.
monthly_production = data.groupby('Reporting Period').sum()
ax = monthly_production['Total Sales'].plot()
def get_Fred_Master():
seriesID_dict = {'RGDPg': {}, 'INDg': {}, 'Spread': {}, 'Stock': {}}
table_dict = {'RGDPg': {}, 'INDg': {}, 'Spread': {}, 'Stock': {}}
country_code_df = pd.read_excel('ISSO_CODE.xlsx')
country_Alpha2_L = []
country_Alpha3_L = []
for country in country_list:
country_Alpha2_L.append(country_code_df.loc[
country_code_df['Country_Name'] == country]['Alpha_2'].values[0])
country_Alpha3_L.append(country_code_df.loc[
country_code_df['Country_Name'] == country]['Alpha_3'].values[0])
#Real GDP for OECD Countries
# get Real Gross Domestic Product series, if not found for certain countries -> use alternatives
fred = Fred(api_key=API_key)
root = 'CLVMNACSCAB1GQ'
list_alt = []
for i, country in enumerate(country_list):
try:
data = fred.get_series(root + country_Alpha2_L[i])
seriesID_dict['RGDPg'][
country_list[i]] = root + country_Alpha2_L[i]
table_dict['RGDPg'][country] = data
except ValueError:
print('no such series for ' + country)
list_alt.append(country)
# handling the rest countries:
list_alt_seriesID = [
'AUSGDPRQDSMEI', 'NAEXKP01CAQ189S', 'NAEXKP01CLQ652S',
'CLVMNACSCAB1GQEL', 'CLVMNACSAB1GQIS', 'CLVMNACSAB1GQIE', '',
'JPNRGDPEXP', 'NAEXKP01KRQ189S', '', 'NAEXKP01NZQ189S',
'CLVMNACSAB1GQSK', 'NAEXKP01TRQ652S', 'GDPC1'
]
for i, country in enumerate(list_alt):
try:
data = fred.get_series(list_alt_seriesID[i])
seriesID_dict['RGDPg'][country_list[i]] = list_alt_seriesID[i]
table_dict['RGDPg'][country] = data
print('alternative series added for ' + country)
except ValueError:
print('no alternative series found for ' + country)
# get production of total industry series, if not found for certain countries -> use alternatives
fred = Fred(api_key=API_key)
root = 'PROINDQISMEI'
list_alt = []
for i, country in enumerate(country_list):
try:
data = fred.get_series(country_Alpha3_L[i] + root)
seriesID_dict['INDg'][country_list[i]] = country_Alpha3_L[i] + root
table_dict['INDg'][country] = data
except ValueError:
print('no such series for ' + country)
list_alt.append(country)
# get production of total industry series, if not found for certain countries -> use alternatives
fred = Fred(api_key=API_key)
root_1 = 'IRLTLT01'
freq = 'M'
root_2 = '156N'
list_alt = []
for i, country in enumerate(country_list):
try:
data = fred.get_series(root_1 + country_Alpha2_L[i] + freq +
root_2)
seriesID_dict['Spread'][
country_list[i]] = root_1 + country_Alpha2_L[i] + freq + root_2
table_dict['Spread'][country] = data
except ValueError:
print('no such series for ' + country)
list_alt.append(country)
# get production of total industry series, if not found for certain countries -> use alternatives
fred = Fred(api_key=API_key)
root_1 = 'SPASTT01'
freq = 'M'
root_2 = '661N'
list_alt = []
for i, country in enumerate(country_list):
try:
data = fred.get_series(root_1 + country_Alpha2_L[i] + freq +
root_2)
seriesID_dict['Stock'][
country_list[i]] = root_1 + country_Alpha2_L[i] + freq + root_2
table_dict['Stock'][country] = data
except ValueError:
print('no such series for ' + country)
list_alt.append(country)
#Combining results into one single table
# Step 1: Merge the indicators
country_table_dict = {}
for country in country_list:
country_table_dict[country] = pd.DataFrame(columns=['date'])
for indicator in table_dict.keys():
if country in table_dict[indicator].keys():
temp = table_dict[indicator][country].reset_index()
temp.columns = ['date', indicator]
# merge indicators into country_table
country_table_dict[country] = country_table_dict[
country].merge(temp, how='outer', on='date')
country_table_dict[country]['country'] = country
fred_table = pd.concat(country_table_dict.values()).reset_index(drop=True)
return fred_table
if recession[t - pd.DateOffset(months = 1)] == 0:
recessionstart.append(t)
elif recession[t + pd.DateOffset(months = 1)] == 0:
recessionend.append(t)
for i in range(len(recessionend)):
rect = plt.Rectangle((recessionstart[i], axes.get_ylim()[0]), (recessionend[i] - recessionstart[i]).days, axes.get_ylim()[1] - axes.get_ylim()[0], color = ".85")
axes.add_patch(rect)
return axes
#################### Main Code ####################
## Insert your API key here
fred = Fred(api_key = '7b90e5ff7af692a8c699383a7f344075')
## Pulling in recession dates for recession shading on graphs
## NOTE: The name is hard coded. Don't mess with it or the addrecessionshading function will break. Feel free to change the start date though if you need to
recession = fred.get_series("USREC", observation_start = '1/1/2000')
## List where each element is a string with the FRED ID of the data series you want to pull
series_names = ["GDPC1", # Real Gross Domestic Product, Billions of Chained 2009 Dollars, Seasonally Adjusted Annual Rate (GDPC1)
"CPILFESL", # Consumer Price Index for All Urban Consumers: All Items Less Food & Energy, Index 1982-84=100, Seasonally Adjusted (CPILFESL)
"PCEPILFE", # Personal Consumption Expenditures Excluding Food and Energy (Chain-Type Price Index), Index 2009=100, Seasonally Adjusted (PCEPILFE)
"UNRATE", # Civilian Unemployment Rate, Percent, Seasonally Adjusted (UNRATE)
"U6RATE", # Total unemployed, plus all marginally attached workers plus total employed part time for economic reasons, Percent, Seasonally Adjusted (U6RATE)
]
## Generate a dictionary of the series name a series containing the data pulled from FRED
data = {}
def effective_fed_rate(start, end):
fred = Fred(api_key='3de60b3b483033f57252b59f497000cf')
s = fred.get_series('FEDFUNDS',
observation_start=start,
observation_end=end)
return s
def gdp(start, end):
fred = Fred(api_key='3de60b3b483033f57252b59f497000cf')
s = fred.get_series('A191RL1Q225SBEA',
observation_start=start,
observation_end=end)
return s
# Step 1 : import data from FRED databases
from fredapi import Fred
fred = Fred(api_key="c3bc005d1a70992b2b3f3ef97d896c32")
query = fred.search("france")
query = query[(query["frequency_short"] == "Q")]
query = query[
(query["observation_start"] <= "1970-01-01, 00:00:00") & (query["observation_end"] > "2013-01-01, 00:00:00")
]
import pandas as pd
series = list()
for series_id in list(query["id"]):
new_serie = fred.get_series(series_id, observation_start="1970-01-01, 00:00:00")
series.append(new_serie)
print(series_id + ": done")
df = pd.concat(series, axis=1)
df.columns = list(query["id"])
print(df.head())
print(query["title"])
df.to_csv("/Users/nicolassaleille/Dropbox/ofpr/data/fred/france_query_2.csv")
# modele de prevision du PIB
def __init__(self):
self.fred=Fred(api_key='2c149bd5d320ce48476143f35aa2bf02')
# add the API keys
def get_keys():
api_kwarg = {}
with open('apikeys.txt', 'r') as f:
hold_lines = f.read().splitlines()
for each in hold_lines:
api_name, api_key_val = each.split('=')
api_kwarg[api_name] = api_key_val
return api_kwarg
api_kwarg = get_keys()
# init Fred
api_handler = Fred(api_key=api_kwarg['fred'])
def get_series(categories, fold):
for each in categories:
hold_series = api_handler.get_series_latest_release(each)
filename = 'data/' + fold + each + '.csv'
hold_series.to_csv(filename)
print(each + ' : ', len(hold_series))
print('First and last rows:')
print(hold_series.iloc[[0, -1]])
# Select Series
KPI_categories = [
'XTEXVA01USM664S', 'GDP', 'PCE', 'A939RC0Q052SBEA', 'FGCCSLQ027S', 'TOTCI',
def getFRED(self):
fred = Fred(api_key=self.fred_api_key)
data = fred.get_series_all_releases("GDP")
print(data)
def get_libor():
fred = Fred(api_key='a0718ea00e6784c5f8b452741622a98c')
Libor_1M = pd.DataFrame(
fred.get_series('USD1MTD156N', observation_start='1/1/2000'))
Libor_3M = pd.DataFrame(
fred.get_series('USD3MTD156N', observation_start='1/1/2000'))
Treasury_1Y = pd.DataFrame(
fred.get_series('DGS1', observation_start='1/1/2000'))
Treasury_5Y = pd.DataFrame(
fred.get_series('DGS5', observation_start='1/1/2000'))
Treasury_10Y = pd.DataFrame(
fred.get_series('DGS10', observation_start='1/1/2000'))
Treasury_20Y = pd.DataFrame(
fred.get_series('DGS20', observation_start='1/1/2000'))
Treasury_30Y = pd.DataFrame(
fred.get_series('DGS30', observation_start='1/1/2000'))
WILLREITIND = pd.DataFrame(
fred.get_series('WILLREITIND', observation_start='1/1/2000'))
OAS = pd.DataFrame(
fred.get_series('BAMLH0A0HYM2', observation_start='1/1/2000'))
GDP = pd.DataFrame(fred.get_series('GDPC1', observation_start='9/19/2011'))
CPI = pd.DataFrame(
fred.get_series('CPIAUCSL', observation_start='9/19/2011'))
Fed_Rate = pd.DataFrame(
fred.get_series('DFF', observation_start='9/19/2011'))
Uneply = pd.DataFrame(
fred.get_series('UNRATE', observation_start='9/19/2011'))
M2 = pd.DataFrame(fred.get_series('M2', observation_start='9/19/2011'))
Non_farm = pd.DataFrame(
fred.get_series('PAYEMS', observation_start='9/19/2011'))
Fed_Debt = pd.DataFrame(
fred.get_series('GFDEGDQ188S', observation_start='9/19/2011'))
Dollar_Index = pd.DataFrame(
fred.get_series('TWEXB', observation_start='9/19/2011'))
HOUST = pd.DataFrame(
fred.get_series('HOUST', observation_start='9/19/2011'))
macro_m_price = pd.concat(
[GDP, CPI, Uneply, M2, Non_farm, Fed_Debt, Dollar_Index, HOUST],
axis=1)
macro_d_price = pd.concat([
Libor_1M, Libor_3M, Treasury_1Y, Treasury_5Y, Treasury_10Y,
Treasury_20Y, Treasury_30Y, WILLREITIND, OAS
],
axis=1)
macro_m_price.columns = [
'GDP', 'CPI', 'Uneply', 'M2', 'Non_farm', 'Fed_Debt', 'Dollar_index',
'HOUST'
]
macro_d_price.columns = [
'Libor_1M', 'Libor_3M', 'Treasury_1Y', 'Treasury_5Y', 'Treasury_10Y',
'Treasury_20Y', 'Treasury_30Y', 'WILLREITIND', 'OAS'
]
# Calculate return
Fed_Rate = (Fed_Rate - Fed_Rate.shift(1)) / Fed_Rate.shift(1)
macro_m = (macro_m_price - macro_m_price.shift(1)) / macro_m_price.shift(1)
macro_d = (macro_d_price - macro_d_price.shift(1)) / macro_d_price.shift(1)
return Fed_Rate, macro_m_price, macro_m, macro_d_price, macro_d
from fredapi import Fred
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
# For reference: https://mortada.net/python-api-for-fred.html
# FRED API Key (must have FRED acct): https://research.stlouisfed.org/useraccount/apikeys
fred = Fred(api_key='ENTER YOUR OWN FRED API HERE')
### pulling Initial Claims data then creating a dataframe
df = fred.get_series('ICSA',
observation_start='2020-03-07',
observation_end='2020-07-14')
df = pd.DataFrame(data=df).reset_index() #turns series data into dataframe
df.columns = ['date', 'ICSA']
df['ICSA_millions'] = df['ICSA'] / 1000000
#print(df)
### creating graph
fig, ax = plt.subplots(figsize=(11, 5.5))
ax.bar(df['date'], df['ICSA_millions'], color='olivedrab', width=3, zorder=3
) # for some reason, zorder = 3 helps bars show up in front of grid
# making axes pretty
ax.spines['right'].set_visible(False)
ax.spines['left'].set_color('whitesmoke')
ax.spines['top'].set_visible(False)
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')
ax.grid(axis='y', color='gainsboro',
response = requests.get(url, headers=headers, params=params)
prices = pd.DataFrame(response.json())
# Get production stats (total employees, total plants, etc.) j3q7-3usu
url = f'{base}/j3q7-3usu.json'
params = {'$limit': 100, '$order': 'activitysummarydate DESC'}
response = requests.get(url, headers=headers, params=params)
production = pd.DataFrame(response.json())
#--------------------------------------------------------------------------
# 2. Clean the data, standardizing variables.
#--------------------------------------------------------------------------
# Initialize Fed Fred.
config = dotenv_values('../.env')
fred = Fred(api_key=config['FRED_API_KEY'])
# Calculate percent of the civilian labor force in Massachusetts.
labor_force = fred.get_series('MALF', observation_start='1/1/2020')
# Calculate total employees as a percent of all employees in MA.
total_ma_employees = fred.get_series('MANA', observation_start='1/1/2020')
# Optional: Correlate total sales with GDP (https://fred.stlouisfed.org/series/MANQGSP)
#--------------------------------------------------------------------------
# 3. Visualize the data.
#--------------------------------------------------------------------------
# plt.plot(production_data.activitysummarydate, production_data.total_employees)
def fred_3mon_ir(start, end):
fred = Fred(api_key='3de60b3b483033f57252b59f497000cf')
s = fred.get_series('DTB3', observation_start=start, observation_end=end)
return s
"font.size": 10,
})
import pymc3 as pm
import arviz as az
pm.__version__, az.__version__
START_DATE = datetime.date(1995, 1, 1)
with open("/home/gsinha/.config/gcm/gcm.toml") as f:
config = pytoml.load(f)
FRED_API_KEY = config["api_keys"]["fred"]
states = [x.abbr for x in us.STATES] + ["DC"]
# +
fred = Fred(api_key=FRED_API_KEY)
labor_df = pd.DataFrame(
1.0e3 *
pd.Series(fred.get_series("PAYEMS"), name="nfp")).reset_index().rename(
columns={"index": "medate"})
labor_df["medate"] += pd.tseries.offsets.MonthEnd(0)
# +
series = ["ICSA", "LAICLAIMS", "PRIICLAIMS", "NJICLAIMS"]
df = []
for v in series:
x = pd.Series(fred.get_series(v, observation_start=START_DATE), name=v)
df.append(x)
y_ts = pd.concat(df, axis=1)
def three_rate(start, end):
fred = Fred(api_key='3de60b3b483033f57252b59f497000cf')
s = fred.get_series('TB3MS', observation_start=start, observation_end=end)
return s
def project_claims(state, covid_wt, sum_df, epi_enc, verbose=False):
''' get labor market data from STL '''
def states_data(suffix, state, fred):
''' gets data from FRED for a list of indices '''
idx = "ICSA" if state == "US" else state + suffix
x = pd.Series(fred.get_series(idx, observation_start=START_DATE),
name=v)
x.name = state
return x
def forecast_claims(initval, initdate, enddate, covid_wt):
''' project initial claims '''
μ_β = sum_df.loc["β", "mean"]
μ_κ = sum_df.loc[["κ: COVID", "κ: Katrina"], "mean"].values
μ_decay = covid_wt * μ_κ[0] + (1 - covid_wt) * μ_κ[1]
dt_range = (pd.date_range(start=initdate, end=enddate, freq="W") -
pd.tseries.offsets.Day(1))
max_x = len(dt_range)
w = np.arange(max_x)
covid_idx = list(epi_enc.classes_).index("COVID")
katrina_idx = list(epi_enc.classes_).index("Katrina")
decay = covid_wt * κ[:, covid_idx] + (1 - covid_wt) * κ[:, katrina_idx]
μ = np.exp(-decay * np.power(w.reshape(-1, 1), β))
μ_df = pd.DataFrame(np.percentile(μ, q=[5, 25, 50, 75, 95], axis=1).T,
columns=["5th", "25th", "50th", "75th", "95th"
]) * initval
μ_df["period"] = w
ic = np.zeros(max_x)
ic[0] = 1
for j in np.arange(1, max_x, 1):
ic[j] = np.exp(-μ_decay * np.power(j, μ_β))
df = pd.concat([
pd.Series(np.arange(max_x), name="period"),
pd.Series(ic, name="ic_ratio"),
pd.Series(ic * initval, name="ic"),
pd.Series((ic * initval).cumsum(), name="cum_ic")
],
axis=1)
df.index = dt_range
μ_df.index = dt_range
return df, μ_df
fred = Fred(api_key=FRED_API_KEY)
ic_raw = states_data("ICLAIMS", state, fred)
init_value, init_date, last_date = (ic_raw[ic_raw.idxmax()],
ic_raw.idxmax(), ic_raw.index[-1])
end_date = (last_date + pd.tseries.offsets.QuarterEnd() +
pd.tseries.offsets.DateOffset(months=3))
if verbose:
print(
f'State: {state}, {init_value}, {init_date}, {end_date}, {last_date}'
)
ic_fct, ic_pct = forecast_claims(init_value, init_date, end_date, covid_wt)
ic_fct["state"] = state
ic_pct["state"] = state
return ic_raw, ic_fct, ic_pct, init_date, end_date
def high_yield_rate(start, end):
fred = Fred(api_key='3de60b3b483033f57252b59f497000cf')
s = fred.get_series('BAMLH0A0HYM2EY',
observation_start=start,
observation_end=end)
return s
import pandas as pd
from fredapi import Fred
filepath = "S:\CurrencyRiskAnalytics\PRESENTATIONS\FRED & OTHER DATA SOURCES\DataList.xlsx"
df = pd.read_excel(filepath)
file_name = "S:\CurrencyRiskAnalytics\PRESENTATIONS\FRED & OTHER DATA SOURCES\FRED Trial.xlsx"
fred = Fred(api_key="46b412ebf2535fc3addf5e27987aa6d6")
writer = pd.ExcelWriter(file_name,
engine='xlsxwriter',
datetime_format='mm/dd/yyyy')
for i, j in zip(df.iloc[:, 0], df.iloc[:, 1]):
df_fred = pd.DataFrame()
df_fred = fred.get_series(j)
sheetname = i[0:30]
df_fred.to_excel(writer, sheetname)
workbook = writer.book
worksheet = writer.sheets[sheetname]
worksheet.write('A1', 'DATE')
worksheet.write('B1', i)
chart = workbook.add_chart({'type': 'line'})
chart.add_series({
'categories': [sheetname, 1, 0, df_fred.shape[0], 0],
'values': [sheetname, 1, 1, df_fred.shape[0], 1],
'line': {
'color': '#503B6B',
'width': 1.5,
},
})
chart.set_x_axis({
'date_axis': True,
'major_unit': 5,
def unemployment(start, end):
fred = Fred(api_key='3de60b3b483033f57252b59f497000cf')
s = fred.get_series('UNRATE', observation_start=start, observation_end=end)
return s
def cpi_set(self):
"""Accesses the federal reserve of St Louis's API to make a data frame of CPI data for the specified CPI data.
This methods also resets the inflation data to a specified base month specified in the Init method.
:return: a dataframe containing the CPI values reindexed to the specified month and year
"""
fred = Fred(api_key=self.fred_api_key)
CPI_df = pd.concat([fred.get_series('CPGDFD01CNM661N').rename('China - Food'),
fred.get_series('CPGDFD01CNM661N').rename('China (inc. Hong Kong) - Food'),
fred.get_series('CUUR0000SEFJ').rename('USA - Dairy'),
fred.get_series('CP0114FRM086NEST').rename('France - Dairy'),
fred.get_series('CP0114PLM086NEST').rename('Poland - Dairy'),
fred.get_series('CPIUFDNS').rename('USA - Food'),
fred.get_series('CUSR0000SAF113').rename('USA - Fruits'),
fred.get_series('CUSR0000SAF113').rename('USA - Vegetables'),
fred.get_series('CUSR0000SEFP01').rename('USA - Coffee'),
fred.get_series('CUSR0000SAF112').rename('USA - Meat'),
fred.get_series('CUSR0000SAF112').rename('USA - Fish'),
fred.get_series('CUSR0000SAF112').rename('USA - Eggs'),
fred.get_series('CUSR0000SAF116').rename('USA - Wine'), # Alcoholic Beverage
fred.get_series('CUSR0000SAF116').rename('USA - Beer'), # Alcoholic Beverage
fred.get_series('CP0114NOM086NEST').rename('Norway - Dairy'),
], axis=1).reset_index()
cols = ['year', 'month', 'day']
CPI_df['year'], CPI_df['month'], CPI_df['day'] = CPI_df['index'].astype(str).str.split('-', 2).str
CPI_df[cols] = CPI_df[cols].apply(pd.to_numeric, errors='coerce', axis=1)
# Take the countries, leave the rest of the columns and re index CPI data based on base monthe and year.
for column in CPI_df.columns.difference(['day', 'month', 'year', 'index']):
mask = (CPI_df['year'] == self.base_year) & (CPI_df['month'] == self.base_month)
index_val = CPI_df.loc[mask, column].values[0]
CPI_df[column + ' - reset'] = CPI_df[column] / index_val
return CPI_df
def ppi(start, end):
fred = Fred(api_key='3de60b3b483033f57252b59f497000cf')
s = fred.get_series('PCUOMFGOMFG',
observation_start=start,
observation_end=end)
return s
import pandas as pd
import numpy as np
import statsmodels.api as sm
import statsmodels.stats.api as sms
from dash.dependencies import Output, Input, State
from dash.exceptions import PreventUpdate
from fredapi import Fred
import plotly.graph_objs as go
app = dash.Dash(__name__)
server = app.server
fred = Fred(api_key='0d3a129121b29e16035b20ea3947ecf5')
gdp = fred.get_series('GDPC1')
inf = fred.get_series('PCEPILFE')
isratio = fred.get_series('ISRATIO')
lei = fred.get_series('USALOLITONOSTSAM')
infyy = inf.pct_change(12) * 100
gdpyy = gdp.pct_change(4) * 100
spx = fred.get_series('SP500')
wti = fred.get_series('WTISPLC')
yc = fred.get_series('T10Y2YM')
dollar = fred.get_series('TWEXBMTH')
unrate = fred.get_series('UNRATE')
ahe = fred.get_series('AHETPI')
hhdebt = fred.get_series('TDSP')
error_log.write('\n' + str(now) + ' - ' + str(message))
def format_series(data, ident): # Turn the response series into a pandas dataframe.
plot_series = pd.DataFrame(index=range(len(data.index)))
plot_series[ident["x_axis"]] = data.index
plot_series[ident["y_axis"]] = list(data)
return plot_series
#################### GET DATA ####################
# Duplicated so that APIs can be swapped using minimal changes to an individual code block.
# Retreive the FIRST time series (FRED Unemployment).
meta_info = {"chart_name":"Unemployment Rate","x_axis":"Date", "y_axis":"Rate", "series_id":"UNRATE", "observation_start":"1995-01-01", "observation_end":FRED_date}
try:
fred = Fred(fred_api_key) # Login.
data = fred.get_series(series_id=meta_info["series_id"], observation_start=meta_info["observation_start"], observation_end=meta_info["observation_end"])
meta_info["data"] = format_series(data, meta_info)
charts.append(meta_info)
print_and_pause('Saved', meta_info["chart_name"])
except:
cleanup(f"Error retreiving and/or parsing series: {meta_info['chart_name']}.")
# Retreive the SECOND time series (FRED GDP).
meta_info = {"chart_name":"Real Gross Domestic Product","x_axis":"Date", "y_axis":"GDP", "series_id":"A191RL1Q225SBEA", "observation_start":"1995-01-01", "observation_end":FRED_date}
try:
fred = Fred(fred_api_key) # Login.
data = fred.get_series(series_id=meta_info["series_id"], observation_start=meta_info["observation_start"], observation_end=meta_info["observation_end"])
meta_info["data"] = format_series(data, meta_info)
charts.append(meta_info)
print_and_pause('Saved', meta_info["chart_name"])
from fredapi import Fred
fred = Fred(api_key='YOUR API')
def treasury_ten_year():
"""
Returns: 10-Year Treasury Constant Maturity Rate; Unit: percent per annum
"""
treasury_lt = fred.get_series('DGS10')
return treasury_lt
def treasury_one_year():
"""
Returns: 1-Year Treasury Constant Maturity Rate; Unit: percent per annum
"""
treasury_st = fred.get_series('DGS1')
return treasury_st
def treasury_three_month():
"""
Returns: 3-Month Treasury Constant Maturity Rate; Unit: percent per annum
"""
treasury_stm = fred.get_series('DGS3MO')
return treasury_stm
def unemployment_rate_us():
"""
Return: Unemployment Rate
from fredapi import Fred
import pandas as pd
from chorogrid import Colorbin, Chorogrid
import os
import numpy as np
from scipy.stats.stats import pearsonr
####### Import the FRED Data #######
api_key = 'f213b90a3ec4042de2259bceceb6ccfa'
os.environ["FRED_API_KEY"] = api_key
apiFred = Fred()
dfFred = apiFred.search_by_release(112)
dfStateUnEmp = dfFred[dfFred['title'].str.contains("Unemployment Rate in ")&~dfFred['title'].str.contains("Census")&dfFred['seasonal_adjustment_short'].str.match('SA')&~dfFred['title'].str.contains('DISCONTINUED')]
srsStateUnIDs = dfStateUnEmp['id'][:]
dictStateUN = {}
for i in srsStateUnIDs:
dictStateUN[i[:2]] = apiFred.get_series(i,observation_start='2007-01-01').mean()
dfStateUN = pd.DataFrame(list(dictStateUN.items()),columns=['State', 'Unemployment'])
##Make the Map
colors = ['#fff5eb', '#fee6ce', '#fdd0a2', '#fdae6b', '#fd8d3c', '#f16913', '#d94801', '#8c2d04']
chBin = Colorbin(dfStateUN['Unemployment'], colors, proportional=True, decimals=None)
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 09 21:15:19 2018
@author: michael
"""
import pandas as pd
from collections import OrderedDict
from fredapi import Fred
start_date = '2017-01-01'
fred = Fred(api_key='0ba41d84c2fee356667bfe102a1ecd6d')
def get_interest_rates(start_date, plot=True):
rates = {
'fed_funds_rate': fred.get_series('FEDFUNDS', observation_start=start_date),
'four_week_tbill': fred.get_series('DTB4WK', observation_start=start_date),
'three_month_tbill': fred.get_series('DTB3', observation_start=start_date),
'six_month_tbill': fred.get_series('DTB6', observation_start=start_date),
'one_year_tbill': fred.get_series('DTB1YR', observation_start=start_date),
}
interest_rates = pd.DataFrame()
for name, data in rates.items():
interest_rates[name] = data
interest_rates.fillna(method='ffill', inplace=True)
class myFred():
savePath=open('ini/FredSavePath.txt').read()
codeDict={
'Real_GDP':'A191RL1Q225SBEA',
'Initial_Claims':'IC4WSA',
'Total_Nonfarm_Payrolls':'PAYEMS',
'Civilian_Unemployment_Rate':'UNRATE',
'Industrial_Production_Index': 'INDPRO'
}
speedList=['Initial_Claims','IC4WSA','Total_Nonfarm_Payrolls','PAYEMS',
'Industrial_Production_Index', 'INDPRO']
def __init__(self):
self.fred=Fred(api_key='2c149bd5d320ce48476143f35aa2bf02')
def getData(self,name=None,code=None,start=datetime(1999,1,1),end=None):
if code is None:
code=self.codeDict[name]
data=self.fred.get_series(code,observation_start=start,observation_end=end)
data=pandas.DataFrame({'Date':data.index.tolist(),'Value':data.tolist()})
date=data['Date'].tolist()
t=[]
for d in date:
try:
t.append(d.timestamp())
except Exception as e:
t.append(None)
data.insert(0,'time',t)
return data.dropna()
def saveMultiple(self,*namelist):
if len(namelist)==0:
namelist=self.codeDict.keys()
con=sqlite3.connect('%s/%s.db' % (self.savePath,'FredData'))
for k in self.codeDict.keys():
print(k)
data=self.getData(name=k)
if k in self.speedList:
data=self.rsi(self.Speed(data))
self.save_sql(k,data.set_index('time').dropna(),con=con)
con.close()
def Speed(self,data):
velocity=[0]
acceleration=[0,0]
value=data['Value'].tolist()
former=value[0]
for v in value[1:]:
velocity.append(v-former)
former=v
former=velocity[1]
for v in velocity[2:]:
acceleration.append(v-former)
former=v
data.insert(2,'Velocity',velocity)
data.insert(3,'Acceleration',acceleration)
return data
def rsi(self,data,column='Velocity',period=5):
RSI=ind.RSI(data['time'],data[column],period)
return data.merge(RSI,'outer','time')
def save_sql(self,table,data,name=None,path=None,con=None,if_exists='replace'):
close=False
if path is None:
path='%s/%s.db' % (self.savePath,name)
if con is None:
con=sqlite3.connect(path)
close=True
data.to_sql(table,con,if_exists=if_exists)
if close:
con.close()
# -*- coding: utf-8 -*-
"""
Spyder Editor
This is a temporary script file.
"""
import pandas as pd
from fredapi import Fred
import matplotlib.pyplot as plt
from matplotlib.dates import date2num , DateFormatter
import datetime as dt
fred = Fred(api_key='70da3ebf45f3904117b34c5147b93443')
from IPython.core.pylabtools import figsize
figsize(20, 5)
oer=fred.get_series('CUSR0000SEHC01',observation_start='2002-01-01') #owner equivalent rent
real_rent=fred.get_series('CUSR0000SEHA',observation_start='2002-01-01')
pce=fred.get_series('PCE',observation_start='2002-01-01')
#not so simple to plot datetime in matplot lib... typically requires a conversion to strings
fig, ax= plt.subplots()
ax.plot_date(oer.index,oer,fmt='-',color='green',label='owner equivalent rent') #this works well... now I want to add other series to this
ax.plot_date(oer.index,real_rent,fmt='-',color='red',label='rent of primary residence')
plt.title('owner equivalent rent')
legend = ax.legend(loc='lower right', shadow=True)
#frame = legend.get_frame()
plt.show()
