class TestFred(unittest.TestCase):
def setUp(self):
self.fred = Fred()
def testGetSeries(self):
s = self.fred.get_series('SP500',
observation_start='9/2/2014',
observation_end='9/5/2014')
self.assertEqual(s.ix['9/2/2014'], 2002.28)
self.assertEqual(len(s), 4)
info = self.fred.get_series_info('PAYEMS')
self.assertEqual(info['title'], 'All Employees: Total nonfarm')
# invalid series id
self.assertRaises(ValueError, self.fred.get_series, 'invalid')
self.assertRaises(ValueError, self.fred.get_series_info, 'invalid')
# invalid parameter
try:
self.fred.get_series('SP500',
observation_start='invalid-datetime-str')
self.assertTrue(False,
'previous line should have thrown a ValueError')
except ValueError:
pass
def tearDown(self):
return
class TestFred(unittest.TestCase):
def setUp(self):
self.fred = Fred()
def testGetSeries(self):
s = self.fred.get_series('SP500', observation_start='9/2/2014', observation_end='9/5/2014')
self.assertEqual(s.ix['9/2/2014'], 2002.28)
self.assertEqual(len(s), 4)
info = self.fred.get_series_info('PAYEMS')
self.assertEqual(info['title'], 'All Employees: Total nonfarm')
# invalid series id
self.assertRaises(ValueError, self.fred.get_series, 'invalid')
self.assertRaises(ValueError, self.fred.get_series_info, 'invalid')
# invalid parameter
try:
self.fred.get_series('SP500', observation_start='invalid-datetime-str')
self.assertTrue(False, 'previous line should have thrown a ValueError')
except ValueError:
pass
def testSearch(self):
personal_income_series = self.fred.search_by_release(175, limit=3, order_by='popularity', sort_order='desc')
series_ids = ['PCPI06037', 'PCPI06075', 'PCPI24510']
for series_id in series_ids:
self.assertTrue(series_id in personal_income_series.index)
self.assertEqual(personal_income_series.ix[series_id, 'observation_start'], datetime(1969, 1, 1))
def tearDown(self):
return
class TestFred(unittest.TestCase):
def setUp(self):
self.fred = Fred()
def testGetSeries(self):
s = self.fred.get_series("SP500", observation_start="9/2/2014", observation_end="9/5/2014")
self.assertEqual(s.ix["9/2/2014"], 2002.28)
self.assertEqual(len(s), 4)
info = self.fred.get_series_info("PAYEMS")
self.assertEqual(info["title"], "All Employees: Total nonfarm")
# invalid series id
self.assertRaises(ValueError, self.fred.get_series, "invalid")
self.assertRaises(ValueError, self.fred.get_series_info, "invalid")
# invalid parameter
try:
self.fred.get_series("SP500", observation_start="invalid-datetime-str")
self.assertTrue(False, "previous line should have thrown a ValueError")
except ValueError:
pass
def testSearch(self):
personal_income_series = self.fred.search_by_release(175, limit=3, order_by="popularity", sort_order="desc")
series_ids = ["PCPI06037", "PCPI06075", "PCPI34039"]
for series_id in series_ids:
self.assertTrue(series_id in personal_income_series.index)
self.assertEqual(personal_income_series.ix[series_id, "observation_start"], datetime(1969, 1, 1))
def tearDown(self):
return
def junk_bond_demand(API):
date_end = time.strftime('%Y.%m.%d',time.localtime(time.time()))#today date
date_start = time.strftime('%Y.%m.%d',time.localtime(time.time()-2592000))#date of one month ago
fred = Fred(api_key=API)
junkbond = fred.get_series('BAMLH0A0HYM2EY',date_start,date_end) #junkbond data
investbond = fred.get_series('DAAA',date_start,date_end) #investment-grade bond
spread = [junkbond [i]-investbond[i] for i in range(min([len(investbond),len(junkbond)]))]
spread = [ i for i in spread if i == i]#remove nan value
#index = (spread[-1] - min(spread))/(max(spread)-min(spread))
index = CalculateIndex(spread)
return index
def get_licensees_by_county_ok(licensees, counties):
"""Get data on licensees by county in Oklahoma.
Calculate licensees per capita for each county in Oklahoma
by finding all licensees in a given county,
getting the population of that county,
and calculating the licensees per capita in that county.
"""
config = dotenv_values('../../../.env')
fred = Fred(api_key=config['FRED_API_KEY'])
county_data = {}
for county in counties:
county_name = county['name']
fred_code = county['population_source_code']
county_licensees = licensees.loc[licensees.county == county_name]
county_population_data = fred.get_series(fred_code)
county_population = int(county_population_data.iloc[-1] * 1000)
county_licensees_per_capita = len(county_licensees) / county_population
population_date = county_population_data.index[-1].isoformat()[:10]
entry = {
'population': f'{county_population:,}',
'population_source_code': fred_code,
'population_source':
f'https://fred.stlouisfed.org/series/{fred_code}',
'licensees_per_capita': county_licensees_per_capita,
'population_at': population_date,
'total_licensees': len(county_licensees)
}
county_data[county_name] = {**county, **entry}
return county_data
def fred_1r_ir_today(end=datetime.now()):
if USE_API:
fred = Fred(api_key='3de60b3b483033f57252b59f497000cf')
s = fred.get_series('DGS1', observation_end=end)
return s[-1] / 100
else:
return 1.2 / 100
def weekly_fred_data(self):
# May have to update to scan for the data frequency and then change it based on that.
fred = Fred(api_key='81fb59583aa03d2ce139e065d694c299')
fred_ids = self.fred_strings
input_df = pd.DataFrame({}) #self.weekly_spy_volume()
for id in fred_ids:
temp_df = pd.DataFrame(
fred.get_series(id, observation_start=self.start_date))
for i in range(len(temp_df.iloc[0:, 0])):
if np.isnan(temp_df.iloc[0:, 0][i]):
try:
temp_df.iloc[0:, 0][i] = temp_df.iloc[0:, 0][i - 1]
except:
temp_df.iloc[0:, 0][i] = temp_df.iloc[0:, 0][i + 1]
avg_temp_data, temp_start_dates = self.weekly_stats(temp_df)
temp_weekly_df = pd.DataFrame({
"Week": temp_start_dates,
id: avg_temp_data
})
temp_weekly_df = temp_weekly_df.set_index("Week")
input_df = pd.concat([input_df, temp_weekly_df], axis=1)
spy_dates = self.weekly_spy_volume().index
drop_indicies = input_df.index.difference(spy_dates)
spy_inputs = self.weekly_spy_volume().drop(drop_indicies)
bond_inputs = self.weekly_bond_volume().drop(drop_indicies)
gold_inputs = self.weekly_gold_volume().drop(drop_indicies)
input_df["SPY_Volume"] = self.weekly_spy_volume()["Weekly_Volume"]
input_df["SHY_Volume"] = self.weekly_bond_volume()["Weekly_Volume"]
input_df["GDX_Volume"] = self.weekly_gold_volume()["Weekly_Volume"]
return input_df
def get_state_current_population(state, api_key=None):
"""Get a given state's latest population from the Fed Fred API,
getting the number in 1000's and returning the absolute value.
Args:
(str): The state abbreviation for the state to retrieve population
data. The abbreviation can be upper or lower case.
(str): A Fed FRED API key. You can sign up for a free API key at
http://research.stlouisfed.org/fred2/. You can also pass `None`
and set the environment variable 'FRED_API_KEY' to the value of
your API key.
Returns:
(dict): Returns a dictionary with population values and source.
"""
fred = Fred(api_key=api_key)
state_code = state.upper()
population_source_code = f'{state_code}POP'
population = fred.get_series(population_source_code)
real_population = int(population.iloc[-1] * 1000)
population_date = population.index[-1].isoformat()[:10]
return {
'population': real_population,
'population_formatted': f'{real_population:,}',
'population_source_code': population_source_code,
'population_source':
f'https://fred.stlouisfed.org/series/{population_source_code}',
'population_at': population_date,
}
def Get_from_FRED(id=id, path=path):
data = list()
info = list()
id_for_loop = id.copy()
while id_for_loop:
for i in id_for_loop:
print(i, 'trying...')
try:
series = fred.get_series(i)
series_info = fred.get_series_info(i)
print('secceed.')
data.append(series)
info.append(series_info)
id_for_loop.remove(i)
except IOError:
print('did not secceed!')
df_info = pd.DataFrame(info).T
df_info.columns = list(df_info.loc['id'])
df_US = pd.DataFrame(data).T
df_US.columns = df_info.columns
df_US.index.name = 'date'
df_US.to_csv(path + 'fromFRED_data' + file_date + '.csv')
df_info.to_csv(path + 'fromFRED_info' + file_date + '.csv')
df_US.to_csv(path + 'fromFRED_data.csv')
df_info.to_csv(path + 'fromFRED_info.csv')
return
def main():
parser = argparse.ArgumentParser(description='scrap fred')
parser.add_argument('-input',
type=str,
default='data_tickers/fred_stats.csv',
help='input csv file list all tickers to scrap')
parser.add_argument('-output_prefix',
type=str,
default='../stock_data/raw_fred/',
help='prefix of the output file')
parser.add_argument('-apikey', type=str, help='Fred API key')
args = parser.parse_args()
# scrap the data
fred = Fred(api_key=args.apikey)
with open(args.input) as csvfile:
fredreader = csv.reader(csvfile, delimiter=',')
next(fredreader)
for row in fredreader:
filename = args.output_prefix + row[0] + '.csv'
print('Getting', row[0], '-', row[1])
try:
s = fred.get_series(row[0])
s.to_csv(filename)
except:
print('failed')
return 0
def update_fred():
con = connect_to_default(DATABASE)
fred = Fred(api_key=APIKEY)
fred_series_list = [
'GDPA',
'BASE',
'DEXUSEU',
'DGS10',
'BAMLH0A0HYM2',
'SP500',
'AAA',
'BAA',
'CIVPART',
'CPIAUCSL',
'CURRCIR',
'EXUSEU',
'FEDFUNDS',
'HOUST',
'INDPRO',
'MORTG',
'PAYEMS',
'PSAVERT',
'TB3MS',
'UMCSENT',
'UNRATE',
'GDP',
'GDPC1',
'GDPDEF',
'M2V',
'PCECC96',
'GFDEBTN',
'STLFSI',
'M1',
'M2',
'PAYEMS',
]
dflist = []
for ser in fred_series_list:
df = pd.DataFrame(fred.get_series(ser), columns=['value'])
df.index.name = 'date'
df.reset_index(inplace=True)
df['field'] = ser
dflist.append(df)
df = pd.concat(dflist, ignore_index=True)
if not 'fred' in con.table_names():
df.to_sql('fred', con, index=False)
else:
current_list = pd.read_sql(
"SELECT field, MAX(date) as 'max_date' from fred group by field",
con)
dfc = df.merge(current_list, on=['field'])
dfc = dfc[dfc['date'] > dfc['max_date']].drop('max_date', axis=1)
if not dfc.empty:
dfc.to_sql('fred', con, index=False, if_exists='append')
def currency_data(api_key):
fred = Fred(api_key=api_key)
USEUROforex = fred.get_series('DEXUSEU')
USEUROforex = pd.DataFrame(USEUROforex)
USEUROforex.columns = ['USvsEURO']
USEUROforex.fillna(
method="bbfill",
inplace=True,
)
JC = pd.read_csv('Joined_Closes.csv').iloc[:1478]
exchange_indices = [str(ix)[0:10] for ix in USEUROforex.index.values]
JC_indices = [ix[0:10] for ix in JC['Periods']]
labs = [i for i in exchange_indices if i not in JC_indices]
USEUROforex.reset_index(inplace=True)
USEUROforex_dict = {'USvsEURO':[]}
for i, ix in enumerate(USEUROforex['index']):
if str(ix)[0:10] in labs:
pass
else:
USEUROforex_dict['USvsEURO'].append(USEUROforex['USvsEURO'][i])
USEUROforex = pd.DataFrame.from_dict(USEUROforex_dict)
JC = JC.join(USEUROforex)
JC.drop('Unnamed: 0', axis=1, inplace=True)
JC.to_csv('Indicators_Joined_.csv')
def get_series_data(series_id: str,
start: str = None,
end: str = None) -> pd.DataFrame:
"""Get Series data. [Source: FRED]
Parameters
----------
series_id : str
Series ID to get data from
start : str
Start date to get data from, format yyyy-mm-dd
end : str
End data to get from, format yyyy-mm-dd
Returns
----------
pd.DataFrame
Series data
"""
df = pd.DataFrame()
try:
fredapi_client = Fred(cfg.API_FRED_KEY)
df = fredapi_client.get_series(series_id, start, end)
# Series does not exist & invalid api keys
except HTTPError as e:
console.print(e)
return df
def freddata(keyid, api):
fred = Fred(api_key=api)
df = fred.get_series(keyid)
df = df.to_frame().reset_index()
df.columns = ['date', 'value']
return df
def pull_data():
API_key = 'YOUR_FRED_API_KEY_HERE'
fred = Fred(api_key=API_key)
home_price = fred.get_series('CSUSHPINSA')
home_price = pd.DataFrame(home_price)
print(home_price.head())
mortgage = fred.get_series('MORTGAGE30US')
mortgage = pd.DataFrame(mortgage)
print(mortgage.head())
data = home_price.merge(mortgage, left_index=True, right_index=True)
data.columns = ['home_price', 'mortgage']
return (data)
def get_state_current_population(state):
"""Get a given state's latest population from the Fed Fred API,
getting the number in 1000's and returning the absolute value."""
config = dotenv_values('../../../.env')
fred = Fred(api_key=config['FRED_API_KEY'])
state_code = state.upper()
population = fred.get_series(f'{state_code}POP')
return population.iloc[-1] * 1000
def get_yield_curve(date: Optional[datetime]) -> Tuple[pd.DataFrame, str]:
"""Gets yield curve data from FRED
Parameters
----------
date: Optional[datetime]
Date to get curve for. If None, gets most recent date
Returns
-------
pd.DataFrame:
Dataframe of yields and maturities
str
Date for which the yield curve is obtained
"""
fredapi_client = Fred(cfg.API_FRED_KEY)
fred_series = {
"1Month": "DGS1MO",
"3Month": "DGS3MO",
"6Month": "DGS6MO",
"1Year": "DGS1",
"2Year": "DGS2",
"3Year": "DGS3",
"5Year": "DGS5",
"7Year": "DGS7",
"10Year": "DGS10",
"20Year": "DGS20",
"30Year": "DGS30",
}
df = pd.DataFrame()
if date is None:
date_to_get = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d")
else:
date_to_get = date.strftime("%Y-%m-%d")
for key, s_id in fred_series.items():
df = pd.concat(
[
df,
pd.DataFrame(fredapi_client.get_series(s_id, date_to_get),
columns=[key]),
],
axis=1,
)
if date is None:
date_of_yield = df.index[-1]
rates = pd.DataFrame(df.iloc[-1, :].values, columns=["Rate"])
else:
date_of_yield = date
series = df[df.index == date]
if series.empty:
return pd.DataFrame(), date.strftime("%Y-%m-%d")
rates = pd.DataFrame(series.values.T, columns=["Rate"])
rates["Maturity"] = [1 / 12, 1 / 4, 1 / 2, 1, 2, 3, 5, 7, 10, 20, 30]
return rates, date_of_yield
def get_fred_data(fredkey: str,
SeriesNameDict: dict = {'SeriesID': 'SeriesName'}):
"""
Imports Data from Federal Reserve
args:
fredkey - an API key from FRED
SeriesNameDict, pairs of FRED Series IDs and Series Names
Series id must match Fred IDs, but name can be anything
if default is use, several default samples are returned
"""
if not _has_fred:
raise ImportError("Package fredapi is required")
fred = Fred(api_key=fredkey)
if SeriesNameDict == {'SeriesID': 'SeriesName'}:
SeriesNameDict = {
'T10Y2Y':
'10 Year Treasury Constant Maturity Minus 2 Year Treasury Constant Maturity',
'DGS10': '10 Year Treasury Constant Maturity Rate',
'DCOILWTICO': 'Crude Oil West Texas Intermediate Cushing Oklahoma',
'SP500': 'S&P 500',
'DEXUSEU': 'US Euro Foreign Exchange Rate',
'DEXCHUS': 'China US Foreign Exchange Rate',
'DEXCAUS': 'Canadian to US Dollar Exchange Rate Daily',
'VIXCLS':
'CBOE Volatility Index: VIX', # this is a more irregular series
'T10YIE': '10 Year Breakeven Inflation Rate',
'USEPUINDXD':
'Economic Policy Uncertainty Index for United States', # also very irregular
}
series_desired = list(SeriesNameDict.keys())
fred_timeseries = pd.DataFrame(
columns=['date', 'value', 'series_id', 'series_name'])
for series in series_desired:
data = fred.get_series(series)
try:
series_name = SeriesNameDict[series]
except Exception:
series_name = series
data_df = pd.DataFrame({
'date': data.index,
'value': data,
'series_id': series,
'series_name': series_name,
})
data_df.reset_index(drop=True, inplace=True)
fred_timeseries = pd.concat([fred_timeseries, data_df],
axis=0,
ignore_index=True)
return fred_timeseries
def getCorporatesFred(self, trim_start, trim_end):
fred = Fred(api_key=FRED_API_KEY)
curr_trim_end = trim_start
if (self.corporates.size != 0):
self.trim_start = self.corporates['OIS'].index.min().date()
curr_trim_end = self.corporates['OIS'].index.max().date()
if trim_end <= curr_trim_end:
self.trim_end = curr_trim_end
return self.corporates
self.trim_start = trim_start
self.trim_end = trim_end
self.OIS = OIS(trim_start=trim_start, trim_end=trim_end)
self.datesAll = self.OIS.datesAll
self.datesAll.columns = [x.upper() for x in self.datesAll.columns]
self.datesAll.index = self.datesAll.DATE
self.OISData = self.OIS.getOIS()
for i in np.arange(len(self.OISData.columns)):
freq = self.OISData.columns[i]
self.tenors.append(self.myScheduler.extractDelay(freq=freq))
for rating in self.ratings.keys():
index = self.ratings[rating]
try:
corpSpreads = 1e-2 * (fred.get_series(
index,
observation_start=trim_start,
observation_end=trim_end).to_frame())
corpSpreads.index = [x.date() for x in corpSpreads.index[:]]
corpSpreads = pd.merge(left=self.datesAll,
right=corpSpreads,
left_index=True,
right_index=True,
how="left")
corpSpreads = corpSpreads.fillna(method='ffill').fillna(
method='bfill')
corpSpreads = corpSpreads.drop("DATE", axis=1)
self.corpSpreads[rating] = corpSpreads.T.fillna(
method='ffill').fillna(method='bfill').T
except Exception as e:
print(e)
print(index, " not found")
self.corpSpreads = pd.Panel.from_dict(self.corpSpreads)
self.corporates = {}
self.OISData.drop('DATE', axis=1, inplace=True)
ntenors = np.shape(self.OISData)[1]
for rating in self.ratings:
try:
tiledCorps = np.tile(self.corpSpreads[rating][0],
ntenors).reshape(np.shape(self.OISData))
self.corporates[rating] = pd.DataFrame(
data=(tiledCorps + self.OISData.values),
index=self.OISData.index,
columns=self.OISData.columns)
except:
print("Error in addition of Corp Spreads")
self.corporates['OIS'] = self.OISData
self.corporates = pd.Panel(self.corporates)
return self.corporates
def DailyRiskFreeRates():
fred = Fred(api_key='5e8edcc00dd0a9e40375540f32696316')
TBills = fred.get_series('DGS10').to_frame().reset_index()
TBills.columns = ['Date', 'RiskFree']
TBills.set_index('Date', inplace=True)
TBills['RiskFree'] = TBills['RiskFree'].apply(lambda x: x / 100)
TBills.to_csv(myPath + 'TBillRate.csv')
return TBills
def extract(self):
''' extracts data directly apply FRED API '''
df = {}
fred = Fred(api_key=self.options["api_key"])
for item in self.options["items"].keys():
try:
if self.options["end_date"]:
df[item] = fred.get_series(
item,
observation_start=self.options["start_date"],
observation_end=self.options["end_date"])
else:
df[item] = fred.get_series(
item, observation_start=self.options["start_date"])
except ValueError as v:
print("%s : %s " % (item, v))
continue
self.df = pd.DataFrame(df)
class FredApiData(AbstractApiData):
'''Fred data API class'''
def __init__(self, api_code, start_date, series_name=None):
super().__init__(api_code, start_date, series_name=series_name)
with open('ignore_folder/fred_api_key.txt','r') as file:
key = file.read().replace('\n', '')
self.fred = Fred(api_key = key)
self.df = pd.DataFrame(self.fred.get_series(self.api_code)).loc[start_date::]
self.df.columns = [self.series_name]
def scrape_macro(series_id: dict):
"""
:param series_id: dictionary of FRED ID (key) and Series name (value) pairs
:return:
"""
fred = Fred(api_key=config.FRED_API_KEY)
return [
pd.Series(data=fred.get_series(series_key).dropna(), name=series_value)
for series_key, series_value in series_id.items()
]
def retrieve_macro_data() -> pd.DataFrame():
"""
Retrieves macro data using FRED api from 01 Jan 2006 to 28 Feb 2021.
:return: DataFrame representing macro data.
"""
print('\n==========Collecting macro data==========')
INDICATORS = {
# 'DFEDTARL': 'Federal Funds Target Range - lower bound',
# 'DFEDTARU': 'Federal Funds Target Range - upper bound',
'GDPC1': 'Real GDP',
'CPIAUCSL': 'Consumer Price Index (CPI)',
'UNRATE': 'Unemployment Rate',
'PAYEMS': 'Total NonFarm payrolls (Employment)',
'RRSFS': 'Real Retail and Food Services Sales',
'GFDEBTN': 'Federal Debt',
'VIXCLS': 'CBOE Volatility Index (VIX)',
'DFF': 'Effective Federal Funds Rate',
}
START_DATE = '1/1/2006'
END_DATE = '28/2/2021'
API_KEY = '2fb48248f0ce3781c82ffca58fecfb36'
fred = Fred(api_key=API_KEY)
full_data = pd.DataFrame()
for indicator, name in INDICATORS.items():
print(f'Indicator: {indicator}, Name: {name}')
data = fred.get_series(indicator,
observation_start=START_DATE,
observation_end=END_DATE).rename(indicator)
df = pd.DataFrame(data)
if full_data.empty:
full_data = df
else:
full_data = pd.concat([full_data, df], axis=1)
print("==========Done==========\n")
# resample daily data to monthly
full_data = full_data \
.resample('M') \
.last() \
.interpolate() \
.rename_axis('date') \
.reset_index()
return full_data
def calc_equity_alloc(start_dt: str = '', end_dt: str = '') -> pd.Series:
if not start_dt:
start_dt = '1920-01-01'
if not end_dt:
end_dt = dt.today().strftime('%Y-%m-%d')
fred = Fred(api_key=os.environ['TOKEN_FRED'])
nonfin_biz_equity_liab = fred.get_series('NCBEILQ027S',
observation_start=start_dt,
observation_end=end_dt)
nonfin_biz_credit_liab = fred.get_series('BCNSDODNS',
observation_start=start_dt,
observation_end=end_dt)
household_nonprofit_credit_liab = fred.get_series(
'CMDEBT', observation_start=start_dt, observation_end=end_dt)
fedgov_credit_liab = fred.get_series('FGSDODNS',
observation_start=start_dt,
observation_end=end_dt)
localgov_ex_retirement_credit_liab = fred.get_series(
'SLGSDODNS', observation_start=start_dt, observation_end=end_dt)
fin_biz_equity_liab = fred.get_series('FBCELLQ027S',
observation_start=start_dt,
observation_end=end_dt)
restofworld_credit_liab = fred.get_series('DODFFSWCMI',
observation_start=start_dt,
observation_end=end_dt)
# Divide nonfinancial and financial business equity reliabilities by all credit instrument liability in the economy
equity_alloc = pd.Series(
(
(
nonfin_biz_equity_liab
+ fin_biz_equity_liab
)
/ 1000
) \
/ (
(
(
nonfin_biz_equity_liab
+ fin_biz_equity_liab
)
/ 1000
)
+ (
nonfin_biz_credit_liab
+ household_nonprofit_credit_liab
+ fedgov_credit_liab
+ localgov_ex_retirement_credit_liab
+ restofworld_credit_liab
)
)
)
return make_qtrly(equity_alloc, 'first')
def findData(self):
#Gets the series under Indusdrial Production & Capacity Utilization
fred = Fred(api_key='e0a47670a791268b5b30cdf7cc217c4c')
series = fred.search_by_category(3,
order_by='title',
filter=('frequency',
'Monthly')) #limit = 300
#Keeps only the Manufacturing series. Removes unnecessary title info
series = series[series['title'].str.startswith(
'Capacity Utilization: Manufacturing')]
series['title'] = series['title'].str.replace(
'Capacity Utilization: Manufacturing: Durable Goods: ', '')
series['title'] = series['title'].str.replace(
'Capacity Utilization: Manufacturing: Non-Durable Goods: ', '')
#Gets the NAICS codes and series IDs for each series
naics_code = series['title'].str.extract(
r'\= (.{3})') #Some have pt. before ). Need to fix
naics_code = naics_code.rename(columns={0: 'NAICS Code'})
naics_code['NAICS Code'] = pd.to_numeric(naics_code['NAICS Code'],
downcast='integer')
series['title'] = series['title'].str.replace(r'\(([^)]+)\)', '')
series_id = series.index.tolist()
#Makes a DataFrame with NAICS code, series ID, and title
dataset = pd.DataFrame(series.iloc[:, 3])
dataset = naics_code.merge(dataset, left_index=True, right_index=True)
#Gets data for each series from 1997 through 2018
data = {}
count = 0
for id in series.index:
data[id] = fred.get_series(id,
observation_start='1997-01-01',
observation_end='2019-12-01')
count += 1
if count == len(series) / 2:
time.sleep(10)
data = pd.DataFrame(data)
#Adds data to dataset and organizes by NAICS
data_id = data.transpose()
dataset['Series ID'] = series_id
dataset = dataset.merge(data_id, left_index=True, right_index=True)
dataset.index = naics_code['NAICS Code'].tolist()
dataset = dataset.drop('NAICS Code', 1)
dataset = dataset.sort_index()
dataset = dataset.rename(columns={'title': 'Industry'})
return dataset
def request(self, data):
"""
:param data: JSON Object to feed Quandl API request
:return: JSON object with the response from he API
"""
current_app.logger.debug("Cache missed")
try:
fred = Fred(api_key=self.api_key)
series = fred.get_series(data.get('series_id'))
return series.last('1D').get(0)
except Exception as e:
raise e
class TestFred(unittest.TestCase):
def setUp(self):
self.fred = Fred()
def testGetSeries(self):
s = self.fred.get_series('SP500',
observation_start='9/2/2014',
observation_end='9/5/2014')
self.assertEqual(s.ix['9/2/2014'], 2002.28)
self.assertEqual(len(s), 4)
info = self.fred.get_series_info('PAYEMS')
self.assertEqual(info['title'], 'All Employees: Total nonfarm')
# invalid series id
self.assertRaises(ValueError, self.fred.get_series, 'invalid')
self.assertRaises(ValueError, self.fred.get_series_info, 'invalid')
# invalid parameter
try:
self.fred.get_series('SP500',
observation_start='invalid-datetime-str')
self.assertTrue(False,
'previous line should have thrown a ValueError')
except ValueError:
pass
def testSearch(self):
personal_income_series = self.fred.search_by_release(
175, limit=3, order_by='popularity', sort_order='desc')
series_ids = ['PCPI06037', 'PCPI06075', 'PCPI34039']
for series_id in series_ids:
self.assertTrue(series_id in personal_income_series.index)
self.assertEqual(
personal_income_series.ix[series_id, 'observation_start'],
datetime(1969, 1, 1))
def tearDown(self):
return
class FredStats:
def __init__(self, api_key):
self.fred = Fred(api_key=api_key)
def retrieve_stats(self, series_names):
# Retrieve the data from the FRED api
frames = []
for series_name in series_names:
data_series = self.fred.get_series(series_name).rename(series_name)
frames.append(data_series)
data = pd.concat(frames, axis=1)
return data
def safe_heaven_demand():
spx = yf.Ticker("^GSPC")
hist = spx.history(period = "1y")
hist_close = hist['Close']
date_end = time.strftime('%Y.%m.%d',time.localtime(time.time()))#today date
date_start = time.strftime('%Y.%m.%d',time.localtime(time.time()-2592000))#date of one month ago
fred = Fred(api_key=Fred_API_key)
treasury = fred.get_series('DGS10',date_start,date_end)# yield of treasuty
Returns = np.diff(hist_close) / hist_close[:-1] # the return of SP 500
returns = Returns[-len(treasury):]
spread = [returns[i] - treasury[i] for i in range(len(returns))]
spread = [ i for i in spread if i == i]#remove nan value
#index = (spread[-1] - min(spread))/(max(spread)-min(spread))
index = CalculateIndex(spread)
return index
def get_series_data(series_id: str, start: str) -> pd.DataFrame:
"""Get Series data. [Source: FRED]
Parameters
----------
series_id : str
Series ID to get data from
start : str
Start date to get data from, format yyyy-mm-dd
Returns
----------
pd.DataFrame
Series data
"""
fredapi_client = Fred(cfg.API_FRED_KEY)
return fredapi_client.get_series(series_id, start)
def get_data(self):
if self.data_source == 'quandl':
self.db_obj['data'] = pull_quandl(self.data_id)
self.db_obj['data'].sort_index(inplace=True)
self.db_obj['start_date'] = self.db_obj['data'].index.min()
self.db_obj['end_date'] = self.db_obj['data'].index.max()
self.db_obj['data'] = self.db_obj['data'].to_json(date_unit='s')
if self.data_source == 'fred':
freddy = Fred(api_key=FRED_API_KEY)
self.db_obj['data'] = pd.DataFrame(freddy.get_series(self.data_id),
columns=[self.symbol_name])
self.db_obj['data'].sort_index(inplace=True)
self.db_obj['start_date'] = self.db_obj['data'].index.min()
self.db_obj['end_date'] = self.db_obj['data'].index.max()
self.db_obj['data'] = self.db_obj['data'].to_json(date_unit='s')
Spyder Editor
This is a temporary script file.
"""
import pandas as pd
from fredapi import Fred
import matplotlib.pyplot as plt
from matplotlib.dates import date2num , DateFormatter
import datetime as dt
fred = Fred(api_key='70da3ebf45f3904117b34c5147b93443')
from IPython.core.pylabtools import figsize
figsize(20, 5)
oer=fred.get_series('CUSR0000SEHC01',observation_start='2002-01-01') #owner equivalent rent
real_rent=fred.get_series('CUSR0000SEHA',observation_start='2002-01-01')
pce=fred.get_series('PCE',observation_start='2002-01-01')
#not so simple to plot datetime in matplot lib... typically requires a conversion to strings
fig, ax= plt.subplots()
ax.plot_date(oer.index,oer,fmt='-',color='green',label='owner equivalent rent') #this works well... now I want to add other series to this
ax.plot_date(oer.index,real_rent,fmt='-',color='red',label='rent of primary residence')
plt.title('owner equivalent rent')
legend = ax.legend(loc='lower right', shadow=True)
#frame = legend.get_frame()
plt.show()
'''playing around
import matplotlib.pyplot as plt
fred = Fred(api_key = """key here""")
"""These are the keys that correspond to certain macro economic indicators and data points from the FRED Database"""
series = ['SP500','GDP', 'A067RL1A156NBEA', 'CPIAUCSL', 'A191RL1Q225SBEA', 'DGS10', 'IC4WSA',
'UNRATE', 'DEXUSEU', 'BAMLH0A0HYM2', 'MEHOINUSA672N', 'M2V', 'GFDEGDQ188S',
'FEDFUNDS', 'NAPM', 'DCOILWTICO', 'M2', 'CIVPART', 'PSAVERT', 'USD3MTD156N',
'T10Y2Y', 'HOUST', 'DGS30', 'MORTG', 'DEXCHUS', 'BUSLOANS', 'UEMPMEAN',
'EXPGSCA', 'NETEXP', 'A067RP1A027NBEA', 'FYFSD']
#strips the data down to the title, frequency of reporting, units, and the latest values
for t in series:
data = fred.get_series(t)
info = fred.get_series_info(t)
print info['title']
print info['frequency']
print info['units']
print " "
print "LATEST VALUES:"
print data.tail()
print " "
#saves a PDF graph in the folder where code is stored
for i in series:
info=fred.get_series_info(i)
title=info['title']
recessionend.append(t)
for i in range(len(recessionend)):
rect = plt.Rectangle((recessionstart[i], axes.get_ylim()[0]), (recessionend[i] - recessionstart[i]).days, axes.get_ylim()[1] - axes.get_ylim()[0], color = ".85")
axes.add_patch(rect)
return axes
#################### Main Code ####################
## Insert your API key here
fred = Fred(api_key = '7b90e5ff7af692a8c699383a7f344075')
## Pulling in recession dates for recession shading on graphs
## NOTE: The name is hard coded. Don't mess with it or the addrecessionshading function will break. Feel free to change the start date though if you need to
recession = fred.get_series("USREC", observation_start = '1/1/2000')
## List where each element is a string with the FRED ID of the data series you want to pull
series_names = ["GDPC1", # Real Gross Domestic Product, Billions of Chained 2009 Dollars, Seasonally Adjusted Annual Rate (GDPC1)
"CPILFESL", # Consumer Price Index for All Urban Consumers: All Items Less Food & Energy, Index 1982-84=100, Seasonally Adjusted (CPILFESL)
"PCEPILFE", # Personal Consumption Expenditures Excluding Food and Energy (Chain-Type Price Index), Index 2009=100, Seasonally Adjusted (PCEPILFE)
"UNRATE", # Civilian Unemployment Rate, Percent, Seasonally Adjusted (UNRATE)
"U6RATE", # Total unemployed, plus all marginally attached workers plus total employed part time for economic reasons, Percent, Seasonally Adjusted (U6RATE)
]
## Generate a dictionary of the series name a series containing the data pulled from FRED
data = {}
for l in series_names:
data[l] = fred.get_series(l, observation_start = '1/1/2003')
## Settings for the GDP graph
from scipy.stats.stats import pearsonr
####### Import the FRED Data #######
api_key = 'f213b90a3ec4042de2259bceceb6ccfa'
os.environ["FRED_API_KEY"] = api_key
apiFred = Fred()
dfFred = apiFred.search_by_release(112)
dfStateUnEmp = dfFred[dfFred['title'].str.contains("Unemployment Rate in ")&~dfFred['title'].str.contains("Census")&dfFred['seasonal_adjustment_short'].str.match('SA')&~dfFred['title'].str.contains('DISCONTINUED')]
srsStateUnIDs = dfStateUnEmp['id'][:]
dictStateUN = {}
for i in srsStateUnIDs:
dictStateUN[i[:2]] = apiFred.get_series(i,observation_start='2007-01-01').mean()
dfStateUN = pd.DataFrame(list(dictStateUN.items()),columns=['State', 'Unemployment'])
##Make the Map
colors = ['#fff5eb', '#fee6ce', '#fdd0a2', '#fdae6b', '#fd8d3c', '#f16913', '#d94801', '#8c2d04']
chBin = Colorbin(dfStateUN['Unemployment'], colors, proportional=True, decimals=None)
chBin.set_decimals(1)
chBin.recalc(fenceposts=True)
chBin.fenceposts
colors_out = chBin.colors_out
legend_colors = chBin.colors_in
legend_labels = chBin.labels
cg = Chorogrid('X:\\Documents\\Research\\Tools\\Python\\chorogrid-master\\chorogrid\\databases\\usa_states.csv', list(dfStateUN['State']), colors_out)
cg.set_title('Average Unemployment by State (%)', font_dict={'font-size': 16})
# Step 1 : import data from FRED databases
from fredapi import Fred
fred = Fred(api_key="c3bc005d1a70992b2b3f3ef97d896c32")
query = fred.search("france")
query = query[(query["frequency_short"] == "Q")]
query = query[
(query["observation_start"] <= "1970-01-01, 00:00:00") & (query["observation_end"] > "2013-01-01, 00:00:00")
]
import pandas as pd
series = list()
for series_id in list(query["id"]):
new_serie = fred.get_series(series_id, observation_start="1970-01-01, 00:00:00")
series.append(new_serie)
print(series_id + ": done")
df = pd.concat(series, axis=1)
df.columns = list(query["id"])
print(df.head())
print(query["title"])
df.to_csv("/Users/nicolassaleille/Dropbox/ofpr/data/fred/france_query_2.csv")
# modele de prevision du PIB
class myFred():
savePath=open('ini/FredSavePath.txt').read()
codeDict={
'Real_GDP':'A191RL1Q225SBEA',
'Initial_Claims':'IC4WSA',
'Total_Nonfarm_Payrolls':'PAYEMS',
'Civilian_Unemployment_Rate':'UNRATE',
'Industrial_Production_Index': 'INDPRO'
}
speedList=['Initial_Claims','IC4WSA','Total_Nonfarm_Payrolls','PAYEMS',
'Industrial_Production_Index', 'INDPRO']
def __init__(self):
self.fred=Fred(api_key='2c149bd5d320ce48476143f35aa2bf02')
def getData(self,name=None,code=None,start=datetime(1999,1,1),end=None):
if code is None:
code=self.codeDict[name]
data=self.fred.get_series(code,observation_start=start,observation_end=end)
data=pandas.DataFrame({'Date':data.index.tolist(),'Value':data.tolist()})
date=data['Date'].tolist()
t=[]
for d in date:
try:
t.append(d.timestamp())
except Exception as e:
t.append(None)
data.insert(0,'time',t)
return data.dropna()
def saveMultiple(self,*namelist):
if len(namelist)==0:
namelist=self.codeDict.keys()
con=sqlite3.connect('%s/%s.db' % (self.savePath,'FredData'))
for k in self.codeDict.keys():
print(k)
data=self.getData(name=k)
if k in self.speedList:
data=self.rsi(self.Speed(data))
self.save_sql(k,data.set_index('time').dropna(),con=con)
con.close()
def Speed(self,data):
velocity=[0]
acceleration=[0,0]
value=data['Value'].tolist()
former=value[0]
for v in value[1:]:
velocity.append(v-former)
former=v
former=velocity[1]
for v in velocity[2:]:
acceleration.append(v-former)
former=v
data.insert(2,'Velocity',velocity)
data.insert(3,'Acceleration',acceleration)
return data
def rsi(self,data,column='Velocity',period=5):
RSI=ind.RSI(data['time'],data[column],period)
return data.merge(RSI,'outer','time')
def save_sql(self,table,data,name=None,path=None,con=None,if_exists='replace'):
close=False
if path is None:
path='%s/%s.db' % (self.savePath,name)
if con is None:
con=sqlite3.connect(path)
close=True
data.to_sql(table,con,if_exists=if_exists)
if close:
con.close()