def GetMeta(self, meta_in):
"""
Get the metadata.
:param metadata: econ_platform_core.SeriesMetadata
:return: econ_platform_core.SeriesMetadata
"""
query_ticker = meta_in.ticker_query
api_key = econ_platform_core.PlatformConfiguration['P_FRED']['api_key']
if api_key.lower() == 'none':
# KeyError - ha, ha, I kill myself...
raise KeyError(
'Error: need to set the FRED API key in the config.txt; available from St. Louis Fed.'
)
fred = fredapi.Fred(api_key=api_key)
data = fred.get_series_info(str(query_ticker))
keys = data.index.to_list()
for k in keys:
meta_in.ProviderMetadata[k] = str(data[k])
if k == 'frequency_short':
meta_in.frequency = meta_in.ProviderMetadata[k]
if k == 'title':
meta_in.series_name = meta_in.ProviderMetadata[k]
meta_in.series_description = '{0} From FRED {1}'.format(
meta_in.ProviderMetadata[k], str(query_ticker))
return meta_in
def __init__(self):
"""
API Github: https://github.com/mortada/fredapi
Data Source: https://research.stlouisfed.org/
"""
DataAPIInterface.__init__(self)
self.key = '4ee85758812cd4eccda171cd90572fb2'
self.fred = fredapi.Fred(api_key=self.key)
self.log.info('Starting FRED API with key %s' % self.key)
def download(cls, **kwargs):
api_key = os.environ.get("FRED_API_KEY", None)
if api_key is None:
raise ValueError(
"Please set the environment key 'FRED_API_KEY' to download resources from FRED"
)
f = fredapi.Fred(api_key=api_key)
df = f.get_series(series_id="ATNHPIUS37964Q")
return df.rename_axis("date").reset_index(
name="housing_index").dropna()
def setUp(self):
"""Set up test.
If the FRED_API_KEY env variable is defined, we use it in the url to
help with quick checks between what's expected and what's returned by
Fred.
Only go against Fred during tests if fake_fred_call variable is True.
as results are subject to change and the tests should be runnable
without an internet connection or a FRED api key so they can be
run as part of automated continuous integration (e.g. travis-ci.org).
"""
self.fred = fredapi.Fred(api_key=fred_api_key)
self.fake_fred_call = fake_fred_call
self.__original_urlopen = fredapi.fred.urlopen
def fetch(self, series_meta):
"""
Do the fetch; will puke if do not have an API key in the config
TODO: Look to see if FRED_API_KEY environment variable is set...
Can only support single series queries...
:param series_meta: str
:return: list
"""
query_ticker = series_meta.ticker_query
api_key = econ_platform_core.PlatformConfiguration['P_FRED']['api_key']
if api_key.lower() == 'none':
# KeyError - ha, ha, I kill myself...
raise KeyError('Error: need to set the FRED API key in the config.txt; available from St. Louis Fed.')
fred = fredapi.Fred(api_key=api_key)
data = fred.get_series(str(query_ticker))
data.name = str(series_meta.ticker_full)
return [data,]
def GetFREDMatrix(seriesnames, progress=False, startdate=None, enddate=None):
#Get a matrix (rows=dates,columns=series) of data
#from FRED database
#See http://mortada.net/python-api-for-fred.html
#for information on how to get the FRED (Federal
#Reserve of St. Louis database) API, and how to get
#an API key. The key below is Ken Winston's.
#Inputs:
# seriesnames - list of strings with desired series
# progress - optional Boolean, print progress report if True
# startdate, enddate - optional 'YYYY-MM-DD' dates
#Returns:
# cdates - list of yyyy-mm-dd dates
# ratematrix - list of time series, one time
# series per exchange rate
import pandas as pd
import numpy as np
import fredapi
fred = fredapi.Fred(api_key='fd97b1fdb076ff1a86aff9b38d7a0e70')
#Get each time series and load it into a common dataframe
initialize = True
for sn in seriesnames:
if progress: print('Processing ', sn)
fs=fred.get_series(sn,observation_start=startdate, \
observation_end=enddate)
fs = fs.rename(sn) #put the name on the column
if initialize:
#Set up the dataframe with the first series
df = pd.DataFrame(fs)
initialize = False
else:
#concatenate the next series to the dataframe
df = pd.concat([df, fs], axis=1)
#The dataframe has aligned the dates
#strip out date series
cdates = df.index.strftime('%Y-%m-%d').tolist()
ratematrix = []
for i in range(len(df)):
x = list(df.iloc[i])
ratematrix.append(x)
return (cdates, ratematrix)
def get_fredapi_data(
diction={
'CPI': 'CPIAUCSL',
'RPP': 'QUSN628BIS',
'CR': 'CRDQUSAPABIS',
'PGDP': 'GDPDEF',
'GDP': 'GDP'
},
api_key='your_fred_key',
cutoff_date_min='1979-12-31',
cutoff_date_max='2017-03-01',
cutoff_date_min_cesano='1975-01-01'):
'''
This function is used to get the appropriate data from the FRED API repository
'''
fred = fredapi.Fred(api_key=api_key)
cpi = fred.get_series(diction['CPI'])
print cpi
RPP = fred.get_series(diction['RPP'])
CR = fred.get_series(diction['CR'])
pgdp = fred.get_series(diction['PGDP'])
gdp = fred.get_series(diction['GDP'])
# cpi = fred.get_series('CPIAUCSL')
# RPP = fred.get_series('QUSN628BIS')
# CR = fred.get_series('CRDQUSAPABIS')
# pgdp = fred.get_series('GDPDEF')
# gdp = fred.get_series('GDP')
# Resampling and making sure we get the correct dates as indices
cpi2 = cpi.resample('Q', label='left', how='mean')
print cpi2
cpi2 = pd.Series(cpi2.values, index=cpi2.index + pd.DateOffset(1))
#cpi2 = pd.Series(cpi2.values.ravel(), index=cpi2.index + pd.DateOffset(1))
# Paper transformations
data_gdp_sa = 100 * np.log(gdp / pgdp)
data_houses = 100 * np.log(RPP / cpi2)
data_credit = 100 * np.log(CR / pgdp)
return _data_transformations(data_gdp_sa, data_houses, data_credit,
cutoff_date_min_cesano, cutoff_date_min,
cutoff_date_max)
import warnings
import datetime
import fredapi
import pandas as pd
from django.conf import settings
from django.conf import settings
from .models import Report
from .svm_model import add_predict
fred = fredapi.Fred(api_key='14d8dd169b3688444536eadb2edf0896')
good_columns = [
u'AWHAETP', #1 Average weekly hours
u'ICNSA', #2 Average weekly jobless claims for unemployment insurance
u'ACDGNO', #3 Manufacturers' new orders for consumer goods/materials
u'M06006USM156NNBR', #4 Vendor performance (slower deliveries diffusion index)
u'NEWORDER', #5 Manufacturers' new orders for non-defense
u'PERMIT', #6 Building permits
u'SP500', #7 Stock price returns of 500 common stocks
u'M2', #8 Money Supply (M2)
u'T10YFF', #9 Interest rate spread
u'UMCSENT', #10 Index of consumer expectations
#Lagging variables
u'UEMPMEAN', #1 The average duration of unemployment (inverted)
u'BUSLOANS', #2 The value of outstanding commercial and industrial loans
#3 The change in labour cost per unit of output
u'MNFCTRIRSA', #4 The ratio of manufacturing and trade inventories to sales
u'MPRIME', #5 The ratio of consumer credit outstanding to personal income
u'PAYNSA', #6 The average prime rate charged by banks
import fredapi
import pandas as pd
import numpy as np
import pickle
fred = fredapi.Fred(
api_key_file=".fred_api_key"
) # You will need to register for this through st. louis fred.
train_data = fred.get_series("UNRATE",
observation_start="1960-01-01",
observation_end="1997-01-01")
test_data = fred.get_series("UNRATE",
observation_start="1997-04-01",
observation_end="2015-12-01")
horiz = 0
def make_data(x):
x = x / 100
x = pd.DataFrame(x)
def differ(x, y):
if y > 0:
y -= 1
x = x - x.shift(1)
return differ(x, y)
else:
return x
lags = pd.DataFrame()
for i in range(0, 36):
import fredapi
import numpy as np
import pandas as pd
from datetime import timedelta
from quant.lib import data_utils as du, timeseries_utils as tu
from quant.lib.main_utils import logger
DATABASE_NAME = 'quant'
INFO_TABLE_NAME = 'fred_econ_info'
SERIES_TABLE_NAME = 'fred_data'
RELEASE_TABLE_NAME = 'fred_econ_data'
CACHE_TABLE_NAME = 'fred_cached_data'
TABLE_FORMAT = "time_index DATETIME, realtime_start DATETIME, series_name VARCHAR(50), value DOUBLE"
INFO_TABLE_FORMAT = "series_name VARCHAR(50), description VARCHAR(50), value VARCHAR(1000)"
FREDKEY = 'ff64294203f79127f8d004d2726386ac'
_api = fredapi.Fred(api_key=FREDKEY)
# data config
US_ECON = [ # Economic indicator
'PAYEMS', 'USSLIND', 'FRBLMCI',
# Inflation
'T10YIE', 'T5YIFR', 'MICH', 'CPILFESL', 'PPIACO', 'CPIAUCSL', 'CPALTT01USQ661S', 'CPALTT01USM661S',
'CPALTT01USM659N',
# Consumption
'PCE', 'BPCCRO1Q156NBEA', 'UMCSENT', 'DPCERO1Q156NBEA', 'PCEDG', 'PCEC96', 'TOTALSA', 'RSXFS',
# Auto
'M12MTVUSM227SFWA', 'AISRSA', 'DAUPSA', 'VMTD11', 'DAUTOSAAR', 'IPG3361T3S', 'LAUTOSA',
'AUESA', 'FAUTOSA', 'MAUISA', 'CAUISA', 'B149RC1Q027SBEA',
# National income
'GDP', 'GDPC1',
# Interest rates
import datetime as dt
import textwrap
import contextlib
import pandas as pd
import fredapi
import fredapi.fred
# Change here if you want to make actual calls to Fred
# (https://api.stlouisfed.org/fred...)
# Make sure you FRED_API_KEY is set up and internet works.
fake_fred_call = True
fred_api_key = 'secret'
if not fake_fred_call:
fred_api_key = fredapi.Fred().api_key
class HTTPCall(object):
"""Encapsulates faked Fred call data."""
root_url = fredapi.Fred.root_url
def __init__(self, rel_url, response=None, side_effect=None):
"""Construct HTTPCall from argument.
Parameters:
-----------
rel_url: relative url to the root url for the call.
response: response to the call if any.
side_effect: side_effect to the call if any.
def create_connection(self):
connection = fredapi.Fred(api_key=self.api_key)
return connection
file_path = './data/author_data.csv'
# These dates are used to define the intervals to be used in the simulation
cutoff_date_min = '1979-12-31'
cutoff_date_max = '2017-03-01'
cutoff_date_min_cesano = '1975-01-01'
# Data can be read either from the author's data files or straight from FRED API database
# data_gdp_sa, d_data_gdp_sa, d_data_houses, d_data_credit = get_authors_data(file_path, cutoff_date_min=cutoff_date_min,
# cutoff_date_max = cutoff_date_max,
# cutoff_date_min_cesano = cutoff_date_min_cesano)
import fredapi
api_key = 'e47df9c3446af26f9c52fc03015ee6ec'
fred = fredapi.Fred(api_key=api_key)
diction = {
'CPI': 'CPIAUCSL',
'RPP': 'QUSN628BIS',
'CR': 'CRDQUSAPABIS',
'PGDP': 'GDPDEF',
'GDP': 'GDP'
}
cpi = fred.get_series(diction['CPI'])
print cpi
RPP = fred.get_series(diction['RPP'])
CR = fred.get_series(diction['CR'])
pgdp = fred.get_series(diction['PGDP'])
gdp = fred.get_series(diction['GDP'])
# cpi = fred.get_series('CPIAUCSL')
