class TestFred(unittest.TestCase): def setUp(self): self.fred = Fred() def testGetSeries(self): s = self.fred.get_series('SP500', observation_start='9/2/2014', observation_end='9/5/2014') self.assertEqual(s.ix['9/2/2014'], 2002.28) self.assertEqual(len(s), 4) info = self.fred.get_series_info('PAYEMS') self.assertEqual(info['title'], 'All Employees: Total nonfarm') # invalid series id self.assertRaises(ValueError, self.fred.get_series, 'invalid') self.assertRaises(ValueError, self.fred.get_series_info, 'invalid') # invalid parameter try: self.fred.get_series('SP500', observation_start='invalid-datetime-str') self.assertTrue(False, 'previous line should have thrown a ValueError') except ValueError: pass def tearDown(self): return
class TestFred(unittest.TestCase): def setUp(self): self.fred = Fred() def testGetSeries(self): s = self.fred.get_series('SP500', observation_start='9/2/2014', observation_end='9/5/2014') self.assertEqual(s.ix['9/2/2014'], 2002.28) self.assertEqual(len(s), 4) info = self.fred.get_series_info('PAYEMS') self.assertEqual(info['title'], 'All Employees: Total nonfarm') # invalid series id self.assertRaises(ValueError, self.fred.get_series, 'invalid') self.assertRaises(ValueError, self.fred.get_series_info, 'invalid') # invalid parameter try: self.fred.get_series('SP500', observation_start='invalid-datetime-str') self.assertTrue(False, 'previous line should have thrown a ValueError') except ValueError: pass def testSearch(self): personal_income_series = self.fred.search_by_release(175, limit=3, order_by='popularity', sort_order='desc') series_ids = ['PCPI06037', 'PCPI06075', 'PCPI24510'] for series_id in series_ids: self.assertTrue(series_id in personal_income_series.index) self.assertEqual(personal_income_series.ix[series_id, 'observation_start'], datetime(1969, 1, 1)) def tearDown(self): return
class TestFred(unittest.TestCase): def setUp(self): self.fred = Fred() def testGetSeries(self): s = self.fred.get_series("SP500", observation_start="9/2/2014", observation_end="9/5/2014") self.assertEqual(s.ix["9/2/2014"], 2002.28) self.assertEqual(len(s), 4) info = self.fred.get_series_info("PAYEMS") self.assertEqual(info["title"], "All Employees: Total nonfarm") # invalid series id self.assertRaises(ValueError, self.fred.get_series, "invalid") self.assertRaises(ValueError, self.fred.get_series_info, "invalid") # invalid parameter try: self.fred.get_series("SP500", observation_start="invalid-datetime-str") self.assertTrue(False, "previous line should have thrown a ValueError") except ValueError: pass def testSearch(self): personal_income_series = self.fred.search_by_release(175, limit=3, order_by="popularity", sort_order="desc") series_ids = ["PCPI06037", "PCPI06075", "PCPI34039"] for series_id in series_ids: self.assertTrue(series_id in personal_income_series.index) self.assertEqual(personal_income_series.ix[series_id, "observation_start"], datetime(1969, 1, 1)) def tearDown(self): return
def junk_bond_demand(API): date_end = time.strftime('%Y.%m.%d',time.localtime(time.time()))#today date date_start = time.strftime('%Y.%m.%d',time.localtime(time.time()-2592000))#date of one month ago fred = Fred(api_key=API) junkbond = fred.get_series('BAMLH0A0HYM2EY',date_start,date_end) #junkbond data investbond = fred.get_series('DAAA',date_start,date_end) #investment-grade bond spread = [junkbond [i]-investbond[i] for i in range(min([len(investbond),len(junkbond)]))] spread = [ i for i in spread if i == i]#remove nan value #index = (spread[-1] - min(spread))/(max(spread)-min(spread)) index = CalculateIndex(spread) return index
def get_licensees_by_county_ok(licensees, counties): """Get data on licensees by county in Oklahoma. Calculate licensees per capita for each county in Oklahoma by finding all licensees in a given county, getting the population of that county, and calculating the licensees per capita in that county. """ config = dotenv_values('../../../.env') fred = Fred(api_key=config['FRED_API_KEY']) county_data = {} for county in counties: county_name = county['name'] fred_code = county['population_source_code'] county_licensees = licensees.loc[licensees.county == county_name] county_population_data = fred.get_series(fred_code) county_population = int(county_population_data.iloc[-1] * 1000) county_licensees_per_capita = len(county_licensees) / county_population population_date = county_population_data.index[-1].isoformat()[:10] entry = { 'population': f'{county_population:,}', 'population_source_code': fred_code, 'population_source': f'https://fred.stlouisfed.org/series/{fred_code}', 'licensees_per_capita': county_licensees_per_capita, 'population_at': population_date, 'total_licensees': len(county_licensees) } county_data[county_name] = {**county, **entry} return county_data
def fred_1r_ir_today(end=datetime.now()): if USE_API: fred = Fred(api_key='3de60b3b483033f57252b59f497000cf') s = fred.get_series('DGS1', observation_end=end) return s[-1] / 100 else: return 1.2 / 100
def weekly_fred_data(self): # May have to update to scan for the data frequency and then change it based on that. fred = Fred(api_key='81fb59583aa03d2ce139e065d694c299') fred_ids = self.fred_strings input_df = pd.DataFrame({}) #self.weekly_spy_volume() for id in fred_ids: temp_df = pd.DataFrame( fred.get_series(id, observation_start=self.start_date)) for i in range(len(temp_df.iloc[0:, 0])): if np.isnan(temp_df.iloc[0:, 0][i]): try: temp_df.iloc[0:, 0][i] = temp_df.iloc[0:, 0][i - 1] except: temp_df.iloc[0:, 0][i] = temp_df.iloc[0:, 0][i + 1] avg_temp_data, temp_start_dates = self.weekly_stats(temp_df) temp_weekly_df = pd.DataFrame({ "Week": temp_start_dates, id: avg_temp_data }) temp_weekly_df = temp_weekly_df.set_index("Week") input_df = pd.concat([input_df, temp_weekly_df], axis=1) spy_dates = self.weekly_spy_volume().index drop_indicies = input_df.index.difference(spy_dates) spy_inputs = self.weekly_spy_volume().drop(drop_indicies) bond_inputs = self.weekly_bond_volume().drop(drop_indicies) gold_inputs = self.weekly_gold_volume().drop(drop_indicies) input_df["SPY_Volume"] = self.weekly_spy_volume()["Weekly_Volume"] input_df["SHY_Volume"] = self.weekly_bond_volume()["Weekly_Volume"] input_df["GDX_Volume"] = self.weekly_gold_volume()["Weekly_Volume"] return input_df
def get_state_current_population(state, api_key=None): """Get a given state's latest population from the Fed Fred API, getting the number in 1000's and returning the absolute value. Args: (str): The state abbreviation for the state to retrieve population data. The abbreviation can be upper or lower case. (str): A Fed FRED API key. You can sign up for a free API key at http://research.stlouisfed.org/fred2/. You can also pass `None` and set the environment variable 'FRED_API_KEY' to the value of your API key. Returns: (dict): Returns a dictionary with population values and source. """ fred = Fred(api_key=api_key) state_code = state.upper() population_source_code = f'{state_code}POP' population = fred.get_series(population_source_code) real_population = int(population.iloc[-1] * 1000) population_date = population.index[-1].isoformat()[:10] return { 'population': real_population, 'population_formatted': f'{real_population:,}', 'population_source_code': population_source_code, 'population_source': f'https://fred.stlouisfed.org/series/{population_source_code}', 'population_at': population_date, }
def Get_from_FRED(id=id, path=path): data = list() info = list() id_for_loop = id.copy() while id_for_loop: for i in id_for_loop: print(i, 'trying...') try: series = fred.get_series(i) series_info = fred.get_series_info(i) print('secceed.') data.append(series) info.append(series_info) id_for_loop.remove(i) except IOError: print('did not secceed!') df_info = pd.DataFrame(info).T df_info.columns = list(df_info.loc['id']) df_US = pd.DataFrame(data).T df_US.columns = df_info.columns df_US.index.name = 'date' df_US.to_csv(path + 'fromFRED_data' + file_date + '.csv') df_info.to_csv(path + 'fromFRED_info' + file_date + '.csv') df_US.to_csv(path + 'fromFRED_data.csv') df_info.to_csv(path + 'fromFRED_info.csv') return
def main(): parser = argparse.ArgumentParser(description='scrap fred') parser.add_argument('-input', type=str, default='data_tickers/fred_stats.csv', help='input csv file list all tickers to scrap') parser.add_argument('-output_prefix', type=str, default='../stock_data/raw_fred/', help='prefix of the output file') parser.add_argument('-apikey', type=str, help='Fred API key') args = parser.parse_args() # scrap the data fred = Fred(api_key=args.apikey) with open(args.input) as csvfile: fredreader = csv.reader(csvfile, delimiter=',') next(fredreader) for row in fredreader: filename = args.output_prefix + row[0] + '.csv' print('Getting', row[0], '-', row[1]) try: s = fred.get_series(row[0]) s.to_csv(filename) except: print('failed') return 0
def update_fred(): con = connect_to_default(DATABASE) fred = Fred(api_key=APIKEY) fred_series_list = [ 'GDPA', 'BASE', 'DEXUSEU', 'DGS10', 'BAMLH0A0HYM2', 'SP500', 'AAA', 'BAA', 'CIVPART', 'CPIAUCSL', 'CURRCIR', 'EXUSEU', 'FEDFUNDS', 'HOUST', 'INDPRO', 'MORTG', 'PAYEMS', 'PSAVERT', 'TB3MS', 'UMCSENT', 'UNRATE', 'GDP', 'GDPC1', 'GDPDEF', 'M2V', 'PCECC96', 'GFDEBTN', 'STLFSI', 'M1', 'M2', 'PAYEMS', ] dflist = [] for ser in fred_series_list: df = pd.DataFrame(fred.get_series(ser), columns=['value']) df.index.name = 'date' df.reset_index(inplace=True) df['field'] = ser dflist.append(df) df = pd.concat(dflist, ignore_index=True) if not 'fred' in con.table_names(): df.to_sql('fred', con, index=False) else: current_list = pd.read_sql( "SELECT field, MAX(date) as 'max_date' from fred group by field", con) dfc = df.merge(current_list, on=['field']) dfc = dfc[dfc['date'] > dfc['max_date']].drop('max_date', axis=1) if not dfc.empty: dfc.to_sql('fred', con, index=False, if_exists='append')
def currency_data(api_key): fred = Fred(api_key=api_key) USEUROforex = fred.get_series('DEXUSEU') USEUROforex = pd.DataFrame(USEUROforex) USEUROforex.columns = ['USvsEURO'] USEUROforex.fillna( method="bbfill", inplace=True, ) JC = pd.read_csv('Joined_Closes.csv').iloc[:1478] exchange_indices = [str(ix)[0:10] for ix in USEUROforex.index.values] JC_indices = [ix[0:10] for ix in JC['Periods']] labs = [i for i in exchange_indices if i not in JC_indices] USEUROforex.reset_index(inplace=True) USEUROforex_dict = {'USvsEURO':[]} for i, ix in enumerate(USEUROforex['index']): if str(ix)[0:10] in labs: pass else: USEUROforex_dict['USvsEURO'].append(USEUROforex['USvsEURO'][i]) USEUROforex = pd.DataFrame.from_dict(USEUROforex_dict) JC = JC.join(USEUROforex) JC.drop('Unnamed: 0', axis=1, inplace=True) JC.to_csv('Indicators_Joined_.csv')
def get_series_data(series_id: str, start: str = None, end: str = None) -> pd.DataFrame: """Get Series data. [Source: FRED] Parameters ---------- series_id : str Series ID to get data from start : str Start date to get data from, format yyyy-mm-dd end : str End data to get from, format yyyy-mm-dd Returns ---------- pd.DataFrame Series data """ df = pd.DataFrame() try: fredapi_client = Fred(cfg.API_FRED_KEY) df = fredapi_client.get_series(series_id, start, end) # Series does not exist & invalid api keys except HTTPError as e: console.print(e) return df
def freddata(keyid, api): fred = Fred(api_key=api) df = fred.get_series(keyid) df = df.to_frame().reset_index() df.columns = ['date', 'value'] return df
def pull_data(): API_key = 'YOUR_FRED_API_KEY_HERE' fred = Fred(api_key=API_key) home_price = fred.get_series('CSUSHPINSA') home_price = pd.DataFrame(home_price) print(home_price.head()) mortgage = fred.get_series('MORTGAGE30US') mortgage = pd.DataFrame(mortgage) print(mortgage.head()) data = home_price.merge(mortgage, left_index=True, right_index=True) data.columns = ['home_price', 'mortgage'] return (data)
def get_state_current_population(state): """Get a given state's latest population from the Fed Fred API, getting the number in 1000's and returning the absolute value.""" config = dotenv_values('../../../.env') fred = Fred(api_key=config['FRED_API_KEY']) state_code = state.upper() population = fred.get_series(f'{state_code}POP') return population.iloc[-1] * 1000
def get_yield_curve(date: Optional[datetime]) -> Tuple[pd.DataFrame, str]: """Gets yield curve data from FRED Parameters ---------- date: Optional[datetime] Date to get curve for. If None, gets most recent date Returns ------- pd.DataFrame: Dataframe of yields and maturities str Date for which the yield curve is obtained """ fredapi_client = Fred(cfg.API_FRED_KEY) fred_series = { "1Month": "DGS1MO", "3Month": "DGS3MO", "6Month": "DGS6MO", "1Year": "DGS1", "2Year": "DGS2", "3Year": "DGS3", "5Year": "DGS5", "7Year": "DGS7", "10Year": "DGS10", "20Year": "DGS20", "30Year": "DGS30", } df = pd.DataFrame() if date is None: date_to_get = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d") else: date_to_get = date.strftime("%Y-%m-%d") for key, s_id in fred_series.items(): df = pd.concat( [ df, pd.DataFrame(fredapi_client.get_series(s_id, date_to_get), columns=[key]), ], axis=1, ) if date is None: date_of_yield = df.index[-1] rates = pd.DataFrame(df.iloc[-1, :].values, columns=["Rate"]) else: date_of_yield = date series = df[df.index == date] if series.empty: return pd.DataFrame(), date.strftime("%Y-%m-%d") rates = pd.DataFrame(series.values.T, columns=["Rate"]) rates["Maturity"] = [1 / 12, 1 / 4, 1 / 2, 1, 2, 3, 5, 7, 10, 20, 30] return rates, date_of_yield
def get_fred_data(fredkey: str, SeriesNameDict: dict = {'SeriesID': 'SeriesName'}): """ Imports Data from Federal Reserve args: fredkey - an API key from FRED SeriesNameDict, pairs of FRED Series IDs and Series Names Series id must match Fred IDs, but name can be anything if default is use, several default samples are returned """ if not _has_fred: raise ImportError("Package fredapi is required") fred = Fred(api_key=fredkey) if SeriesNameDict == {'SeriesID': 'SeriesName'}: SeriesNameDict = { 'T10Y2Y': '10 Year Treasury Constant Maturity Minus 2 Year Treasury Constant Maturity', 'DGS10': '10 Year Treasury Constant Maturity Rate', 'DCOILWTICO': 'Crude Oil West Texas Intermediate Cushing Oklahoma', 'SP500': 'S&P 500', 'DEXUSEU': 'US Euro Foreign Exchange Rate', 'DEXCHUS': 'China US Foreign Exchange Rate', 'DEXCAUS': 'Canadian to US Dollar Exchange Rate Daily', 'VIXCLS': 'CBOE Volatility Index: VIX', # this is a more irregular series 'T10YIE': '10 Year Breakeven Inflation Rate', 'USEPUINDXD': 'Economic Policy Uncertainty Index for United States', # also very irregular } series_desired = list(SeriesNameDict.keys()) fred_timeseries = pd.DataFrame( columns=['date', 'value', 'series_id', 'series_name']) for series in series_desired: data = fred.get_series(series) try: series_name = SeriesNameDict[series] except Exception: series_name = series data_df = pd.DataFrame({ 'date': data.index, 'value': data, 'series_id': series, 'series_name': series_name, }) data_df.reset_index(drop=True, inplace=True) fred_timeseries = pd.concat([fred_timeseries, data_df], axis=0, ignore_index=True) return fred_timeseries
def getCorporatesFred(self, trim_start, trim_end): fred = Fred(api_key=FRED_API_KEY) curr_trim_end = trim_start if (self.corporates.size != 0): self.trim_start = self.corporates['OIS'].index.min().date() curr_trim_end = self.corporates['OIS'].index.max().date() if trim_end <= curr_trim_end: self.trim_end = curr_trim_end return self.corporates self.trim_start = trim_start self.trim_end = trim_end self.OIS = OIS(trim_start=trim_start, trim_end=trim_end) self.datesAll = self.OIS.datesAll self.datesAll.columns = [x.upper() for x in self.datesAll.columns] self.datesAll.index = self.datesAll.DATE self.OISData = self.OIS.getOIS() for i in np.arange(len(self.OISData.columns)): freq = self.OISData.columns[i] self.tenors.append(self.myScheduler.extractDelay(freq=freq)) for rating in self.ratings.keys(): index = self.ratings[rating] try: corpSpreads = 1e-2 * (fred.get_series( index, observation_start=trim_start, observation_end=trim_end).to_frame()) corpSpreads.index = [x.date() for x in corpSpreads.index[:]] corpSpreads = pd.merge(left=self.datesAll, right=corpSpreads, left_index=True, right_index=True, how="left") corpSpreads = corpSpreads.fillna(method='ffill').fillna( method='bfill') corpSpreads = corpSpreads.drop("DATE", axis=1) self.corpSpreads[rating] = corpSpreads.T.fillna( method='ffill').fillna(method='bfill').T except Exception as e: print(e) print(index, " not found") self.corpSpreads = pd.Panel.from_dict(self.corpSpreads) self.corporates = {} self.OISData.drop('DATE', axis=1, inplace=True) ntenors = np.shape(self.OISData)[1] for rating in self.ratings: try: tiledCorps = np.tile(self.corpSpreads[rating][0], ntenors).reshape(np.shape(self.OISData)) self.corporates[rating] = pd.DataFrame( data=(tiledCorps + self.OISData.values), index=self.OISData.index, columns=self.OISData.columns) except: print("Error in addition of Corp Spreads") self.corporates['OIS'] = self.OISData self.corporates = pd.Panel(self.corporates) return self.corporates
def DailyRiskFreeRates(): fred = Fred(api_key='5e8edcc00dd0a9e40375540f32696316') TBills = fred.get_series('DGS10').to_frame().reset_index() TBills.columns = ['Date', 'RiskFree'] TBills.set_index('Date', inplace=True) TBills['RiskFree'] = TBills['RiskFree'].apply(lambda x: x / 100) TBills.to_csv(myPath + 'TBillRate.csv') return TBills
def extract(self): ''' extracts data directly apply FRED API ''' df = {} fred = Fred(api_key=self.options["api_key"]) for item in self.options["items"].keys(): try: if self.options["end_date"]: df[item] = fred.get_series( item, observation_start=self.options["start_date"], observation_end=self.options["end_date"]) else: df[item] = fred.get_series( item, observation_start=self.options["start_date"]) except ValueError as v: print("%s : %s " % (item, v)) continue self.df = pd.DataFrame(df)
class FredApiData(AbstractApiData): '''Fred data API class''' def __init__(self, api_code, start_date, series_name=None): super().__init__(api_code, start_date, series_name=series_name) with open('ignore_folder/fred_api_key.txt','r') as file: key = file.read().replace('\n', '') self.fred = Fred(api_key = key) self.df = pd.DataFrame(self.fred.get_series(self.api_code)).loc[start_date::] self.df.columns = [self.series_name]
def scrape_macro(series_id: dict): """ :param series_id: dictionary of FRED ID (key) and Series name (value) pairs :return: """ fred = Fred(api_key=config.FRED_API_KEY) return [ pd.Series(data=fred.get_series(series_key).dropna(), name=series_value) for series_key, series_value in series_id.items() ]
def retrieve_macro_data() -> pd.DataFrame(): """ Retrieves macro data using FRED api from 01 Jan 2006 to 28 Feb 2021. :return: DataFrame representing macro data. """ print('\n==========Collecting macro data==========') INDICATORS = { # 'DFEDTARL': 'Federal Funds Target Range - lower bound', # 'DFEDTARU': 'Federal Funds Target Range - upper bound', 'GDPC1': 'Real GDP', 'CPIAUCSL': 'Consumer Price Index (CPI)', 'UNRATE': 'Unemployment Rate', 'PAYEMS': 'Total NonFarm payrolls (Employment)', 'RRSFS': 'Real Retail and Food Services Sales', 'GFDEBTN': 'Federal Debt', 'VIXCLS': 'CBOE Volatility Index (VIX)', 'DFF': 'Effective Federal Funds Rate', } START_DATE = '1/1/2006' END_DATE = '28/2/2021' API_KEY = '2fb48248f0ce3781c82ffca58fecfb36' fred = Fred(api_key=API_KEY) full_data = pd.DataFrame() for indicator, name in INDICATORS.items(): print(f'Indicator: {indicator}, Name: {name}') data = fred.get_series(indicator, observation_start=START_DATE, observation_end=END_DATE).rename(indicator) df = pd.DataFrame(data) if full_data.empty: full_data = df else: full_data = pd.concat([full_data, df], axis=1) print("==========Done==========\n") # resample daily data to monthly full_data = full_data \ .resample('M') \ .last() \ .interpolate() \ .rename_axis('date') \ .reset_index() return full_data
def calc_equity_alloc(start_dt: str = '', end_dt: str = '') -> pd.Series: if not start_dt: start_dt = '1920-01-01' if not end_dt: end_dt = dt.today().strftime('%Y-%m-%d') fred = Fred(api_key=os.environ['TOKEN_FRED']) nonfin_biz_equity_liab = fred.get_series('NCBEILQ027S', observation_start=start_dt, observation_end=end_dt) nonfin_biz_credit_liab = fred.get_series('BCNSDODNS', observation_start=start_dt, observation_end=end_dt) household_nonprofit_credit_liab = fred.get_series( 'CMDEBT', observation_start=start_dt, observation_end=end_dt) fedgov_credit_liab = fred.get_series('FGSDODNS', observation_start=start_dt, observation_end=end_dt) localgov_ex_retirement_credit_liab = fred.get_series( 'SLGSDODNS', observation_start=start_dt, observation_end=end_dt) fin_biz_equity_liab = fred.get_series('FBCELLQ027S', observation_start=start_dt, observation_end=end_dt) restofworld_credit_liab = fred.get_series('DODFFSWCMI', observation_start=start_dt, observation_end=end_dt) # Divide nonfinancial and financial business equity reliabilities by all credit instrument liability in the economy equity_alloc = pd.Series( ( ( nonfin_biz_equity_liab + fin_biz_equity_liab ) / 1000 ) \ / ( ( ( nonfin_biz_equity_liab + fin_biz_equity_liab ) / 1000 ) + ( nonfin_biz_credit_liab + household_nonprofit_credit_liab + fedgov_credit_liab + localgov_ex_retirement_credit_liab + restofworld_credit_liab ) ) ) return make_qtrly(equity_alloc, 'first')
def findData(self): #Gets the series under Indusdrial Production & Capacity Utilization fred = Fred(api_key='e0a47670a791268b5b30cdf7cc217c4c') series = fred.search_by_category(3, order_by='title', filter=('frequency', 'Monthly')) #limit = 300 #Keeps only the Manufacturing series. Removes unnecessary title info series = series[series['title'].str.startswith( 'Capacity Utilization: Manufacturing')] series['title'] = series['title'].str.replace( 'Capacity Utilization: Manufacturing: Durable Goods: ', '') series['title'] = series['title'].str.replace( 'Capacity Utilization: Manufacturing: Non-Durable Goods: ', '') #Gets the NAICS codes and series IDs for each series naics_code = series['title'].str.extract( r'\= (.{3})') #Some have pt. before ). Need to fix naics_code = naics_code.rename(columns={0: 'NAICS Code'}) naics_code['NAICS Code'] = pd.to_numeric(naics_code['NAICS Code'], downcast='integer') series['title'] = series['title'].str.replace(r'\(([^)]+)\)', '') series_id = series.index.tolist() #Makes a DataFrame with NAICS code, series ID, and title dataset = pd.DataFrame(series.iloc[:, 3]) dataset = naics_code.merge(dataset, left_index=True, right_index=True) #Gets data for each series from 1997 through 2018 data = {} count = 0 for id in series.index: data[id] = fred.get_series(id, observation_start='1997-01-01', observation_end='2019-12-01') count += 1 if count == len(series) / 2: time.sleep(10) data = pd.DataFrame(data) #Adds data to dataset and organizes by NAICS data_id = data.transpose() dataset['Series ID'] = series_id dataset = dataset.merge(data_id, left_index=True, right_index=True) dataset.index = naics_code['NAICS Code'].tolist() dataset = dataset.drop('NAICS Code', 1) dataset = dataset.sort_index() dataset = dataset.rename(columns={'title': 'Industry'}) return dataset
def request(self, data): """ :param data: JSON Object to feed Quandl API request :return: JSON object with the response from he API """ current_app.logger.debug("Cache missed") try: fred = Fred(api_key=self.api_key) series = fred.get_series(data.get('series_id')) return series.last('1D').get(0) except Exception as e: raise e
class TestFred(unittest.TestCase): def setUp(self): self.fred = Fred() def testGetSeries(self): s = self.fred.get_series('SP500', observation_start='9/2/2014', observation_end='9/5/2014') self.assertEqual(s.ix['9/2/2014'], 2002.28) self.assertEqual(len(s), 4) info = self.fred.get_series_info('PAYEMS') self.assertEqual(info['title'], 'All Employees: Total nonfarm') # invalid series id self.assertRaises(ValueError, self.fred.get_series, 'invalid') self.assertRaises(ValueError, self.fred.get_series_info, 'invalid') # invalid parameter try: self.fred.get_series('SP500', observation_start='invalid-datetime-str') self.assertTrue(False, 'previous line should have thrown a ValueError') except ValueError: pass def testSearch(self): personal_income_series = self.fred.search_by_release( 175, limit=3, order_by='popularity', sort_order='desc') series_ids = ['PCPI06037', 'PCPI06075', 'PCPI34039'] for series_id in series_ids: self.assertTrue(series_id in personal_income_series.index) self.assertEqual( personal_income_series.ix[series_id, 'observation_start'], datetime(1969, 1, 1)) def tearDown(self): return
class FredStats: def __init__(self, api_key): self.fred = Fred(api_key=api_key) def retrieve_stats(self, series_names): # Retrieve the data from the FRED api frames = [] for series_name in series_names: data_series = self.fred.get_series(series_name).rename(series_name) frames.append(data_series) data = pd.concat(frames, axis=1) return data
def safe_heaven_demand(): spx = yf.Ticker("^GSPC") hist = spx.history(period = "1y") hist_close = hist['Close'] date_end = time.strftime('%Y.%m.%d',time.localtime(time.time()))#today date date_start = time.strftime('%Y.%m.%d',time.localtime(time.time()-2592000))#date of one month ago fred = Fred(api_key=Fred_API_key) treasury = fred.get_series('DGS10',date_start,date_end)# yield of treasuty Returns = np.diff(hist_close) / hist_close[:-1] # the return of SP 500 returns = Returns[-len(treasury):] spread = [returns[i] - treasury[i] for i in range(len(returns))] spread = [ i for i in spread if i == i]#remove nan value #index = (spread[-1] - min(spread))/(max(spread)-min(spread)) index = CalculateIndex(spread) return index
def get_series_data(series_id: str, start: str) -> pd.DataFrame: """Get Series data. [Source: FRED] Parameters ---------- series_id : str Series ID to get data from start : str Start date to get data from, format yyyy-mm-dd Returns ---------- pd.DataFrame Series data """ fredapi_client = Fred(cfg.API_FRED_KEY) return fredapi_client.get_series(series_id, start)
def get_data(self): if self.data_source == 'quandl': self.db_obj['data'] = pull_quandl(self.data_id) self.db_obj['data'].sort_index(inplace=True) self.db_obj['start_date'] = self.db_obj['data'].index.min() self.db_obj['end_date'] = self.db_obj['data'].index.max() self.db_obj['data'] = self.db_obj['data'].to_json(date_unit='s') if self.data_source == 'fred': freddy = Fred(api_key=FRED_API_KEY) self.db_obj['data'] = pd.DataFrame(freddy.get_series(self.data_id), columns=[self.symbol_name]) self.db_obj['data'].sort_index(inplace=True) self.db_obj['start_date'] = self.db_obj['data'].index.min() self.db_obj['end_date'] = self.db_obj['data'].index.max() self.db_obj['data'] = self.db_obj['data'].to_json(date_unit='s')
Spyder Editor This is a temporary script file. """ import pandas as pd from fredapi import Fred import matplotlib.pyplot as plt from matplotlib.dates import date2num , DateFormatter import datetime as dt fred = Fred(api_key='70da3ebf45f3904117b34c5147b93443') from IPython.core.pylabtools import figsize figsize(20, 5) oer=fred.get_series('CUSR0000SEHC01',observation_start='2002-01-01') #owner equivalent rent real_rent=fred.get_series('CUSR0000SEHA',observation_start='2002-01-01') pce=fred.get_series('PCE',observation_start='2002-01-01') #not so simple to plot datetime in matplot lib... typically requires a conversion to strings fig, ax= plt.subplots() ax.plot_date(oer.index,oer,fmt='-',color='green',label='owner equivalent rent') #this works well... now I want to add other series to this ax.plot_date(oer.index,real_rent,fmt='-',color='red',label='rent of primary residence') plt.title('owner equivalent rent') legend = ax.legend(loc='lower right', shadow=True) #frame = legend.get_frame() plt.show() '''playing around
import matplotlib.pyplot as plt fred = Fred(api_key = """key here""") """These are the keys that correspond to certain macro economic indicators and data points from the FRED Database""" series = ['SP500','GDP', 'A067RL1A156NBEA', 'CPIAUCSL', 'A191RL1Q225SBEA', 'DGS10', 'IC4WSA', 'UNRATE', 'DEXUSEU', 'BAMLH0A0HYM2', 'MEHOINUSA672N', 'M2V', 'GFDEGDQ188S', 'FEDFUNDS', 'NAPM', 'DCOILWTICO', 'M2', 'CIVPART', 'PSAVERT', 'USD3MTD156N', 'T10Y2Y', 'HOUST', 'DGS30', 'MORTG', 'DEXCHUS', 'BUSLOANS', 'UEMPMEAN', 'EXPGSCA', 'NETEXP', 'A067RP1A027NBEA', 'FYFSD'] #strips the data down to the title, frequency of reporting, units, and the latest values for t in series: data = fred.get_series(t) info = fred.get_series_info(t) print info['title'] print info['frequency'] print info['units'] print " " print "LATEST VALUES:" print data.tail() print " " #saves a PDF graph in the folder where code is stored for i in series: info=fred.get_series_info(i) title=info['title']
recessionend.append(t) for i in range(len(recessionend)): rect = plt.Rectangle((recessionstart[i], axes.get_ylim()[0]), (recessionend[i] - recessionstart[i]).days, axes.get_ylim()[1] - axes.get_ylim()[0], color = ".85") axes.add_patch(rect) return axes #################### Main Code #################### ## Insert your API key here fred = Fred(api_key = '7b90e5ff7af692a8c699383a7f344075') ## Pulling in recession dates for recession shading on graphs ## NOTE: The name is hard coded. Don't mess with it or the addrecessionshading function will break. Feel free to change the start date though if you need to recession = fred.get_series("USREC", observation_start = '1/1/2000') ## List where each element is a string with the FRED ID of the data series you want to pull series_names = ["GDPC1", # Real Gross Domestic Product, Billions of Chained 2009 Dollars, Seasonally Adjusted Annual Rate (GDPC1) "CPILFESL", # Consumer Price Index for All Urban Consumers: All Items Less Food & Energy, Index 1982-84=100, Seasonally Adjusted (CPILFESL) "PCEPILFE", # Personal Consumption Expenditures Excluding Food and Energy (Chain-Type Price Index), Index 2009=100, Seasonally Adjusted (PCEPILFE) "UNRATE", # Civilian Unemployment Rate, Percent, Seasonally Adjusted (UNRATE) "U6RATE", # Total unemployed, plus all marginally attached workers plus total employed part time for economic reasons, Percent, Seasonally Adjusted (U6RATE) ] ## Generate a dictionary of the series name a series containing the data pulled from FRED data = {} for l in series_names: data[l] = fred.get_series(l, observation_start = '1/1/2003') ## Settings for the GDP graph
from scipy.stats.stats import pearsonr ####### Import the FRED Data ####### api_key = 'f213b90a3ec4042de2259bceceb6ccfa' os.environ["FRED_API_KEY"] = api_key apiFred = Fred() dfFred = apiFred.search_by_release(112) dfStateUnEmp = dfFred[dfFred['title'].str.contains("Unemployment Rate in ")&~dfFred['title'].str.contains("Census")&dfFred['seasonal_adjustment_short'].str.match('SA')&~dfFred['title'].str.contains('DISCONTINUED')] srsStateUnIDs = dfStateUnEmp['id'][:] dictStateUN = {} for i in srsStateUnIDs: dictStateUN[i[:2]] = apiFred.get_series(i,observation_start='2007-01-01').mean() dfStateUN = pd.DataFrame(list(dictStateUN.items()),columns=['State', 'Unemployment']) ##Make the Map colors = ['#fff5eb', '#fee6ce', '#fdd0a2', '#fdae6b', '#fd8d3c', '#f16913', '#d94801', '#8c2d04'] chBin = Colorbin(dfStateUN['Unemployment'], colors, proportional=True, decimals=None) chBin.set_decimals(1) chBin.recalc(fenceposts=True) chBin.fenceposts colors_out = chBin.colors_out legend_colors = chBin.colors_in legend_labels = chBin.labels cg = Chorogrid('X:\\Documents\\Research\\Tools\\Python\\chorogrid-master\\chorogrid\\databases\\usa_states.csv', list(dfStateUN['State']), colors_out) cg.set_title('Average Unemployment by State (%)', font_dict={'font-size': 16})
# Step 1 : import data from FRED databases from fredapi import Fred fred = Fred(api_key="c3bc005d1a70992b2b3f3ef97d896c32") query = fred.search("france") query = query[(query["frequency_short"] == "Q")] query = query[ (query["observation_start"] <= "1970-01-01, 00:00:00") & (query["observation_end"] > "2013-01-01, 00:00:00") ] import pandas as pd series = list() for series_id in list(query["id"]): new_serie = fred.get_series(series_id, observation_start="1970-01-01, 00:00:00") series.append(new_serie) print(series_id + ": done") df = pd.concat(series, axis=1) df.columns = list(query["id"]) print(df.head()) print(query["title"]) df.to_csv("/Users/nicolassaleille/Dropbox/ofpr/data/fred/france_query_2.csv") # modele de prevision du PIB
class myFred(): savePath=open('ini/FredSavePath.txt').read() codeDict={ 'Real_GDP':'A191RL1Q225SBEA', 'Initial_Claims':'IC4WSA', 'Total_Nonfarm_Payrolls':'PAYEMS', 'Civilian_Unemployment_Rate':'UNRATE', 'Industrial_Production_Index': 'INDPRO' } speedList=['Initial_Claims','IC4WSA','Total_Nonfarm_Payrolls','PAYEMS', 'Industrial_Production_Index', 'INDPRO'] def __init__(self): self.fred=Fred(api_key='2c149bd5d320ce48476143f35aa2bf02') def getData(self,name=None,code=None,start=datetime(1999,1,1),end=None): if code is None: code=self.codeDict[name] data=self.fred.get_series(code,observation_start=start,observation_end=end) data=pandas.DataFrame({'Date':data.index.tolist(),'Value':data.tolist()}) date=data['Date'].tolist() t=[] for d in date: try: t.append(d.timestamp()) except Exception as e: t.append(None) data.insert(0,'time',t) return data.dropna() def saveMultiple(self,*namelist): if len(namelist)==0: namelist=self.codeDict.keys() con=sqlite3.connect('%s/%s.db' % (self.savePath,'FredData')) for k in self.codeDict.keys(): print(k) data=self.getData(name=k) if k in self.speedList: data=self.rsi(self.Speed(data)) self.save_sql(k,data.set_index('time').dropna(),con=con) con.close() def Speed(self,data): velocity=[0] acceleration=[0,0] value=data['Value'].tolist() former=value[0] for v in value[1:]: velocity.append(v-former) former=v former=velocity[1] for v in velocity[2:]: acceleration.append(v-former) former=v data.insert(2,'Velocity',velocity) data.insert(3,'Acceleration',acceleration) return data def rsi(self,data,column='Velocity',period=5): RSI=ind.RSI(data['time'],data[column],period) return data.merge(RSI,'outer','time') def save_sql(self,table,data,name=None,path=None,con=None,if_exists='replace'): close=False if path is None: path='%s/%s.db' % (self.savePath,name) if con is None: con=sqlite3.connect(path) close=True data.to_sql(table,con,if_exists=if_exists) if close: con.close()