def test_fred_category_children(self):
fred.key('abc123')
fred.children()
expected = 'http://api.stlouisfed.org/fred/category/children'
params = {
'api_key': 'abc123',
'category_id': None,
'file_type': 'json'
}
self.get.assert_called_with(expected, params=params)
def analysis_categories_child(categories):
for categories_item in categories:
insert_sql("category", categories_item)
child = fred.children(categories_item["id"])['categories']
if len(child):
analysis_categories_child(child)
else:
analysis_series_child(categories_item["id"])
def test_fred_category_children(self):
fred.key('abc123')
fred.children()
expected = 'https://api.stlouisfed.org/fred/category/children'
params = {'api_key': 'abc123', 'category_id': None, 'file_type': 'json'}
self.get.assert_called_with(expected, params=params)
def crawl_fred(api_key:str, nodes_to_visit:List[int]=[0], sleep_time:int=60, rate_limit:int=100) -> None:
"""
Crawling the FRED dataset. Saving all time series ids and metadata.
"""
fred.key(api_key)
file_number = 0
tot_downloaded = 0
num_nodes_visited = 0
num_requests = 0
list_json:List[Dict] = []
num_files_written = 0
curr_dir = f"dir{tot_downloaded // cfg.source.files_per_folder :04d}/"
# initialize
category_names = {}
for node in nodes_to_visit:
node_children = fred.children(node)
for child in node_children["categories"]:
category_names[child["id"]] = {"name": child["name"], "parent_id": child["parent_id"]}
while nodes_to_visit:
curr_node = nodes_to_visit.pop()
#logger.info(f"Current node: {curr_node:>4}")
try:
children = fred.children(curr_node)
num_requests += 1
if children["categories"]:
for child in children["categories"]:
nodes_to_visit.append(child["id"])
category_names[child["id"]] = {"name": child["name"], "parent_id": child["parent_id"]}
seriess = fred.category_series(curr_node)["seriess"]
num_requests += 1
for ts in seriess:
id_meta = ts
id_meta["source"] = "FRED"
id_meta["node_id"] = curr_node
id_meta["category_name"] = category_names[curr_node]["name"]
id_meta["parent_id"] = category_names[curr_node]["parent_id"]
tot_downloaded += 1
list_json.append(id_meta)
if len(list_json) > cfg.source.samples_per_json:
filename = f"meta_{num_files_written:>06}.json"
if num_files_written % cfg.source.files_per_folder == 0:
curr_dir = f"dir{num_files_written // cfg.source.files_per_folder :04d}/"
os.makedirs(os.path.join(cfg.source.path.FRED.meta, curr_dir), exist_ok=True)
with open(os.path.join(*[cfg.source.path.FRED.meta, curr_dir, filename]), "w") as fp:
json.dump(list_json, fp, sort_keys=True, indent=4, separators=(",", ": "))
fp.close()
num_files_written += 1
list_json = []
num_nodes_visited += 1
if num_nodes_visited % 100 == 0:
logger.info(f"Visited {num_nodes_visited:>5} nodes and currently have {tot_downloaded:>6} time series ids saved")
fname = time.ctime().replace(" ", "-").replace(":","-")+"-nodes-to-visit.txt"
with open(os.path.join(cfg.source.path.FRED.meta, fname), "w") as f:
f.write("\n".join([str(node) for node in nodes_to_visit]))
f.close()
if num_requests > rate_limit:
time.sleep(sleep_time)
num_requests=0
except Exception as e:
logger.debug(e)
logger.debug(f"Current node {curr_node}")
logger.debug(f"{num_requests:>3} requests last minute")
import fred
# Save your FRED API key.
fred.key('63ef8588c3a78e956fb156c8a1603152')
x = fred.categories(23)
print x
# Interact with economic data categories.
x1 = fred.category()
#print x1
#
x2 = fred.categories(24)
#print x2
#
#fred.children(24)
x3 = fred.children(24)
#print x3
for k, v in x3.items():
print v[0]
print v[1]
y = fred.related(32073)
#print y
#
z = fred.category(series=True)
print z
#
#fred.category_series(123)
#
#
## Interact with economic data releases.