def build_url_for_query(config, args):
"""
Creates a base url which requires string substitutions that depend on data source
:param config: dictionary, FBA yaml
:param args: dictionary, load parameters 'source' and 'year'
:return: base url used to load data
"""
# if there are url parameters defined in the yaml, then build a url, else use "base_url"
urlinfo = config["url"]
if urlinfo != 'None':
if 'url_params' in urlinfo:
params = ""
for k, v in urlinfo['url_params'].items():
params = params + '&' + k + "=" + str(v)
if 'url_params' in urlinfo:
build_url = "{0}{1}{2}".format(urlinfo['base_url'],
urlinfo['api_path'], params)
else:
build_url = "{0}".format(urlinfo['base_url'])
# substitute year from arguments and users api key into the url
if "__year__" in build_url:
build_url = build_url.replace("__year__", str(args["year"]))
if "__apiKey__" in build_url:
userAPIKey = load_api_key(config['api_name']) # (common.py fxn)
build_url = build_url.replace("__apiKey__", userAPIKey)
return build_url
def Census_pop_URL_helper(build_url, config, args):
urls = []
# get date code for july 1 population numbers
for k, v in config['datecodes'].items():
if str(args['year']) == str(k):
dc = str(v)
# the url for 2010 and earlier is different
url2000 = 'https://api.census.gov/data/2000/pep/int_population?get=POP,DATE_DESC&for=__aggLevel__:*&DATE_=12&key=__apiKey__'
# state fips required for county level 13-14
FIPS_2 = get_all_state_FIPS_2()['FIPS_2']
# drop puerto rico
FIPS_2 = FIPS_2[FIPS_2 != '72']
for c in config['agg_levels']:
# this timeframe requires state fips at the county level in the url
if '2010' < args['year'] < '2015' and c == 'county':
for b in FIPS_2:
url = build_url
url = url.replace("__aggLevel__", c)
url = url.replace("__DateCode__", dc)
url = url.replace("population?&", 'cty?')
url = url.replace("DATE_CODE", 'DATE_')
url = url.replace("&key", "&in=state:" + b + "&key")
urls.append(url)
else:
if args['year'] > '2010':
url = build_url
url = url.replace("__aggLevel__", c)
url = url.replace("__DateCode__", dc)
# url date variable different pre 2018
if args['year'] < '2018':
url = url.replace("DATE_CODE", 'DATE_')
# url for 2011 - 2014 slightly modified
if args['year'] < '2015' and c != 'county':
url = url.replace("population?&", 'natstprc?')
urls.append(url)
elif args['year'] == '2010':
url = url2000
url = url.replace("__aggLevel__", c)
url = url.replace("&in=state:__stateFIPS__", '')
if c == "us":
url = url.replace("*", "1")
userAPIKey = load_api_key(
config['api_name']) # (common.py fxn)
url = url.replace("__apiKey__", userAPIKey)
urls.append(url)
return urls
def assemble_urls_for_query(*, source, year, config):
"""
Calls on helper functions defined in source.py files to
replace parts of the url string
:param source: str, data source
:param year: str, year
:param config: dictionary, FBA yaml
:return: list, urls to call data from
"""
# if there are url parameters defined in the yaml,
# then build a url, else use "base_url"
urlinfo = config['url']
if urlinfo == 'None':
return [None]
if 'url_params' in urlinfo:
params = parse.urlencode(urlinfo['url_params'], safe='=&%',
quote_via=parse.quote)
build_url = urlinfo['base_url'] + urlinfo['api_path'] + params
else:
build_url = urlinfo['base_url']
# substitute year from arguments and users api key into the url
build_url = build_url.replace("__year__", str(year))
if "__apiKey__" in build_url:
userAPIKey = load_api_key(config['api_name']) # (common.py fxn)
build_url = build_url.replace("__apiKey__", userAPIKey)
if "url_replace_fxn" in config:
# dynamically import and call on function
urls = dynamically_import_fxn(
source, config["url_replace_fxn"])(build_url=build_url,
source=source,
year=year,
config=config)
return urls
else:
return [build_url]
def Census_pop_URL_helper(**kwargs):
"""
This helper function uses the "build_url" input from flowbyactivity.py, which
is a base url for data imports that requires parts of the url text string
to be replaced with info specific to the data year.
This function does not parse the data, only modifies the urls from which data is obtained.
:param kwargs: potential arguments include:
build_url: string, base url
config: dictionary, items in FBA method yaml
args: dictionary, arguments specified when running flowbyactivity.py
flowbyactivity.py ('year' and 'source')
:return: list, urls to call, concat, parse, format into Flow-By-Activity format
"""
# load the arguments necessary for function
build_url = kwargs['build_url']
config = kwargs['config']
args = kwargs['args']
urls = []
# get date code for july 1 population numbers
for k, v in config['datecodes'].items():
if str(args['year']) == str(k):
dc = str(v)
# the url for 2010 and earlier is different
url2000 = 'https://api.census.gov/data/2000/pep/int_population?get=' \
'POP,DATE_DESC&for=__aggLevel__:*&DATE_=12&key=__apiKey__'
# state fips required for county level 13-14
FIPS_2 = get_all_state_FIPS_2()['FIPS_2']
# drop puerto rico
FIPS_2 = FIPS_2[FIPS_2 != '72']
for c in config['agg_levels']:
# this timeframe requires state fips at the county level in the url
if '2010' < args['year'] < '2015' and c == 'county':
for b in FIPS_2:
url = build_url
url = url.replace("__aggLevel__", c)
url = url.replace("__DateCode__", dc)
url = url.replace("population?&", 'cty?')
url = url.replace("DATE_CODE", 'DATE_')
url = url.replace("&key", "&in=state:" + b + "&key")
urls.append(url)
else:
if args['year'] > '2010':
url = build_url
url = url.replace("__aggLevel__", c)
url = url.replace("__DateCode__", dc)
# url date variable different pre 2018
if args['year'] < '2018':
url = url.replace("DATE_CODE", 'DATE_')
# url for 2011 - 2014 slightly modified
if args['year'] < '2015' and c != 'county':
url = url.replace("population?&", 'natstprc?')
urls.append(url)
elif args['year'] == '2010':
url = url2000
url = url.replace("__aggLevel__", c)
url = url.replace("&in=state:__stateFIPS__", '')
if c == "us":
url = url.replace("*", "1")
userAPIKey = load_api_key(config['api_name']) # (common.py fxn)
url = url.replace("__apiKey__", userAPIKey)
urls.append(url)
return urls