def download_tract_data(state_id, puma_id, output_dir, census_api_key,
puma_tract_mappings, households_data, persons_data):
'''Download tract data from the US Census' API.
Initilize an allocator, capable of allocating PUMS households as best as possible based on
marginal census (currently tract) data using a cvx-solver.
Args:
state_id: 2-digit state fips code
puma_id: 5-digit puma code
output_dir: dir to write outWriter the generated bayesian nets to
census_api_key: key used to download data from the U.S. Census
puma_tract_mappings: filepath to the puma-tract mappings
households_data: pums households data frame
persons_data: pums persons data frame
Returns:
An allocator described above.
'''
marginal_path = os.path.join(
output_dir, FILE_PATTERN.format(state_id, puma_id, 'marginals.csv'))
try: # Already have marginals file
marginals = Marginals.from_csv(marginal_path)
except Exception: # Download marginal data from the Census API
with builtins.open(puma_tract_mappings) as csv_file:
csv_reader = csv.DictReader(csv_file)
marginals = Marginals.from_census_data(csv_reader,
census_api_key,
state=state_id,
pumas=puma_id)
if len(marginals.data) <= 1:
logging.exception(
'Couldn\'t fetch data from the census. Check your API key')
raise CensusFetchException()
else:
logging.info(
'Writing outWriter marginal file for state: %s, puma: %s',
state_id, puma_id)
marginals.write(marginal_path)
'''With the above marginal controls (tract data), the methods in allocation.py
allocate discrete PUMS households to the subject PUMA.'''
try:
allocator = HouseholdAllocator.from_cleaned_data(
marginals=marginals,
households_data=households_data,
persons_data=persons_data)
except Exception as e:
logging.exception('Error Allocating state: %s, puma: %s\n%s', state_id,
puma_id, e)
__builtin__.exit()
return marginals, allocator
def _check_wv_cdm_version_current(self):
self._set_wvcdm_current_ver_data()
cdm_fn = self._get_wvcdm_filename()
cdm_paths = self._get_wvcdm_paths(self.addon)
for p in cdm_paths:
path = os.path.join(p, cdm_fn)
if os.path.isfile(path):
md5 = hashlib.md5(builtins.open(path, 'rb').read()).hexdigest()
for entry in self.wvcdm_download_data:
if md5 == entry.get('md5'):
return True
return False
def _progress_download(self, url, download_path, display_filename=None):
"""Progress download
Download file in Kodi with progress bar
"""
utils.log('Downloading {0}'.format(url))
try:
res = requests.get(url, stream=True, verify=False)
res.raise_for_status()
except requests.exceptions.HTTPError:
utils.dialog('Download failed',
'HTTP ' + str(res.status_code) + ' error')
return False
except Exception as exc:
utils.dialog('Download failed', 'Exception was: {0}'.format(exc))
return False
total_length = float(res.headers.get('content-length'))
dp = xbmcgui.DialogProgress()
if not display_filename:
display_filename = download_path.split()[-1]
dp.create("Downloading {0}".format(display_filename),
"Downloading File: {0}".format(url))
with builtins.open(download_path, 'wb') as f:
chunk_size = 1024
downloaded = 0
for chunk in res.iter_content(chunk_size=chunk_size):
f.write(chunk)
downloaded += len(chunk)
percent = int(downloaded * 100 / total_length)
if dp.iscanceled():
dp.close()
res.close()
dp.update(percent)
utils.log('Download {0} bytes complete, saved in {1}'.format(
int(total_length), download_path))
dp.close()
return True