def download_tract_data(state_id, puma_id, output_dir, census_api_key, puma_tract_mappings, households_data, persons_data): '''Download tract data from the US Census' API. Initilize an allocator, capable of allocating PUMS households as best as possible based on marginal census (currently tract) data using a cvx-solver. Args: state_id: 2-digit state fips code puma_id: 5-digit puma code output_dir: dir to write outWriter the generated bayesian nets to census_api_key: key used to download data from the U.S. Census puma_tract_mappings: filepath to the puma-tract mappings households_data: pums households data frame persons_data: pums persons data frame Returns: An allocator described above. ''' marginal_path = os.path.join( output_dir, FILE_PATTERN.format(state_id, puma_id, 'marginals.csv')) try: # Already have marginals file marginals = Marginals.from_csv(marginal_path) except Exception: # Download marginal data from the Census API with builtins.open(puma_tract_mappings) as csv_file: csv_reader = csv.DictReader(csv_file) marginals = Marginals.from_census_data(csv_reader, census_api_key, state=state_id, pumas=puma_id) if len(marginals.data) <= 1: logging.exception( 'Couldn\'t fetch data from the census. Check your API key') raise CensusFetchException() else: logging.info( 'Writing outWriter marginal file for state: %s, puma: %s', state_id, puma_id) marginals.write(marginal_path) '''With the above marginal controls (tract data), the methods in allocation.py allocate discrete PUMS households to the subject PUMA.''' try: allocator = HouseholdAllocator.from_cleaned_data( marginals=marginals, households_data=households_data, persons_data=persons_data) except Exception as e: logging.exception('Error Allocating state: %s, puma: %s\n%s', state_id, puma_id, e) __builtin__.exit() return marginals, allocator
def _check_wv_cdm_version_current(self): self._set_wvcdm_current_ver_data() cdm_fn = self._get_wvcdm_filename() cdm_paths = self._get_wvcdm_paths(self.addon) for p in cdm_paths: path = os.path.join(p, cdm_fn) if os.path.isfile(path): md5 = hashlib.md5(builtins.open(path, 'rb').read()).hexdigest() for entry in self.wvcdm_download_data: if md5 == entry.get('md5'): return True return False
def _progress_download(self, url, download_path, display_filename=None): """Progress download Download file in Kodi with progress bar """ utils.log('Downloading {0}'.format(url)) try: res = requests.get(url, stream=True, verify=False) res.raise_for_status() except requests.exceptions.HTTPError: utils.dialog('Download failed', 'HTTP ' + str(res.status_code) + ' error') return False except Exception as exc: utils.dialog('Download failed', 'Exception was: {0}'.format(exc)) return False total_length = float(res.headers.get('content-length')) dp = xbmcgui.DialogProgress() if not display_filename: display_filename = download_path.split()[-1] dp.create("Downloading {0}".format(display_filename), "Downloading File: {0}".format(url)) with builtins.open(download_path, 'wb') as f: chunk_size = 1024 downloaded = 0 for chunk in res.iter_content(chunk_size=chunk_size): f.write(chunk) downloaded += len(chunk) percent = int(downloaded * 100 / total_length) if dp.iscanceled(): dp.close() res.close() dp.update(percent) utils.log('Download {0} bytes complete, saved in {1}'.format( int(total_length), download_path)) dp.close() return True