def collect_references_files(): """Retrieve locally CSV files in use for dynamic territories' resources.""" REFERENCES_PATH = references.root if not os.path.exists(REFERENCES_PATH): os.makedirs(REFERENCES_PATH) if current_app.config.get('ACTIVATE_TERRITORIES'): # TERRITORY_DATASETS is a dict of dicts and we want all values # that are resource based to collect related files. territory_classes = [ territory_class for territory_dict in TERRITORY_DATASETS.values() for territory_class in territory_dict.values() if issubclass(territory_class, ResourceBasedTerritoryDataset) ] for territory_class in territory_classes: dataset = Dataset.objects.get(id=territory_class.dataset_id) for resource in dataset.resources: if str(resource.id) != str(territory_class.resource_id): continue filename = resource.url.split('/')[-1] reference_path = references.path(filename) log.info('Found reference: %s', reference_path) if os.path.exists(reference_path): log.info('Reference already downloaded') continue log.info('Downloading from: %s', resource.url) with codecs.open(reference_path, 'w', encoding='utf8') as fd: r = requests.get(resource.url, stream=True) for chunk in r.iter_content(chunk_size=1024): fd.write(chunk.decode('latin-1')) # TODO: detect? log.info('Done')
def render_territory(territory): if not current_app.config.get('ACTIVATE_TERRITORIES'): return abort(404) from udata.models import TERRITORY_DATASETS territory_dataset_classes = sorted( TERRITORY_DATASETS.values(), key=lambda a: a.order) territory_datasets = [ territory_dataset_class(territory) for territory_dataset_class in territory_dataset_classes ] datasets = list(Dataset.objects.visible().filter(spatial__zones=territory)) context = { 'territory': territory, 'territory_datasets': territory_datasets, 'datasets': datasets, } return theme.render('territories/territory.html', **context)
def render_territory(territory): if not current_app.config.get('ACTIVATE_TERRITORIES'): return abort(404) territory_dataset_classes = sorted(TERRITORY_DATASETS.values(), key=lambda a: a.order) territory_datasets = [ territory_dataset_class(territory) for territory_dataset_class in territory_dataset_classes ] # Retrieve all datasets then split between those optionaly owned # by an org for that zone and others. We need to know if the current # user has datasets for that zone in order to display a custom # message to ease the conversion. datasets = Dataset.objects.visible().filter(spatial__zones=territory) town_datasets = [] other_datasets = [] editable_datasets = [] if datasets: for dataset in datasets: if (dataset.organization and territory.id == dataset.organization.zone): town_datasets.append(dataset) else: other_datasets.append(dataset) editable_datasets.append(current_user.is_authenticated and DatasetEditPermission(dataset).can()) context = { 'territory': territory, 'territory_datasets': territory_datasets, 'other_datasets': other_datasets, 'has_pertinent_datasets': any(editable_datasets), 'town_datasets': town_datasets } return theme.render('territories/territory.html', **context)
def collect_references_files(): """Retrieve locally CSV files in use for dynamic territories' resources.""" REFERENCES_PATH = references.root if not os.path.exists(REFERENCES_PATH): os.makedirs(REFERENCES_PATH) if current_app.config.get('ACTIVATE_TERRITORIES'): from udata.models import TERRITORY_DATASETS for territory_class in TERRITORY_DATASETS.values(): if not issubclass(territory_class, ResourceBasedTerritoryDataset): continue dataset = Dataset.objects.get(id=territory_class.dataset_id) for resource in dataset.resources: if resource.id == territory_class.resource_id: break filename = resource.url.split('/')[-1] reference_path = references.path(filename) if os.path.exists(reference_path): continue with codecs.open(reference_path, 'w', encoding='utf8') as fd: r = requests.get(resource.url, stream=True) for chunk in r.iter_content(chunk_size=1024): fd.write(chunk.decode('latin-1')) # TODO: detect? log.info('Done')
temporal_coverage = {'start': 2015} class BanODBLTownDataset(TerritoryDataset): order = 10 id = 'ban_odbl' title = 'Adresses' # Etalab. organization_id = '534fff75a3a7292c64a77de4' url_template = ('http://bano.openstreetmap.fr/BAN_odbl/communes/' 'BAN_odbl_{code}.csv') description = ''' Données de la [Base Adresse Nationale ](/datasets/ban-base-adresse-nationale/) sur le périmètre de la commune. '''.strip() TERRITORY_DATASETS.update({ 'zonages': ZonagesTownDataset, 'comptes': ComptesTownDataset, 'logement': LogementTownDataset, 'emploi_population': EmploiPopulationTownDataset, 'emploi_chiffres': EmploiChiffresTownDataset, 'population': PopulationTownDataset, 'population_chiffres': PopulationChiffresTownDataset, 'population_diplomes': PopulationDiplomesTownDataset, 'elections_regionales_2015_2': ElectionsRegionales2015Tour2TownDataset, 'ban_odbl': BanODBLTownDataset })
class BanODBLTownDataset(TerritoryDataset): order = 10 id = "ban_odbl" title = "Adresses" # Etalab. organization_id = "534fff75a3a7292c64a77de4" url_template = "http://bano.openstreetmap.fr/BAN_odbl/communes/" "BAN_odbl_{code}.csv" description = """ Données de la [Base Adresse Nationale ](/datasets/ban-base-adresse-nationale/) sur le périmètre de la commune. """.strip() TERRITORY_DATASETS.update( { "zonages": ZonagesTownDataset, "comptes": ComptesTownDataset, "logement": LogementTownDataset, "emploi_population": EmploiPopulationTownDataset, "emploi_chiffres": EmploiChiffresTownDataset, "population": PopulationTownDataset, "population_chiffres": PopulationChiffresTownDataset, "population_diplomes": PopulationDiplomesTownDataset, "elections_regionales_2015_2": ElectionsRegionales2015Tour2TownDataset, "ban_odbl": BanODBLTownDataset, } )