def open_osm_overpass(requete, cache_filename, metropole=False): ok_filename = cache_filename + ".ok" try: if not (os.path.exists(cache_filename) and os.path.exists(ok_filename)): if os.path.exists(cache_filename): os.remove(cache_filename) if os.path.exists(ok_filename): os.remove(ok_filename) if metropole: # oapi-fr.openstreetmap.fr n'a que la métropole, pas l'outre mer overvass_server = "http://oapi-fr.openstreetmap.fr/oapi/interpreter?" else: overvass_server = "http://overpass-api.de/api/interpreter?" url = overvass_server + urllib.urlencode({'data':requete}) sys.stdout.write((urllib.unquote(url) + "\n").encode("utf-8")) sys.stdout.flush() write_stream_to_file(urllib2.urlopen(url), cache_filename) open(ok_filename, "a").close() return OsmParser().parse(cache_filename) except Exception as ex: if os.path.isfile(ok_filename): os.unlink(ok_filename) if metropole: # Essai avec l'autre serveur overpass (metropole=False) print "ERREUR:", ex print "Tentative depuis le serveur allemand:" return open_osm_overpass(requete, cache_filename, False) else: raise ex
def iter_download_pdfs(cadastreWebsite, code_departement, code_commune, ratio=PDF_DOWNLOAD_PIXELS_RATIO, mode=PDF_DOWNLOAD_SPLIT_MODE, nb=PDF_DOWNLOAD_SPLIT_NB, size=PDF_DOWNLOAD_SPLIT_SIZE, wait=PDF_DOWNALOD_WAIT_SECONDS,force_bbox=None): cadastreWebsite.set_departement(code_departement) cadastreWebsite.set_commune(code_commune) projection = cadastreWebsite.get_projection() bbox = cadastreWebsite.get_bbox() write_string_to_file(projection + ":%f,%f,%f,%f" % bbox, code_commune + ".bbox") if force_bbox: bbox = OSMToCadastreTransform(projection).transform_bbox( BoundingBox(*force_bbox)) if mode=="SIZE": liste = decoupage_bbox_cadastre_size(bbox, size, ratio) else: liste = decoupage_bbox_cadastre_nb(bbox, nb, ratio) for ((i,j), sous_bbox, (largeur,hauteur)) in liste: pdf_filename = code_commune + ("-%d-%d" % (i,j)) + ".pdf" bbox_filename = code_commune + ("-%d-%d" % (i,j)) + ".bbox" sous_bbox_str = projection + (":%f,%f,%f,%f" % sous_bbox) #sys.stdout.write((pdf_filename + " " + sous_bbox_str + "\n").encode("utf-8")) #sys.stdout.flush(); write_string_to_file(sous_bbox_str, bbox_filename) if not (os.path.exists(pdf_filename) and os.path.exists(pdf_filename + ".ok")): if os.path.exists(pdf_filename + ".ok"): os.remove(pdf_filename + ".ok") write_stream_to_file( cadastreWebsite.open_pdf(sous_bbox, largeur, hauteur), pdf_filename) open(pdf_filename + ".ok", 'a').close() time.sleep(wait) yield pdf_filename
def get_dict_fantoir(code_departement, code_commune): """ Retourne un dictionnaire qui mappe un nom normalizé du Fantoir (nature + libele de la voie) vers un tuple (string, boolean) représentant le CODE FANTOIR, et s'il s'agit d'un lieu dit non bâti (place=locality). """ code_insee = cadastre.code_insee(code_departement, code_commune) dict_fantoir = {} try: db_cursor = addr_fantoir_building.get_pgc().cursor() sql_query = ''' SELECT code_insee||id_voie||cle_rivoli, nature_voie||' '||libelle_voie, type_voie, ld_bati FROM fantoir_voie WHERE code_insee = \'''' + code_insee + '''\' AND caractere_annul NOT IN ('O','Q');''' db_cursor.execute(sql_query) for result in db_cursor: code_fantoir = result[0] nom_fantoir = ' '.join(result[1].replace('-',' ').split()) #lieu_dit_non_bati = (result[2] == '3') and (result[3] == '0') highway = result[2] in ['1', '4', '5'] dict_fantoir[normalize(nom_fantoir)] = (code_fantoir, highway) assert(len(dict_fantoir) > 0) return dict_fantoir except: # La connexion avec la base SQL a du échouer, on # charge les fichiers zip fantoir manuellement: filename = FANTOIR_ZIP ok_filename = filename + ".ok" if not (os.path.exists(filename) and os.path.exists(ok_filename)): sys.stdout.write("Téléchargement du fichier Fantoir " + FANTOIR_URL + "\n") if os.path.exists(filename): os.remove(filename) if os.path.exists(ok_filename): os.remove(ok_filename) write_stream_to_file(urllib2.urlopen(FANTOIR_URL), filename) open(ok_filename, "a").close() else: sys.stdout.write("Lecture du fichier FANTOIR.zip") sys.stdout.flush() num_commune = code_insee[2:5] debut = get_fantoir_code_departement(code_departement) + num_commune for line in ZipFile(filename, "r").open("FANTOIR.txt"): if line.startswith(debut): if line[108:109] != ' ': # C'est un unregistrement de voie if line[73] == ' ': # la voie n'est pas annulée assert(code_insee == line[0:2] + line[3:6]) id_voie = line[6:10] cle_rivoli = line[10] nature_voie = line[11:15].strip() libele_voie = line[15:41].strip() code_fantoir = code_insee + id_voie + cle_rivoli nom_fantoir = nature_voie + " " + libele_voie #lieu_dit_non_bati = line[108:110] == '30' highway = line[108:109] in ['1', '4', '5'] dict_fantoir[normalize(nom_fantoir)] = \ (code_fantoir, highway) return dict_fantoir
def iter_download_parcelles_info_pdf(cadastreWebsite, ids_parcelles): ids_parcelles.sort() for name, ids in named_chunks(ids_parcelles, MAX_PARCELLES_PAR_INFO_PDF): filename = cadastreWebsite.code_commune + "-parcelles-" + name + ".pdf" ok_filename = filename + ".ok" sys.stdout.write((filename + "\n").encode("utf-8")) sys.stdout.flush() if not (os.path.exists(filename) and os.path.exists(ok_filename)): if os.path.exists(ok_filename): os.remove(ok_filename) write_stream_to_file(cadastreWebsite.open_pdf_infos_parcelles(ids), filename) open(ok_filename, 'a').close() time.sleep(ATTENTE_EN_SECONDE_ENTRE_DOWNLOAD) yield filename
def iter_download_parcelles_info_pdf(cadastreWebsite, ids_parcelles): ids_parcelles.sort() for name, ids in named_chunks(ids_parcelles, MAX_PARCELLES_PAR_INFO_PDF): filename = cadastreWebsite.code_commune + "-parcelles-" + name + ".pdf" ok_filename = filename + ".ok" sys.stdout.write((filename + "\n").encode("utf-8")) sys.stdout.flush() if not (os.path.exists(filename) and os.path.exists(ok_filename)): if os.path.exists(ok_filename): os.remove(ok_filename) write_stream_to_file( cadastreWebsite.open_pdf_infos_parcelles(ids), filename) open(ok_filename, 'a').close() time.sleep(ATTENTE_EN_SECONDE_ENTRE_DOWNLOAD) yield filename
def get_dict_fantoir(code_departement, code_commune): """ Retourne un dictionnaire qui mappe un nom normalizé du Fantoir (nature + libele de la voie) vers le code Fantoir. """ code_insee = cadastre.code_insee(code_departement, code_commune) try: addr_fantoir_building.pgc = addr_fantoir_building.get_pgc() addr_fantoir_building.dicts.load_fantoir(code_insee) assert len(addr_fantoir_building.dicts.fantoir) > 0 return addr_fantoir_building.dicts.fantoir except: # La connexion avec la base SQL a du échouer, on # charge les fichiers zip fantoir manuellement: dict_fantoir = {} zip_filename = get_fantoir_zip_filename(code_departement) filename = os.path.join(FANTOIR_DIR, zip_filename) ok_filename = filename + ".ok" if not (os.path.exists(filename) and os.path.exists(ok_filename)): sys.stdout.write("Téléchargement du fichier Fantoir " + zip_filename + "\n") if os.path.exists(filename): os.remove(filename) if os.path.exists(ok_filename): os.remove(ok_filename) if not os.path.exists(FANTOIR_DIR): os.mkdir(FANTOIR_DIR) write_stream_to_file(urllib2.urlopen(FANTOIR_URL + zip_filename), filename) open(ok_filename, "a").close() else: sys.stdout.write("Lecture du fichier Fantoir " + zip_filename + "\n") sys.stdout.flush() txt_filename = get_fantoir_txt_filename(code_departement) num_commune = code_insee[2:5] for line in ZipFile(filename, "r").open(txt_filename): if line[3:6] == num_commune: if line[108:109] != " ": # C'est un unregistrement de voie if line[73] == " ": # la voie n'est pas annulée assert code_insee == line[0:2] + line[3:6] id_voie = line[6:10] cle_rivoli = line[10] nature_voie = line[11:15].strip() libele_voie = line[15:41].strip() code_fantoir = code_insee + id_voie + cle_rivoli nom_fantoir = nature_voie + " " + libele_voie dict_fantoir[normalize(nom_fantoir)] = code_fantoir return dict_fantoir
def open_osm_overpass(requete, cache_filename, metropole=False): ok_filename = cache_filename + ".ok" if not (os.path.exists(cache_filename) and os.path.exists(ok_filename)): if os.path.exists(cache_filename): os.remove(cache_filename) if os.path.exists(ok_filename): os.remove(ok_filename) if metropole: # oapi-fr.openstreetmap.fr n'a que la métropole, pas l'outre mer overvass_server = "http://oapi-fr.openstreetmap.fr/oapi/interpreter?" else: overvass_server = "http://overpass-api.de/api/interpreter?" url = overvass_server + urllib.urlencode({'data':requete}) sys.stdout.write((urllib.unquote(url) + "\n").encode("utf-8")) sys.stdout.flush() write_stream_to_file(urllib2.urlopen(url), cache_filename) open(ok_filename, "a").close() return OsmParser().parse(cache_filename)
def get_dict_fantoir(code_departement, code_commune): """ Retourne un dictionnaire qui mappe un nom normalizé du Fantoir (nature + libele de la voie) vers le code Fantoir. """ code_insee = cadastre.code_insee(code_departement, code_commune) try: addr_fantoir_building.pgc = addr_fantoir_building.get_pgc() addr_fantoir_building.dicts.load_fantoir(code_insee) assert(len(addr_fantoir_building.dicts.fantoir) > 0) return addr_fantoir_building.dicts.fantoir except: # La connexion avec la base SQL a du échouer, on # charge les fichiers zip fantoir manuellement: dict_fantoir = {} zip_filename = get_fantoir_zip_filename(code_departement) filename = os.path.join(FANTOIR_DIR, zip_filename) ok_filename = filename + ".ok" if not (os.path.exists(filename) and os.path.exists(ok_filename)): sys.stdout.write("Téléchargement du fichier Fantoir " + zip_filename + "\n") if os.path.exists(filename): os.remove(filename) if os.path.exists(ok_filename): os.remove(ok_filename) if not os.path.exists(FANTOIR_DIR): os.mkdir(FANTOIR_DIR) write_stream_to_file(urllib2.urlopen(FANTOIR_URL + zip_filename), filename) open(ok_filename, "a").close() else: sys.stdout.write("Lecture du fichier Fantoir " + zip_filename + "\n") sys.stdout.flush() txt_filename = get_fantoir_txt_filename(code_departement) num_commune = code_insee[2:5] for line in ZipFile(filename, "r").open(txt_filename): if line[3:6] == num_commune: if line[108:109] != ' ': # C'est un unregistrement de voie if line[73] == ' ': # la voie n'est pas annulée assert(code_insee == line[0:2] + line[3:6]) id_voie = line[6:10] cle_rivoli = line[10] nature_voie = line[11:15].strip() libele_voie = line[15:41].strip() code_fantoir = code_insee + id_voie + cle_rivoli nom_fantoir = nature_voie + " " + libele_voie dict_fantoir[normalize(nom_fantoir)] = code_fantoir return dict_fantoir
def iter_download_pdfs(cadastreWebsite, code_departement, code_commune, ratio=PDF_DOWNLOAD_PIXELS_RATIO, mode=PDF_DOWNLOAD_SPLIT_MODE, nb=PDF_DOWNLOAD_SPLIT_NB, size=PDF_DOWNLOAD_SPLIT_SIZE, wait=PDF_DOWNALOD_WAIT_SECONDS, force_bbox=None): cadastreWebsite.set_departement(code_departement) cadastreWebsite.set_commune(code_commune) projection = cadastreWebsite.get_projection() bbox = cadastreWebsite.get_bbox() write_string_to_file(projection + ":%f,%f,%f,%f" % bbox, code_commune + ".bbox") if force_bbox: bbox = OSMToCadastreTransform(projection).transform_bbox( BoundingBox(*force_bbox)) if mode == "SIZE": liste = decoupage_bbox_cadastre_size(bbox, size, ratio) else: liste = decoupage_bbox_cadastre_nb(bbox, nb, ratio) for ((i, j), sous_bbox, (largeur, hauteur)) in liste: pdf_filename = code_commune + ("-%d-%d" % (i, j)) + ".pdf" bbox_filename = code_commune + ("-%d-%d" % (i, j)) + ".bbox" sous_bbox_str = projection + (":%f,%f,%f,%f" % sous_bbox) #sys.stdout.write((pdf_filename + " " + sous_bbox_str + "\n").encode("utf-8")) #sys.stdout.flush(); write_string_to_file(sous_bbox_str, bbox_filename) if not (os.path.exists(pdf_filename) and os.path.exists(pdf_filename + ".ok")): if os.path.exists(pdf_filename + ".ok"): os.remove(pdf_filename + ".ok") write_stream_to_file( cadastreWebsite.open_pdf(sous_bbox, largeur, hauteur), pdf_filename) open(pdf_filename + ".ok", 'a').close() time.sleep(wait) yield pdf_filename