def get_split(self):
"""Get boundary file for splitting."""
self.split = None
if self.options.split:
fn = self.options.split
if not os.path.exists(fn):
if "." not in os.path.basename(fn):
fn += ".osm"
if not os.path.exists(fn) and fn == os.path.basename(fn):
fn = self.cat.get_path(fn)
if fn.endswith(".osm"):
fn += "|layername=multipolygons"
split = geo.BaseLayer(fn, "zoningsplit", "ogr")
if not split.isValid():
msg = "Can't open %s" % self.options.split
fn = self.options.split
if os.path.basename(fn) == fn and "." not in fn:
report.split_id = fn
fn = boundary.get_boundary(self.path, self.boundary_search_area, fn)
name = fn.replace(".osm|layername=multipolygons", "")
report.split_name = name.split("/")[-1].replace("_", " ")
split = geo.BaseLayer(fn, "zoningsplit", "ogr")
if not split.isValid():
raise CatIOError(msg)
else:
raise CatIOError(msg)
self.split = geo.PolygonLayer("MultiPolygon", "split", "memory")
q = lambda f, __: f.geometry().wkbType() == geo.types.WKBMultiPolygon
self.split.append(split, query=q)
if self.split.featureCount() == 0:
msg = _("'%s' does not include any polygon") % self.options.split
raise CatValueError(msg)
def get_metadata(self, md_path, zip_path=""):
"""Get the metadata of the source file."""
fo = self.get_file_object(md_path, zip_path)
try:
text = fo.read()
except IOError:
raise CatIOError(_("Could not read metadata from '%s'") % md_path)
finally:
fo.close()
root = etree.fromstring(text)
is_empty = len(root) == 0 or len(root[0]) == 0
namespace = {
"gco": "http://www.isotc211.org/2005/gco",
"gmd": "http://www.isotc211.org/2005/gmd",
}
if hasattr(root, "nsmap"):
namespace = root.nsmap
src_date = root.find("gmd:dateStamp/gco:Date", namespace)
if is_empty or src_date is None:
raise CatIOError(_("Could not read metadata from '%s'") % md_path)
self.src_date = src_date.text
gml_title = root.find(".//gmd:title/gco:CharacterString", namespace)
self.cat_mun = gml_title.text.split("-")[-1].split("(")[0].strip()
gml_code = root.find(".//gmd:code/gco:CharacterString", namespace)
self.crs_ref = int(gml_code.text.split("/")[-1])
def csv2dict(csv_path, a_dict=None, exists=False):
"""Read a dictionary from a csv file."""
a_dict = {} if a_dict is None else a_dict
msg = _("Failed to load CSV file '%s'") % os.path.basename(csv_path)
if os.path.exists(csv_path):
with open(csv_path) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=str(delimiter))
for row in csv_reader:
if len(row) < 2:
raise CatIOError(msg)
a_dict[row[0]] = row[1]
elif exists:
raise CatIOError(msg)
return a_dict
def read(self, layername, allow_empty=False, force_zip=False):
"""
Create a QGIS vector layer for a Cadastre layername.
Derive the GML filename from layername. Downloads the file if not is
present. First try to read the ZIP file, if fails try with the GML file.
Args:
layername (str): Short name of the Cadastre layer. Any of
'building', 'buildingpart', 'otherconstruction',
'cadastralparcel', 'cadastralzoning', 'address',
'thoroughfarename', 'postaldescriptor', 'adminunitname'
allow_empty (bool): If False (default), raise a exception for empty
layer, else returns None
force_zip (bool): Force to use ZIP file.
Returns:
QgsVectorLayer: Vector layer.
"""
(md_path, gml_path, zip_path, group) = self.get_layer_paths(layername)
url = config.prov_url[group].format(code=self.prov_code)
if not os.path.exists(zip_path) and (not os.path.exists(gml_path) or force_zip):
self.get_atom_file(url)
if layername == "cadastralparcel":
self.fix_encoding(gml_path, zip_path)
if layername == "address":
self.fix_amp(gml_path, zip_path)
self.get_metadata(md_path, zip_path)
if self.is_empty(gml_path, zip_path):
if not allow_empty:
raise CatIOError(_("The layer '%s' is empty") % gml_path)
else:
log.info(_("The layer '%s' is empty"), gml_path)
return None
fn = gml_path
if group == "AD":
fn += "|layername=" + layername
gml = geo.BaseLayer(fn, layername + ".gml", "ogr")
if not gml.isValid():
gml = self.get_gml_from_zip(gml_path, zip_path, group, layername)
if gml is None:
raise CatIOError(_("Failed to load layer '%s'") % gml_path)
crs = QgsCoordinateReferenceSystem.fromEpsgId(self.crs_ref)
if not crs.isValid():
raise CatIOError(_("Could not determine the CRS of '%s'") % gml_path)
gml.setCrs(crs)
log.info(_("Read %d features in '%s'"), gml.featureCount(), gml_path)
gml.source_date = self.src_date
return gml
def create_shp(name, crs, fields=QgsFields(), geom_type=WKBMultiPolygon):
writer = BaseLayer.get_writer(name, crs, fields, geom_type)
if writer.hasError() != QgsVectorFileWriter.NoError:
msg = _(
"Error when creating shapefile: '%s'") % writer.errorMessage()
raise CatIOError(msg)
return writer
def download(self, filename, log=False):
"""Download query results to filename."""
for i in range(len(api_servers)):
try:
if log:
log.debug(self.get_url(i))
download.wget(self.get_url(i), filename)
return
except IOError:
pass
raise CatIOError("Can't read from any Overpass server'")
def __init__(self, a_path):
"""
Construct a CDAU reader.
Args:
a_path (str): Directory where the source files are located.
"""
self.path = a_path
if not os.path.exists(a_path):
os.makedirs(a_path)
if not os.path.isdir(a_path):
raise CatIOError(_("Not a directory: '%s'") % a_path)
self.crs_ref = cdau_crs
self.src_date = None
def read(self):
fn = os.path.join(self.path, self.cbcn_fn)
if not os.path.exists(fn):
log.info(_("Downloading '%s'"), self.cbcn_fn)
download.wget(self.url, fn)
cbcn = BaseLayer(fn, "cbcn", "ogr")
if not cbcn.isValid():
raise CatIOError(_("Failed to load layer '%s'") % self.cbcn_fn)
cbcn.setProviderEncoding("ISO-8859-1")
log.info(_("Read %d features in '%s'"), cbcn.featureCount(),
self.cbcn_fn)
self.get_metadata()
cbcn.source_date = self.src_date
return cbcn
def export_layer(self, layer, filename, driver_name="GeoJSON", target_crs_id=None):
"""
Export a vector layer.
Args:
layer (QgsVectorLayer): Source layer.
filename (str): Output filename.
driver_name (str): name of OGR driver (or get it from filename).
target_crs_id (int): Defaults to source CRS.
"""
out_path = self.cat.get_path(filename)
if layer.export(out_path, driver_name, target_crs_id=target_crs_id):
log.info(_("Generated '%s'"), filename)
else:
raise CatIOError(_("Failed to write layer: '%s'") % filename)
def read(self, prov_code):
if prov_code not in list(andalucia.keys()):
msg = _("Province code '%s' is not valid") % prov_code
raise CatValueError(msg)
csv_fn = csv_name.format(andalucia[prov_code])
csv_path = os.path.join(self.path, csv_fn)
if not os.path.exists(csv_path):
log.info(_("Downloading '%s'"), csv_path)
url = cdau_url.format(csv_fn)
download.wget(url, csv_path)
csv = geo.BaseLayer(csv_path, csv_fn, "ogr")
if not csv.isValid():
raise CatIOError(_("Failed to load layer '%s'") % csv_path)
csv.setCrs(QgsCoordinateReferenceSystem.fromEpsgId(cdau_crs))
log.info(_("Read %d features in '%s'"), csv.featureCount(), csv_path)
self.get_metadata(csv_path.replace(".csv", ".txt"))
csv.source_date = self.src_date
return csv
def get_metadata(self, md_path):
if os.path.exists(md_path):
with open(md_path, "r") as fo:
self.src_date = fo.read()
else:
with download.get_response(meta_url) as response:
s = re.search(
r"fecha de referencia.*([0-9]{1,2}\sde\s.+\sde\s[0-9]{4})",
response.text,
)
try:
self.src_date = datetime.strptime(
s.group(1), "%d de %B de %Y").strftime("%Y-%m-%d")
except Exception:
raise CatIOError(
_("Could not read metadata from '%s'") % "CDAU")
with open(md_path, "w") as fo:
fo.write(self.src_date)
def get_metadata(self):
fn = os.path.join(self.path, self.cbcn_fn + ".txt")
if os.path.exists(fn):
with open(fn, "r") as fo:
self.src_date = fo.read()
else:
with download.get_response(self.meta_url) as response:
s = re.search(
r"Fecha publicación</th>[\n\r ]*<td>([\d/]+)",
response.text,
)
try:
self.src_date = datetime.strptime(
s.group(1), "%d/%m/%Y").strftime("%Y-%m-%d")
except Exception:
raise CatIOError(
_("Could not read metadata from '%s'") % "Carto BCN")
with open(fn, "w") as fo:
fo.write(self.src_date)
def get_address(self):
"""Read Address GML dataset."""
if self.cat.zip_code == "08900":
self.get_cbcn()
return
address_gml = self.cat.read("address")
report.address_date = address_gml.source_date
if address_gml.writer.fieldNameIndex("component_href") == -1:
address_gml = self.cat.read("address", force_zip=True)
if address_gml.writer.fieldNameIndex("component_href") == -1:
msg = (
_("Could not resolve joined tables for the '%s' layer")
% address_gml.name()
)
raise CatIOError(msg)
self.address = geo.AddressLayer(source_date=address_gml.source_date)
q = None
if self.split or self.options.parcel:
q = lambda f, kw: self.address.get_id(f) in kw["keys"] # NOQA: E731
self.boundary_bbox = self.parcel.bounding_box()
self.address.append(address_gml, query=q, keys=self.tasks.keys())
del address_gml
report.inp_address = self.address.featureCount()
report.inp_address_entrance = self.address.count("spec='Entrance'")
report.inp_address_parcel = self.address.count("spec='Parcel'")
self.address.remove_address_wo_building(self.building)
if report.inp_address == 0:
msg = _("No addresses data")
if not self.options.building:
raise CatValueError(msg)
log.info(msg)
return
postaldescriptor = self.cat.read("postaldescriptor")
thoroughfarename = self.cat.read("thoroughfarename")
self.address.join_field(postaldescriptor, "PD_id", "gml_id", ["postCode"])
self.address.join_field(thoroughfarename, "TN_id", "gml_id", ["text"], "TN_")
del postaldescriptor, thoroughfarename
report.inp_zip_codes = self.address.count(unique="postCode")
report.inp_street_names = self.address.count(unique="TN_text")
self.get_auxiliary_addresses()
self.export_layer(self.address, "address.geojson", target_crs_id=4326)
self.get_translations(self.address)
def __init__(self, a_path):
"""
Construct a CBCN reader.
Args:
a_path (str): Directory where the source files are located.
"""
self.path = a_path
if not os.path.exists(a_path):
os.makedirs(a_path)
if not os.path.isdir(a_path):
raise CatIOError(_("Not a directory: '%s'") % a_path)
self.cbcn_fn = "0501040100_Adreces.zip"
self.url = ("https://opendata-ajuntament.barcelona.cat/"
"data/dataset/6b5cfa7b-1d8d-45f0-990a-d1844d43ffd1/"
"resource/6bfe63d8-8c6c-4cde-aaa3-b7c48fa66e34/download")
self.meta_url = ("https://opendata-ajuntament.barcelona.cat/"
"data/es/dataset/taula-direle/"
"resource/6bfe63d8-8c6c-4cde-aaa3-b7c48fa66e34")
self.src_date = None
def __init__(self, a_path):
"""
Construct a cadastre reader.
Args:
a_path (str): Directory where the source files are located.
"""
self.path = a_path
m = re.match(r"^\d{5}$", os.path.split(a_path)[-1])
if not m:
msg = _("Last directory name must be a 5 digits ZIP code")
raise CatValueError(msg)
self.zip_code = m.group()
self.prov_code = self.zip_code[0:2]
if self.prov_code not in config.prov_codes:
msg = _("Province code '%s' is not valid") % self.prov_code
raise CatValueError(msg)
if not os.path.exists(a_path):
os.makedirs(a_path)
if not os.path.isdir(a_path):
raise CatIOError(_("Not a directory: '%s'") % a_path)
def raiseIOError(*args, **kwargs):
raise CatIOError("bartaz")
def raises_io(*args):
raise CatIOError()
def raises_io1(url, fn):
if url == api_servers[0]:
raise CatIOError()
