def get_atlas_data(atlas_name):
""" returns all atlas data """
atlas = wrap_atlas(atlas_name)
data = {"vector": [], "raster": [], "external": []}
for layer in atlas.get_layer_list(atlas_name):
if ":" in layer["layerName"]:
layer["layername"] = layer["layerName"].split(":")[-1]
else:
layer["layername"] = layer["layerName"]
layer["slug"] = layer["layerName"]
servers = [key for key, link in SERVERS.items() if link in layer["url"]]
if len(servers) > 0:
url_split = layer["url"].split("/")
layer["workspace"] = url_split[-2]
layer["geoserver"] = servers[0]
data["vector"].append(layer)
elif "https://demo.lizard.net/" in layer["url"]:
data["raster"].append(layer)
elif "lizard" not in layer["url"]:
data["external"].append(layer)
else:
print("Could not assign", layer["url"])
return data
def extract_atlas(atlas_name, wd, download, resolution=10):
""" Returns batch upload shapes for one geoserver """
os.chdir(wd)
vector_dir = mk_dir("extract_vector",path=wd)
raster_dir = mk_dir("extract_raster",path=wd)
external_dir = mk_dir("extract_external",path=wd)
data = get_atlas_data(atlas_name)
unique_data = unique(data)
vectors = unique_data["vector"]
rasters = unique_data["raster"]
if "external" in unique_data.keys():
externals = unique_data["external"]
else:
externals = None
log_time("info", "Raster directory:", raster_dir)
log_time("info", "Vector directory:", vector_dir)
log_time("info", "Exteral wms directory:", external_dir)
log_time("info", "Amount of vectors: {}".format(len(vectors)))
log_time("info", "Amount of rasters: {}".format(len(rasters)))
if externals:
log_time("info", "Amount of external wms: {}".format(len(externals)))
else:
log_time("info", "Found no external wms")
atlas = wrap_atlas(atlas_name)
clip_geom = atlas.get_boundaring_polygon(atlas_name, "boundary", write = False)
clip_geom_nl = atlas.get_boundaring_polygon(atlas_name, "boundary",
write=False, epsg=28992)
# extract vector data from their respective sources
extract_vector_succes, extract_vector_failures = extract_vectors(
vectors, vector_dir, atlas_name, clip_geom, download
)
extract_raster_succes, extract_raster_failures = extract_rasters(
rasters, raster_dir, atlas_name, False, resolution, clip_geom, clip_geom_nl
)
if externals:
extract_ext_succes, exteract_ext_failures = extract_external(
externals, external_dir
)
else:
exteract_ext_failures = None
return (
vectors,
rasters,
externals,
exteract_ext_failures,
extract_raster_failures,
extract_vector_failures,
)
def create_catalogue(inifile):
""" Returns batch upload shapes for one geoserver """
setting = settings_object(inifile)
# set logging
set_log_config(setting.ini_location)
sys.stdout = logger(setting.ini_location)
# setting.wd = "C:/Users/chris.kerklaan/Documents/temp_meta_only"
# setting.download = False
os.chdir(setting.wd)
(
vectors,
rasters,
externals,
extract_ext_failures,
extract_rast_failures,
extract_vect_failures,
) = extract_atlas(setting.atlas_name, setting.wd, setting.download,
setting.resolution)
if setting.extract_only:
return print("Finished extracting data")
atlas = wrap_atlas(setting.atlas_name)
clip_geom = atlas.get_boundaring_polygon(setting.atlas_name, "boundary")
upload_dir = mk_dir(path=os.getcwd(), folder_name="upload_vector")
ready_succes, ready_failure = upload_ready_vectors(
upload_dir,
clip_geom,
setting.organisatie,
dataset=setting.dataset,
)
# create wms layers
wmslayer_succes, wmslayer_failures = create_wmslayers(
upload_dir, setting, clip_geom.GetEnvelope())
wmslayer_succes, wmslayer_failures = create_wmslayers(
"extract_external", setting, clip_geom.GetEnvelope())
raster_clip_ids = get_clip_id(clip_geom)
raster_succes, raster_failures = create_atlasstores(
"extract_raster",
raster_clip_ids,
setting,
overwrite=True,
use_nens=setting.use_nens,
)
summary(
vectors,
extract_vect_failures,
wmslayer_failures,
ready_failure,
rasters,
extract_rast_failures,
raster_failures,
externals,
extract_ext_failures,
)