def postgis(conn, config, out_dir, doexec=True):
"""Export as PostgreSQL backup file
For example the backup can be restored as:
.. code-block:: sh
createdb <db>
psql -d <db> -c 'create extension postgis;'
pg_restore --no-owner --no-privileges -h <host> -U <user> -d <db> -w bagactueel_schema.backup
pg_restore --no-owner --no-privileges -j 2 --clean -h <host> -U <user> -d <db> -w <bag3d backup>.backup
Parameters
----------
conn : :py:class:`bag3d.config.db.db`
Open connection
config : dict
Configuration
out_dir : str
Path to the output directory. The directory 'csv' will be created if
doesn't exist.
"""
schema = config['output']['production']['schema']
bag3d = config['output']['production']['bag3d_table']
date = datetime.date.today().isoformat()
postgis_dir = os.path.join(out_dir, "postgis")
os.makedirs(postgis_dir, exist_ok=True)
# PostGIS schema (required because of the pandstatus custom data type)
f = os.path.join(postgis_dir, "bagactueel_schema.backup")
command = [
"pg_dump", "--host", conn.host, "--port", conn.port, "--username",
conn.user, "--no-password", "--format", "custom", "--no-owner",
"--compress", "7", "--encoding", "UTF8", "--verbose", "--schema-only",
"--schema", "bagactueel", "--file", f, conn.dbname
]
bag.run_subprocess(command, shell=True, doexec=doexec)
compute_md5(f, postgis_dir)
# The 3D BAG (building heights + footprint geom)s
f = os.path.join(postgis_dir, 'bag3d_{d}.backup'.format(d=date))
tbl = '%s.%s' % (schema, bag3d)
command = [
"pg_dump", "--host", conn.host, "--port", conn.port, "--username",
conn.user, "--no-password", "--format", "custom", "--no-owner",
"--compress", "7", "--encoding", "UTF8", "--verbose", "--file", f,
"--table", tbl, conn.dbname
]
logger.info("Exporting PostGIS backup")
bag.run_subprocess(command, shell=True, doexec=doexec)
compute_md5(f, postgis_dir)
def gpkg(conn, config, out_dir, doexec=True):
"""Export into GeoPackage
Parameters
----------
conn : :py:class:`bag3d.config.db.db`
Open connection
config : dict
Configuration
out_dir : str
Path to the output directory. The directory 'csv' will be created if
doesn't exist.
"""
schema = config['output']['production']['schema']
bag3d = config['output']['production']['bag3d_table']
date = datetime.date.today().isoformat()
x = "bag3d_{d}.gpkg".format(d=date)
d = os.path.join(out_dir, "gpkg")
os.makedirs(d, exist_ok=True)
f = os.path.join(d, x)
if conn.password:
dns = "PG:'dbname={db} host={h} port={p} user={u} password={pw} \
schemas={schema} tables={bag3d}'".format(db=conn.dbname,
h=conn.host,
p=conn.port,
pw=conn.password,
u=conn.user,
schema=schema,
bag3d=bag3d)
else:
dns = "PG:'dbname={db} host={h} port={p} user={u} schemas={schema} \
tables={bag3d}'".format(db=conn.dbname,
h=conn.host,
p=conn.port,
pw=conn.password,
u=conn.user,
schema=schema,
bag3d=bag3d)
command = ["ogr2ogr", "-f", "GPKG", f, dns]
logger.info("Exporting GPKG")
bag.run_subprocess(command, shell=True, doexec=doexec)
compute_md5(f, d)
def downloader(tile_list, url, dir_out, doexec=True):
is_wget = which("wget")
is_unzip = which("unzip")
is_rm = which("rm")
if is_wget is None:
logger.error("'wget' not found, exiting")
exit(1)
if is_unzip is None:
logger.error("'unzip' not found, exiting")
exit(1)
if is_rm is None:
logger.error("'rm' not found, exiting")
exit(1)
try:
fout = os.path.join(dir_out, 'tile')
for tile in tile_list:
command = [
'wget', '-nc', '-P', dir_out,
url.format(tile), '-O', fout
]
bag.run_subprocess(command, shell=False, doexec=doexec)
command = ['unzip', '-o', fout, '-d', dir_out]
bag.run_subprocess(command, shell=False, doexec=doexec)
command = ['rm', fout]
bag.run_subprocess(command, shell=False, doexec=doexec)
logger.info("Downloaded %s tiles", len(tile_list))
except Exception as e:
logger.exception(e)
def downloader(tile_list, url, dir_out, doexec=True):
try:
fout = os.path.join(dir_out, 'tile')
for tile in tile_list:
command = [
'wget', '-nc', '-P', dir_out,
url.format(tile), '-O', fout
]
bag.run_subprocess(command, shell=False, doexec=doexec)
command = ['unzip', '-o', fout, '-d', dir_out]
bag.run_subprocess(command, shell=False, doexec=doexec)
command = ['rm', fout]
bag.run_subprocess(command, shell=False, doexec=doexec)
logger.info("Downloaded %s tiles", len(tile_list))
except Exception as e:
logger.exception(e)
def call_3dfier(db,
tile,
schema_tiles,
table_index_pc,
fields_index_pc,
idx_identical,
table_index_footprint,
fields_index_footprint,
uniqueid,
extent_ewkb,
clip_prefix,
prefix_tile_footprint,
yml_dir,
tile_out,
output_format,
output_dir,
path_3dfier,
thread,
pc_file_index,
tile_group,
doexec=True):
"""Call 3dfier with the YAML config created by yamlr().
Note
----
For the rest of the parameters see batch3dfier_config.yml.
Parameters
----------
db : :py:class:`bag3d.config.db.db`
Open connection
tile : str
Name of of the 2D tile.
schema_tiles : str
Schema of the footprint tiles.
thread : str
Name/ID of the active thread.
extent_ewkb : str
EWKB representation of 'extent' in batch3dfier_config.yml.
clip_prefix : str
Prefix for naming the clipped/united views. This value shouldn't be a substring of the pointcloud file names.
prefix_tile_footprint : str or None
Prefix prepended to the footprint tile view names. If None, the views are named as
the values in fields_index_fooptrint['unit_name'].
Returns
-------
dict
tile_skipped : str
The tiles that are skipped because no corresponding pointcloud file
was found in 'dataset_dir' (YAML)
out_path : str
Output path of 3dfier
"""
# perf = report_procs()
# if perf:
# logger_perf.debug("%s - %s - %s" % (tile_group, tile, perf))
start = time.process_time()
tiles = find_pc_tiles(db,
table_index_pc,
fields_index_pc,
idx_identical,
table_index_footprint,
fields_index_footprint,
extent_ewkb,
tile_footprint=tile,
prefix_tile_footprint=prefix_tile_footprint)
p = [pc_file_index[tile] for tile in pc_file_index.keys() & tiles.keys()]
ahn_version = set([tiles[v] for v in pc_file_index.keys() & tiles.keys()])
pc_path = list(chain.from_iterable(p))
# prepare output file name
if not tile_out:
tile_out = tile.replace(clip_prefix, '', 1)
if pc_path:
# Needs a YAML per thread so one doesn't overwrite it while the other
# uses it
yml_name = tile + ".yml"
yml_path = os.path.join(yml_dir, yml_name)
config = yamlr(dbname=db.dbname,
host=db.host,
port=db.port,
user=db.user,
pw=db.password,
schema_tiles=schema_tiles,
bag_tile=tile,
pc_path=pc_path,
uniqueid=uniqueid,
ahn_version=ahn_version)
# Write temporary config file
logger.debug(config)
try:
with open(yml_path, "w") as text_file:
text_file.write(config)
except BaseException as e:
logger.exception("Error: cannot write %s", yml_path)
# Prep output file name
if "obj" in output_format.lower():
o = tile_out + ".obj"
output_path = os.path.join(output_dir, o)
elif "csv" in output_format.lower():
o = tile_out + ".csv"
output_path = os.path.join(output_dir, o)
else:
output_path = os.path.join(output_dir, tile_out)
# Run 3dfier
command = [
path_3dfier, yml_path, "--stat_RMSE", "--CSV-BUILDINGS-MULTIPLE",
output_path
]
try:
logger.debug(" ".join(command))
success = bag.run_subprocess(command,
shell=True,
doexec=doexec,
monitor=True,
tile_id=tile)
if success:
tile_skipped = None
else:
tile_skipped = tile
output_path = None
try:
remove(yml_path)
except Exception as e:
logger.error(e)
except BaseException as e:
logger.exception("Cannot run 3dfier on tile %s", tile)
tile_skipped = tile
output_path = None
else:
logger.debug("Pointcloud file(s) %s not available. Skipping tile.",
str(tiles.keys()))
tile_skipped = tile
output_path = None
# end = time.process_time()
# proc_time = (end - start) / 60
# logger_perf.debug("%s - %s - process_time: %s" % (tile_group, tile, proc_time))
return {'tile_skipped': tile_skipped, 'out_path': output_path}
def compute_md5(file, d):
"""Compute the md5sum of a file"""
filename = os.path.splitext(os.path.basename(file))
md5_file = filename[0] + ".md5"
cmd = ["md5sum", "--tag", file, ">", os.path.join(d, md5_file)]
bag.run_subprocess(cmd, shell=True)