def __apply_metric_on_all_images(self, metric_function):
if self.__parallel:
pool = Pool()
return pool.starmap(metric_function,
zip(self.true_labels, self.inferred_labels))
else:
return [
metric_function(x, y)
for (x, y) in zip(self.true_labels, self.inferred_labels)
]
def single_country(country, regionalized=False, create_poly_files=False):
"""
Clip a country from the planet osm file and save to individual osm.pbf files
This function has the option to extract individual regions
Arguments:
*country* : The country for which we want extract the data.
Keyword Arguments:
*regionalized* : Default is **False**. Set to **True** will parallelize the extraction over all regions within a country.
*create_poly_files* : Default is **False**. Set to **True** will create new .poly files.
"""
# set data path
data_path = load_config()['paths']['data']
# path to planet file
planet_path = os.path.join(data_path, 'planet_osm',
'planet-latest.osm.pbf')
# global shapefile path
if regionalized == True:
world_path = os.path.join(data_path, 'input_data',
'global_regions.shp')
else:
world_path = os.path.join(data_path, 'input_data',
'global_countries.shp')
# create poly files for all countries
if create_poly_files == True:
poly_files(data_path,
world_path,
save_shapefile=False,
regionalized=regionalized)
if not os.path.exists(os.path.join(data_path, 'country_poly_files')):
os.makedirs(os.path.join(data_path, 'country_poly_files'))
if not os.path.exists(os.path.join(data_path, 'country_osm')):
os.makedirs(os.path.join(data_path, 'country_osm'))
ctry_poly = os.path.join(data_path, 'country_poly_files',
'{}.poly'.format(country))
ctry_pbf = os.path.join(data_path, 'country_osm',
'{}.osm.pbf'.format(country))
if regionalized == False:
clip_osm(data_path, planet_path, ctry_poly, ctry_pbf)
elif regionalized == True:
if (os.path.exists(ctry_pbf) is not True):
clip_osm(data_path, planet_path, ctry_poly, ctry_pbf)
if not os.path.exists(os.path.join(data_path, 'regional_poly_files')):
os.makedirs(os.path.join(data_path, 'regional_poly_files'))
if not os.path.exists(os.path.join(data_path, 'region_osm_admin1')):
os.makedirs(os.path.join(data_path, 'region_osm_admin1'))
get_poly_files = [
x
for x in os.listdir(os.path.join(data_path, 'regional_poly_files'))
if x.startswith(country)
]
polyPaths = [
os.path.join(data_path, 'regional_poly_files', x)
for x in get_poly_files
]
area_pbfs = [
os.path.join(data_path, 'region_osm_admin1',
x.split('.')[0] + '.osm.pbf') for x in get_poly_files
]
data_paths = [data_path] * len(polyPaths)
planet_paths = [ctry_pbf] * len(polyPaths)
# and run all regions parallel to each other
pool = Pool(cpu_count() - 1)
pool.starmap(clip_osm,
zip(data_paths, planet_paths, polyPaths, area_pbfs))
def all_countries(subset=[], regionalized=False, reversed_order=False):
"""
Clip all countries from the planet osm file and save them to individual osm.pbf files
Optional Arguments:
*subset* : allow for a pre-defined subset of countries. REquires ISO3 codes. Will run all countries if left empty.
*regionalized* : Default is **False**. Set to **True** if you want to have the regions of a country as well.
*reversed_order* : Default is **False**. Set to **True** to work backwards for a second process of the same country set to prevent overlapping calculations.
Returns:
clipped osm.pbf files for the defined set of countries (either the whole world by default or the specified subset)
"""
# set data path
data_path = load_config()['paths']['data']
# path to planet file
planet_path = os.path.join(data_path, 'planet_osm',
'planet-latest.osm.pbf')
# global shapefile path
if regionalized == True:
world_path = os.path.join(data_path, 'input_data',
'global_regions.shp')
else:
world_path = os.path.join(data_path, 'input_data',
'global_countries.shp')
# create poly files for all countries
poly_files(data_path,
world_path,
save_shapefile=False,
regionalized=regionalized)
# prepare lists for multiprocessing
if not os.path.exists(os.path.join(data_path, 'country_poly_files')):
os.makedirs(os.path.join(data_path, 'country_poly_files'))
if not os.path.exists(os.path.join(data_path, 'country_osm')):
os.makedirs(os.path.join(data_path, 'country_osm'))
if regionalized == False:
get_poly_files = os.listdir(
os.path.join(data_path, 'country_poly_files'))
if len(subset) > 0:
polyPaths = [
os.path.join(data_path, 'country_poly_files', x)
for x in get_poly_files if x[:3] in subset
]
area_pbfs = [
os.path.join(data_path, 'region_osm_admin1',
x.split('.')[0] + '.osm.pbf')
for x in get_poly_files if x[:3] in subset
]
else:
polyPaths = [
os.path.join(data_path, 'country_poly_files', x)
for x in get_poly_files
]
area_pbfs = [
os.path.join(data_path, 'region_osm_admin1',
x.split('.')[0] + '.osm.pbf')
for x in get_poly_files
]
big_osm_paths = [planet_path] * len(polyPaths)
elif regionalized == True:
if not os.path.exists(os.path.join(data_path, 'regional_poly_files')):
os.makedirs(os.path.join(data_path, 'regional_poly_files'))
if not os.path.exists(os.path.join(data_path, 'region_osm')):
os.makedirs(os.path.join(data_path, 'region_osm_admin1'))
get_poly_files = os.listdir(
os.path.join(data_path, 'regional_poly_files'))
if len(subset) > 0:
polyPaths = [
os.path.join(data_path, 'regional_poly_files', x)
for x in get_poly_files if x[:3] in subset
]
area_pbfs = [
os.path.join(data_path, 'region_osm_admin1',
x.split('.')[0] + '.osm.pbf')
for x in get_poly_files if x[:3] in subset
]
big_osm_paths = [
os.path.join(data_path, 'country_osm', x[:3] + '.osm.pbf')
for x in get_poly_files if x[:3] in subset
]
else:
polyPaths = [
os.path.join(data_path, 'regional_poly_files', x)
for x in get_poly_files
]
area_pbfs = [
os.path.join(data_path, 'region_osm_admin1',
x.split('.')[0] + '.osm.pbf')
for x in get_poly_files
]
big_osm_paths = [
os.path.join(data_path, 'country_osm', x[:3] + '.osm.pbf')
for x in get_poly_files
]
data_paths = [data_path] * len(polyPaths)
# allow for reversed order if you want to run two at the same time (convenient to work backwards for the second process, to prevent overlapping calculation)
if reversed_order == True:
polyPaths = polyPaths[::-1]
area_pbfs = area_pbfs[::-1]
big_osm_paths = big_osm_paths[::-1]
# extract all country osm files through multiprocesing
pool = Pool(cpu_count() - 1)
pool.starmap(clip_osm, zip(data_paths, big_osm_paths, polyPaths,
area_pbfs))
def render(figures, titles=None, build=True, ncores=None):
from shutil import rmtree, copytree
from os.path import join, dirname, abspath
from json import loads
from pathos.multiprocessing import Pool, cpu_count
from queue import Queue
@dataclass
class RenderedFigure:
name: str
fig_fname: str
title: str
argdict: dict
docs: str
html: str
idx: int
def _asdict(self):
return {
"name": self.name,
"fig_fname": self.fig_fname,
"title": self.title,
"argdict": self.argdict,
"docs": self.docs,
"html": self.html,
"idx": self.idx,
}
pkg_dir = dirname(abspath(__file__))
output_dir = CONFIG["output_dir"]
figure_dir = join(output_dir, "figures")
if build:
rmtree(output_dir, ignore_errors=True)
makedirs(output_dir, exist_ok=True)
copytree(join(pkg_dir, "static", "css"), join(output_dir, "css"))
copytree(join(pkg_dir, "static", "icons"), join(output_dir, "icons"))
makedirs(join(output_dir, "aux_figures"), exist_ok=True)
makedirs(figure_dir, exist_ok=True)
nplots = len(figures)
args = []
for idx, (name, figure) in enumerate(figures.items()):
if type(figure) != Figure:
raise TypeError(name + " must be Figure, found: " + str(type(figure)))
args.append((idx, nplots, name, figure, figure_dir))
CONFIG['data_loader']()
if CONFIG["multiprocess"]:
n_procs = ncores if ncores is not None else cpu_count()
pool = Pool(n_procs)
q = Queue()
for _ in range(n_procs):
q.put(d)
args = [[*arg, q] for arg in args]
pool.starmap(_render_one, args)
else:
for arg in args:
_render_one(*arg, None)
try:
for idx, (name, _) in enumerate(figures.items()):
fig_fname = join(figure_dir, name + ".png")
with open(join(figure_dir, name + ".docs.html"), "r") as f:
docs = f.read()
with open(join(figure_dir, name + ".retval.html"), "r") as f:
retval = f.read()
with open(join(figure_dir, name + ".args.json"), "r") as f:
argdict = loads(f.read())
title = titles[name] if titles is not None else name
figures[name] = RenderedFigure(
name, fig_fname, title, argdict, docs, retval, idx
)
except IOError:
print(
"File not found, you probably need to generate the plots! (ie set refresh=True)"
)
sys.exit(-1)
