def step_download_and_convert_planet(env: Env, force_download: bool, **kwargs):
if force_download or not is_verified(env.paths.planet_osm_pbf):
download_files(
{
settings.PLANET_URL: env.paths.planet_osm_pbf,
settings.PLANET_MD5_URL: md5_ext(env.paths.planet_osm_pbf),
},
env.force_download_files,
)
if not is_verified(env.paths.planet_osm_pbf):
raise ValidationError(
f"Wrong md5 sum for {env.paths.planet_osm_pbf}.")
convert_planet(
env[settings.OSM_TOOL_CONVERT],
env.paths.planet_osm_pbf,
env.paths.planet_o5m,
output=env.get_subprocess_out(),
error=env.get_subprocess_out(),
)
os.remove(env.paths.planet_osm_pbf)
os.remove(md5_ext(env.paths.planet_osm_pbf))
def step_preprocess(env: Env, **kwargs):
run_gen_tool(
env.gen_tool,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
osm_file_type="o5m",
osm_file_name=env.paths.planet_o5m,
node_storage=env.node_storage,
user_resource_path=env.paths.user_resource_path,
preprocess=True,
**kwargs,
)
def step_routing_transit(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
transit_path=env.paths.transit_path,
make_transit_cross_mwm=True,
output=country,
**kwargs,
)
def step_srtm(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
user_resource_path=env.paths.user_resource_path,
srtm_path=env.paths.srtm_path(),
output=country,
**kwargs,
)
def step_isolines_info(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
generate_isolines_info=True,
isolines_path=PathProvider.isolines_path(),
output=country,
**kwargs,
)
def step_popularity(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
popular_places_data=env.paths.popularity_path,
generate_popular_places=True,
output=country,
**kwargs,
)
def step_prepare_routing_world(env: Env, country: AnyStr, **kwargs):
world_roads_builder_tool_with_args = [env.world_roads_builder_tool,
f"--path_roads_file={env.paths.planet_o5m}",
f"--path_resources={env.paths.user_resource_path}",
f"--path_res_file={env.paths.world_roads_path}"]
logger.info(f"Starting {world_roads_builder_tool_with_args}")
sub_proc = subprocess.Popen(
world_roads_builder_tool_with_args,
stdout=env.get_subprocess_out(country),
stderr=env.get_subprocess_out(country),
env=os.environ
)
wait_and_raise_if_fail(sub_proc)
def _generate_common_index(env: Env, country: AnyStr, **kwargs):
run_gen_tool(
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
node_storage=env.node_storage,
planet_version=env.planet_version,
generate_geometry=True,
generate_index=True,
output=country,
**kwargs,
)
def apply(self, env: Env):
run_gen_tool(
env.gen_tool,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
dump_wikipedia_urls=env.paths.wiki_url_path,
idToWikidata=env.paths.id_to_wikidata_path,
)
langs = ("en", "ru", "es", "fr", "de")
checker = check_and_get_checker(env.paths.popularity_path)
download_from_wikipedia_tags(env.paths.wiki_url_path,
env.paths.descriptions_path, langs,
checker)
download_from_wikidata_tags(env.paths.id_to_wikidata_path,
env.paths.descriptions_path, langs,
checker)
def step_routing(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
cities_boundaries_data=env.paths.cities_boundaries_path,
generate_maxspeed=True,
make_city_roads=True,
make_cross_mwm=True,
disable_cross_mwm_progress=True,
generate_cameras=True,
make_routing_index=True,
generate_traffic_keys=True,
output=country,
**kwargs,
)
def step_statistics(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
type_statistics=True,
output=country,
**kwargs,
)
with open(os.path.join(env.paths.stats_path, f"{country}.json"), "w") as f:
json.dump(
make_stats(
settings.STATS_TYPES_CONFIG,
os.path.join(env.paths.intermediate_data_path,
f"{country}.stats")), f)
