def make_mwm(country: AnyStr, env: Env):
world_stages = {
WORLD_NAME: [
StageIndex,
StageCitiesIdsWorld,
StagePrepareRoutingWorld,
StageRoutingWorld,
StageMwmStatistics,
],
WORLD_COASTS_NAME: [StageIndex, StageMwmStatistics],
}
mwm_stages = [
StageIndex,
StageUgc,
StagePopularity,
StageSrtm,
StageIsolinesInfo,
StageDescriptions,
StageRouting,
StageRoutingTransit,
StageMwmDiffs,
StageMwmStatistics,
]
for stage in world_stages.get(country, mwm_stages):
stage(country=country)(env)
env.finish_mwm(country)
def make_coastline(env: Env):
coastline_o5m = os.path.join(env.paths.coastline_path, "coastline.o5m")
filter_coastline(
env[settings.OSM_TOOL_FILTER],
env.paths.planet_o5m,
coastline_o5m,
output=env.get_subprocess_out(),
error=env.get_subprocess_out(),
)
run_gen_tool(
env.gen_tool,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.paths.coastline_path,
osm_file_type="o5m",
osm_file_name=coastline_o5m,
node_storage=env.node_storage,
user_resource_path=env.paths.user_resource_path,
preprocess=True,
)
run_gen_tool(
env.gen_tool,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.paths.coastline_path,
osm_file_type="o5m",
osm_file_name=coastline_o5m,
node_storage=env.node_storage,
user_resource_path=env.paths.user_resource_path,
make_coasts=True,
fail_on_coasts=True,
threads_count=settings.THREADS_COUNT,
)
def step_download_and_convert_planet(env: Env, force_download: bool, **kwargs):
if force_download or not is_verified(env.paths.planet_osm_pbf):
download_planet(env.paths.planet_osm_pbf)
convert_planet(
env[settings.OSM_TOOL_CONVERT],
env.paths.planet_osm_pbf,
env.paths.planet_o5m,
output=env.get_subprocess_out(),
error=env.get_subprocess_out(),
)
os.remove(env.paths.planet_osm_pbf)
os.remove(md5(env.paths.planet_osm_pbf))
def build_coastline(**kwargs):
env = Env()
kwargs["ti"].xcom_push(key="build_name", value=env.build_name)
run_generation(
env,
(
sd.StageDownloadAndConvertPlanet(),
sd.StageCoastline(use_old_if_fail=False),
sd.StageCleanup(),
),
)
env.finish()
def step_update_planet(env: Env, **kwargs):
tmp = f"{env.paths.planet_o5m}.tmp"
osmupdate(
env[settings.OSM_TOOL_UPDATE],
env.paths.planet_o5m,
tmp,
output=env.get_subprocess_out(),
error=env.get_subprocess_out(),
**kwargs,
)
os.remove(env.paths.planet_o5m)
os.rename(tmp, env.paths.planet_o5m)
write_md5sum(env.paths.planet_o5m, md5_ext(env.paths.planet_o5m))
def step_prepare_routing_world(env: Env, country: AnyStr, logger, **kwargs):
filter_roads(
env[settings.OSM_TOOL_FILTER],
env.paths.planet_o5m,
env.paths.world_roads_o5m,
env.get_subprocess_out(country),
env.get_subprocess_out(country),
)
make_world_road_graph(env.world_roads_builder_tool,
env.paths.world_roads_o5m,
env.paths.user_resource_path,
env.paths.world_roads_path, logger,
env.get_subprocess_out(country),
env.get_subprocess_out(country))
def apply(self, env: Env):
with ThreadPool(settings.THREADS_COUNT) as pool:
pool.map(
lambda c: StageMwm.make_mwm(c, env),
env.get_tmp_mwm_names(),
chunksize=1,
)
def step_features(env: Env, **kwargs):
if env.production:
kwargs.update({"add_ads": True})
if any(x not in WORLDS_NAMES for x in env.countries):
kwargs.update({"generate_packed_borders": True})
if any(x == WORLD_NAME for x in env.countries):
kwargs.update({"generate_world": True})
if len(env.countries) == len(
get_all_countries_list(PathProvider.borders_path())):
kwargs.update({"have_borders_for_whole_world": True})
run_gen_tool(
env.gen_tool,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
data_path=env.paths.data_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
osm_file_type="o5m",
osm_file_name=env.paths.planet_o5m,
node_storage=env.node_storage,
user_resource_path=env.paths.user_resource_path,
dump_cities_boundaries=True,
cities_boundaries_data=env.paths.cities_boundaries_path,
generate_features=True,
threads_count=settings.THREADS_COUNT_FEATURES_STAGE,
**kwargs,
)
def build_epilog(**kwargs):
build_name = kwargs["ti"].xcom_pull(key="build_name")
params = MapsGenerationDAG.get_params(**kwargs)
params.update({"build_name": build_name})
env = Env(**params)
run_generation_from_first_stage(
env,
(
sd.StageCountriesTxt(),
sd.StageExternalResources(),
sd.StageLocalAds(),
sd.StageStatistics(),
sd.StageCleanup(),
),
)
env.finish()
def make_mwm(country: AnyStr, env: Env):
if country == WORLD_NAME:
StageIndex(country=country)(env)
StageCitiesIdsWorld(country=country)(env)
elif country == WORLD_COASTS_NAME:
StageIndex(country=country)(env)
else:
StageIndex(country=country)(env)
StageUgc(country=country)(env)
StagePopularity(country=country)(env)
StageSrtm(country=country)(env)
StageDescriptions(country=country)(env)
StageRouting(country=country)(env)
StageRoutingTransit(country=country)(env)
StageMwmStatistics(country=country)(env)
env.finish_mwm(country)
def publish_maps(**kwargs):
build_name = kwargs["ti"].xcom_pull(key="build_name")
params = MapsGenerationDAG.get_params(**kwargs)
params.update({"build_name": build_name})
env = Env(**params)
subdir = MapsGenerationDAG.get_params(namespace="storage",
**kwargs)["subdir"]
storage_path = f"{MAPS_STORAGE_PATH}/{subdir}"
storage.wd_publish(env.paths.mwm_path,
f"{storage_path}/{env.mwm_version}/")
def build_mwm(**kwargs):
build_name = kwargs["ti"].xcom_pull(key="build_name")
params = MapsGenerationDAG.get_params(**kwargs)
params.update({
"build_name": build_name,
"countries": [
country,
]
})
env = Env(**params)
# We need to check existing of mwm.tmp. It is needed if we want to
# build mwms from part of planet.
tmp_mwm_name = env.get_tmp_mwm_names()
assert len(tmp_mwm_name) <= 1
if not tmp_mwm_name:
logger.warning(f"mwm.tmp does not exist for {country}.")
return
run_generation_from_first_stage(env, (sd.StageMwm(), ),
build_lock=False)
def step_routing_world(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
user_resource_path=env.paths.user_resource_path,
output=country,
world_roads_path=env.paths.world_roads_path,
**kwargs,
)
def step_cities_ids_world(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
user_resource_path=env.paths.user_resource_path,
output=country,
generate_cities_ids=True,
**kwargs,
)
def step_preprocess(env: Env, **kwargs):
run_gen_tool(
env.gen_tool,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.paths.intermediate_data_path,
osm_file_type="o5m",
osm_file_name=env.paths.planet_o5m,
node_storage=env.node_storage,
user_resource_path=env.paths.user_resource_path,
preprocess=True,
**kwargs,
)
def step_description(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
user_resource_path=env.paths.user_resource_path,
wikipedia_pages=env.paths.descriptions_path,
idToWikidata=env.paths.id_to_wikidata_path,
output=country,
**kwargs,
)
def step_srtm(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
srtm_path=env.paths.srtm_path(),
output=country,
**kwargs,
)
def step_download_and_convert_planet(env: Env, force_download: bool, **kwargs):
if force_download or not is_verified(env.paths.planet_osm_pbf):
download_files(
{
settings.PLANET_URL: env.paths.planet_osm_pbf,
settings.PLANET_MD5_URL: md5_ext(env.paths.planet_osm_pbf),
},
env.force_download_files,
)
if not is_verified(env.paths.planet_osm_pbf):
raise ValidationError(f"Wrong md5 sum for {env.paths.planet_osm_pbf}.")
convert_planet(
env[settings.OSM_TOOL_CONVERT],
env.paths.planet_osm_pbf,
env.paths.planet_o5m,
output=env.get_subprocess_out(),
error=env.get_subprocess_out(),
)
os.remove(env.paths.planet_osm_pbf)
os.remove(md5_ext(env.paths.planet_osm_pbf))
def step_routing_transit(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
transit_path=env.paths.transit_path,
make_transit_cross_mwm=True,
output=country,
**kwargs,
)
def build_prolog(**kwargs):
params = MapsGenerationDAG.get_params(**kwargs)
env = Env(**params)
kwargs["ti"].xcom_push(key="build_name", value=env.build_name)
run_generation(
env,
(
sd.StageDownloadAndConvertPlanet(),
sd.StageCoastline(),
sd.StagePreprocess(),
sd.StageFeatures(),
sd.StageDownloadDescriptions(),
),
)
def step_isolines_info(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
generate_isolines_info=True,
isolines_path=PathProvider.isolines_path(),
output=country,
**kwargs,
)
def step_prepare_routing_world(env: Env, country: AnyStr, **kwargs):
world_roads_builder_tool_with_args = [env.world_roads_builder_tool,
f"--path_roads_file={env.paths.planet_o5m}",
f"--path_resources={env.paths.user_resource_path}",
f"--path_res_file={env.paths.world_roads_path}"]
logger.info(f"Starting {world_roads_builder_tool_with_args}")
sub_proc = subprocess.Popen(
world_roads_builder_tool_with_args,
stdout=env.get_subprocess_out(country),
stderr=env.get_subprocess_out(country),
env=os.environ
)
wait_and_raise_if_fail(sub_proc)
def step_popularity(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
popular_places_data=env.paths.popularity_path,
generate_popular_places=True,
output=country,
**kwargs,
)
def publish_coastline(**kwargs):
build_name = kwargs["ti"].xcom_pull(key="build_name")
env = Env(build_name=build_name)
for name in (f"{WORLD_COASTS_NAME}.geom", f"{WORLD_COASTS_NAME}.rawgeom"):
coastline = put_current_date_in_filename(name)
latest = get_latest_filename(name)
coastline_full = os.path.join(env.paths.coastline_path, coastline)
latest_full = os.path.join(env.paths.coastline_path, latest)
shutil.move(os.path.join(env.paths.coastline_path, name),
coastline_full)
os.symlink(coastline, latest_full)
storage.wd_publish(coastline_full,
f"{COASTLINE_STORAGE_PATH}/{coastline}")
storage.wd_publish(latest_full, f"{COASTLINE_STORAGE_PATH}/{latest}")
def _generate_common_index(env: Env, country: AnyStr, **kwargs):
run_gen_tool(
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
node_storage=env.node_storage,
planet_version=env.planet_version,
generate_geometry=True,
generate_index=True,
output=country,
**kwargs,
)
def apply(self, env: Env):
run_gen_tool(
env.gen_tool,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
dump_wikipedia_urls=env.paths.wiki_url_path,
idToWikidata=env.paths.id_to_wikidata_path,
)
langs = ("en", "ru", "es", "fr", "de")
checker = check_and_get_checker(env.paths.popularity_path)
download_from_wikipedia_tags(env.paths.wiki_url_path,
env.paths.descriptions_path, langs,
checker)
download_from_wikidata_tags(env.paths.id_to_wikidata_path,
env.paths.descriptions_path, langs,
checker)
def apply(self, env: Env):
steps_info = get_stages_info(env.paths.log_path, {"statistics"})
stats = defaultdict(lambda: defaultdict(dict))
stats["steps"] = steps_info["steps"]
for country in env.get_tmp_mwm_names():
with open(os.path.join(env.paths.stats_path, f"{country}.json")) as f:
stats["countries"][country] = {
"types": json.load(f),
"steps": steps_info["countries"][country]
}
def default(o):
if isinstance(o, datetime.timedelta):
return str(o)
with open(os.path.join(env.paths.stats_path, "stats.json"), "w") as f:
json.dump(
stats, f, ensure_ascii=False, sort_keys=True, indent=2, default=default
)
def step_statistics(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
type_statistics=True,
output=country,
**kwargs,
)
with open(os.path.join(env.paths.stats_path, f"{country}.json"), "w") as f:
json.dump(
make_stats(
settings.STATS_TYPES_CONFIG,
os.path.join(env.paths.intermediate_data_path,
f"{country}.stats")), f)
def step_routing(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
cities_boundaries_data=env.paths.cities_boundaries_path,
generate_maxspeed=True,
make_city_roads=True,
make_cross_mwm=True,
disable_cross_mwm_progress=True,
generate_cameras=True,
make_routing_index=True,
generate_traffic_keys=True,
output=country,
**kwargs,
)
def update_planet(**kwargs):
env = Env()
kwargs["ti"].xcom_push(key="build_name", value=env.build_name)
if settings.DEBUG:
env.add_skipped_stage(sd.StageUpdatePlanet)
run_generation(
env,
(
sd.StageDownloadAndConvertPlanet(),
sd.StageUpdatePlanet(),
sd.StageCleanup(),
),
)
env.finish()
