def make_mwm(country: AnyStr, env: Env): world_stages = { WORLD_NAME: [ StageIndex, StageCitiesIdsWorld, StagePrepareRoutingWorld, StageRoutingWorld, StageMwmStatistics, ], WORLD_COASTS_NAME: [StageIndex, StageMwmStatistics], } mwm_stages = [ StageIndex, StageUgc, StagePopularity, StageSrtm, StageIsolinesInfo, StageDescriptions, StageRouting, StageRoutingTransit, StageMwmDiffs, StageMwmStatistics, ] for stage in world_stages.get(country, mwm_stages): stage(country=country)(env) env.finish_mwm(country)
def make_coastline(env: Env): coastline_o5m = os.path.join(env.paths.coastline_path, "coastline.o5m") filter_coastline( env[settings.OSM_TOOL_FILTER], env.paths.planet_o5m, coastline_o5m, output=env.get_subprocess_out(), error=env.get_subprocess_out(), ) run_gen_tool( env.gen_tool, out=env.get_subprocess_out(), err=env.get_subprocess_out(), intermediate_data_path=env.paths.coastline_path, osm_file_type="o5m", osm_file_name=coastline_o5m, node_storage=env.node_storage, user_resource_path=env.paths.user_resource_path, preprocess=True, ) run_gen_tool( env.gen_tool, out=env.get_subprocess_out(), err=env.get_subprocess_out(), intermediate_data_path=env.paths.coastline_path, osm_file_type="o5m", osm_file_name=coastline_o5m, node_storage=env.node_storage, user_resource_path=env.paths.user_resource_path, make_coasts=True, fail_on_coasts=True, threads_count=settings.THREADS_COUNT, )
def step_download_and_convert_planet(env: Env, force_download: bool, **kwargs): if force_download or not is_verified(env.paths.planet_osm_pbf): download_planet(env.paths.planet_osm_pbf) convert_planet( env[settings.OSM_TOOL_CONVERT], env.paths.planet_osm_pbf, env.paths.planet_o5m, output=env.get_subprocess_out(), error=env.get_subprocess_out(), ) os.remove(env.paths.planet_osm_pbf) os.remove(md5(env.paths.planet_osm_pbf))
def build_coastline(**kwargs): env = Env() kwargs["ti"].xcom_push(key="build_name", value=env.build_name) run_generation( env, ( sd.StageDownloadAndConvertPlanet(), sd.StageCoastline(use_old_if_fail=False), sd.StageCleanup(), ), ) env.finish()
def step_update_planet(env: Env, **kwargs): tmp = f"{env.paths.planet_o5m}.tmp" osmupdate( env[settings.OSM_TOOL_UPDATE], env.paths.planet_o5m, tmp, output=env.get_subprocess_out(), error=env.get_subprocess_out(), **kwargs, ) os.remove(env.paths.planet_o5m) os.rename(tmp, env.paths.planet_o5m) write_md5sum(env.paths.planet_o5m, md5_ext(env.paths.planet_o5m))
def step_prepare_routing_world(env: Env, country: AnyStr, logger, **kwargs): filter_roads( env[settings.OSM_TOOL_FILTER], env.paths.planet_o5m, env.paths.world_roads_o5m, env.get_subprocess_out(country), env.get_subprocess_out(country), ) make_world_road_graph(env.world_roads_builder_tool, env.paths.world_roads_o5m, env.paths.user_resource_path, env.paths.world_roads_path, logger, env.get_subprocess_out(country), env.get_subprocess_out(country))
def apply(self, env: Env): with ThreadPool(settings.THREADS_COUNT) as pool: pool.map( lambda c: StageMwm.make_mwm(c, env), env.get_tmp_mwm_names(), chunksize=1, )
def step_features(env: Env, **kwargs): if env.production: kwargs.update({"add_ads": True}) if any(x not in WORLDS_NAMES for x in env.countries): kwargs.update({"generate_packed_borders": True}) if any(x == WORLD_NAME for x in env.countries): kwargs.update({"generate_world": True}) if len(env.countries) == len( get_all_countries_list(PathProvider.borders_path())): kwargs.update({"have_borders_for_whole_world": True}) run_gen_tool( env.gen_tool, out=env.get_subprocess_out(), err=env.get_subprocess_out(), data_path=env.paths.data_path, intermediate_data_path=env.paths.intermediate_data_path, cache_path=env.paths.cache_path, osm_file_type="o5m", osm_file_name=env.paths.planet_o5m, node_storage=env.node_storage, user_resource_path=env.paths.user_resource_path, dump_cities_boundaries=True, cities_boundaries_data=env.paths.cities_boundaries_path, generate_features=True, threads_count=settings.THREADS_COUNT_FEATURES_STAGE, **kwargs, )
def build_epilog(**kwargs): build_name = kwargs["ti"].xcom_pull(key="build_name") params = MapsGenerationDAG.get_params(**kwargs) params.update({"build_name": build_name}) env = Env(**params) run_generation_from_first_stage( env, ( sd.StageCountriesTxt(), sd.StageExternalResources(), sd.StageLocalAds(), sd.StageStatistics(), sd.StageCleanup(), ), ) env.finish()
def make_mwm(country: AnyStr, env: Env): if country == WORLD_NAME: StageIndex(country=country)(env) StageCitiesIdsWorld(country=country)(env) elif country == WORLD_COASTS_NAME: StageIndex(country=country)(env) else: StageIndex(country=country)(env) StageUgc(country=country)(env) StagePopularity(country=country)(env) StageSrtm(country=country)(env) StageDescriptions(country=country)(env) StageRouting(country=country)(env) StageRoutingTransit(country=country)(env) StageMwmStatistics(country=country)(env) env.finish_mwm(country)
def publish_maps(**kwargs): build_name = kwargs["ti"].xcom_pull(key="build_name") params = MapsGenerationDAG.get_params(**kwargs) params.update({"build_name": build_name}) env = Env(**params) subdir = MapsGenerationDAG.get_params(namespace="storage", **kwargs)["subdir"] storage_path = f"{MAPS_STORAGE_PATH}/{subdir}" storage.wd_publish(env.paths.mwm_path, f"{storage_path}/{env.mwm_version}/")
def build_mwm(**kwargs): build_name = kwargs["ti"].xcom_pull(key="build_name") params = MapsGenerationDAG.get_params(**kwargs) params.update({ "build_name": build_name, "countries": [ country, ] }) env = Env(**params) # We need to check existing of mwm.tmp. It is needed if we want to # build mwms from part of planet. tmp_mwm_name = env.get_tmp_mwm_names() assert len(tmp_mwm_name) <= 1 if not tmp_mwm_name: logger.warning(f"mwm.tmp does not exist for {country}.") return run_generation_from_first_stage(env, (sd.StageMwm(), ), build_lock=False)
def step_routing_world(env: Env, country: AnyStr, **kwargs): run_gen_tool_with_recovery_country( env, env.gen_tool, out=env.get_subprocess_out(country), err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, user_resource_path=env.paths.user_resource_path, output=country, world_roads_path=env.paths.world_roads_path, **kwargs, )
def step_cities_ids_world(env: Env, country: AnyStr, **kwargs): run_gen_tool_with_recovery_country( env, env.gen_tool, out=env.get_subprocess_out(country), err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, user_resource_path=env.paths.user_resource_path, output=country, generate_cities_ids=True, **kwargs, )
def step_preprocess(env: Env, **kwargs): run_gen_tool( env.gen_tool, out=env.get_subprocess_out(), err=env.get_subprocess_out(), intermediate_data_path=env.paths.intermediate_data_path, osm_file_type="o5m", osm_file_name=env.paths.planet_o5m, node_storage=env.node_storage, user_resource_path=env.paths.user_resource_path, preprocess=True, **kwargs, )
def step_description(env: Env, country: AnyStr, **kwargs): run_gen_tool_with_recovery_country( env, env.gen_tool, out=env.get_subprocess_out(country), err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, user_resource_path=env.paths.user_resource_path, wikipedia_pages=env.paths.descriptions_path, idToWikidata=env.paths.id_to_wikidata_path, output=country, **kwargs, )
def step_srtm(env: Env, country: AnyStr, **kwargs): run_gen_tool_with_recovery_country( env, env.gen_tool, out=env.get_subprocess_out(country), err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, user_resource_path=env.paths.user_resource_path, srtm_path=env.paths.srtm_path(), output=country, **kwargs, )
def step_download_and_convert_planet(env: Env, force_download: bool, **kwargs): if force_download or not is_verified(env.paths.planet_osm_pbf): download_files( { settings.PLANET_URL: env.paths.planet_osm_pbf, settings.PLANET_MD5_URL: md5_ext(env.paths.planet_osm_pbf), }, env.force_download_files, ) if not is_verified(env.paths.planet_osm_pbf): raise ValidationError(f"Wrong md5 sum for {env.paths.planet_osm_pbf}.") convert_planet( env[settings.OSM_TOOL_CONVERT], env.paths.planet_osm_pbf, env.paths.planet_o5m, output=env.get_subprocess_out(), error=env.get_subprocess_out(), ) os.remove(env.paths.planet_osm_pbf) os.remove(md5_ext(env.paths.planet_osm_pbf))
def step_routing_transit(env: Env, country: AnyStr, **kwargs): run_gen_tool_with_recovery_country( env, env.gen_tool, out=env.get_subprocess_out(country), err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, user_resource_path=env.paths.user_resource_path, transit_path=env.paths.transit_path, make_transit_cross_mwm=True, output=country, **kwargs, )
def build_prolog(**kwargs): params = MapsGenerationDAG.get_params(**kwargs) env = Env(**params) kwargs["ti"].xcom_push(key="build_name", value=env.build_name) run_generation( env, ( sd.StageDownloadAndConvertPlanet(), sd.StageCoastline(), sd.StagePreprocess(), sd.StageFeatures(), sd.StageDownloadDescriptions(), ), )
def step_isolines_info(env: Env, country: AnyStr, **kwargs): run_gen_tool_with_recovery_country( env, env.gen_tool, out=env.get_subprocess_out(country), err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, user_resource_path=env.paths.user_resource_path, generate_isolines_info=True, isolines_path=PathProvider.isolines_path(), output=country, **kwargs, )
def step_prepare_routing_world(env: Env, country: AnyStr, **kwargs): world_roads_builder_tool_with_args = [env.world_roads_builder_tool, f"--path_roads_file={env.paths.planet_o5m}", f"--path_resources={env.paths.user_resource_path}", f"--path_res_file={env.paths.world_roads_path}"] logger.info(f"Starting {world_roads_builder_tool_with_args}") sub_proc = subprocess.Popen( world_roads_builder_tool_with_args, stdout=env.get_subprocess_out(country), stderr=env.get_subprocess_out(country), env=os.environ ) wait_and_raise_if_fail(sub_proc)
def step_popularity(env: Env, country: AnyStr, **kwargs): run_gen_tool_with_recovery_country( env, env.gen_tool, out=env.get_subprocess_out(country), err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, user_resource_path=env.paths.user_resource_path, popular_places_data=env.paths.popularity_path, generate_popular_places=True, output=country, **kwargs, )
def publish_coastline(**kwargs): build_name = kwargs["ti"].xcom_pull(key="build_name") env = Env(build_name=build_name) for name in (f"{WORLD_COASTS_NAME}.geom", f"{WORLD_COASTS_NAME}.rawgeom"): coastline = put_current_date_in_filename(name) latest = get_latest_filename(name) coastline_full = os.path.join(env.paths.coastline_path, coastline) latest_full = os.path.join(env.paths.coastline_path, latest) shutil.move(os.path.join(env.paths.coastline_path, name), coastline_full) os.symlink(coastline, latest_full) storage.wd_publish(coastline_full, f"{COASTLINE_STORAGE_PATH}/{coastline}") storage.wd_publish(latest_full, f"{COASTLINE_STORAGE_PATH}/{latest}")
def _generate_common_index(env: Env, country: AnyStr, **kwargs): run_gen_tool( env.gen_tool, out=env.get_subprocess_out(country), err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, user_resource_path=env.paths.user_resource_path, node_storage=env.node_storage, planet_version=env.planet_version, generate_geometry=True, generate_index=True, output=country, **kwargs, )
def apply(self, env: Env): run_gen_tool( env.gen_tool, out=env.get_subprocess_out(), err=env.get_subprocess_out(), intermediate_data_path=env.paths.intermediate_data_path, user_resource_path=env.paths.user_resource_path, dump_wikipedia_urls=env.paths.wiki_url_path, idToWikidata=env.paths.id_to_wikidata_path, ) langs = ("en", "ru", "es", "fr", "de") checker = check_and_get_checker(env.paths.popularity_path) download_from_wikipedia_tags(env.paths.wiki_url_path, env.paths.descriptions_path, langs, checker) download_from_wikidata_tags(env.paths.id_to_wikidata_path, env.paths.descriptions_path, langs, checker)
def apply(self, env: Env): steps_info = get_stages_info(env.paths.log_path, {"statistics"}) stats = defaultdict(lambda: defaultdict(dict)) stats["steps"] = steps_info["steps"] for country in env.get_tmp_mwm_names(): with open(os.path.join(env.paths.stats_path, f"{country}.json")) as f: stats["countries"][country] = { "types": json.load(f), "steps": steps_info["countries"][country] } def default(o): if isinstance(o, datetime.timedelta): return str(o) with open(os.path.join(env.paths.stats_path, "stats.json"), "w") as f: json.dump( stats, f, ensure_ascii=False, sort_keys=True, indent=2, default=default )
def step_statistics(env: Env, country: AnyStr, **kwargs): run_gen_tool_with_recovery_country( env, env.gen_tool, out=env.get_subprocess_out(country), err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, user_resource_path=env.paths.user_resource_path, type_statistics=True, output=country, **kwargs, ) with open(os.path.join(env.paths.stats_path, f"{country}.json"), "w") as f: json.dump( make_stats( settings.STATS_TYPES_CONFIG, os.path.join(env.paths.intermediate_data_path, f"{country}.stats")), f)
def step_routing(env: Env, country: AnyStr, **kwargs): run_gen_tool_with_recovery_country( env, env.gen_tool, out=env.get_subprocess_out(country), err=env.get_subprocess_out(country), data_path=env.paths.mwm_path, intermediate_data_path=env.paths.intermediate_data_path, user_resource_path=env.paths.user_resource_path, cities_boundaries_data=env.paths.cities_boundaries_path, generate_maxspeed=True, make_city_roads=True, make_cross_mwm=True, disable_cross_mwm_progress=True, generate_cameras=True, make_routing_index=True, generate_traffic_keys=True, output=country, **kwargs, )
def update_planet(**kwargs): env = Env() kwargs["ti"].xcom_push(key="build_name", value=env.build_name) if settings.DEBUG: env.add_skipped_stage(sd.StageUpdatePlanet) run_generation( env, ( sd.StageDownloadAndConvertPlanet(), sd.StageUpdatePlanet(), sd.StageCleanup(), ), ) env.finish()