def step_features(env: Env, **kwargs):
if env.production:
kwargs.update({"add_ads": True})
if any(x not in WORLDS_NAMES for x in env.countries):
kwargs.update({"generate_packed_borders": True})
if any(x == WORLD_NAME for x in env.countries):
kwargs.update({"generate_world": True})
if len(env.countries) == len(
get_all_countries_list(PathProvider.borders_path())):
kwargs.update({"have_borders_for_whole_world": True})
run_gen_tool(
env.gen_tool,
out=env.get_subprocess_out(),
err=env.get_subprocess_out(),
data_path=env.paths.data_path,
intermediate_data_path=env.paths.intermediate_data_path,
cache_path=env.paths.cache_path,
osm_file_type="o5m",
osm_file_name=env.paths.planet_o5m,
node_storage=env.node_storage,
user_resource_path=env.paths.user_resource_path,
dump_cities_boundaries=True,
cities_boundaries_data=env.paths.cities_boundaries_path,
generate_features=True,
threads_count=settings.THREADS_COUNT_FEATURES_STAGE,
**kwargs,
)
def apply(self, env: Env):
extra = {}
if is_accepted(env, StageDescriptions):
extra.update({"idToWikidata": env.paths.id_to_wikidata_path})
if env.production:
extra.update({
"booking_data":
env.paths.hotels_path,
"promo_catalog_cities":
env.paths.promo_catalog_cities_path,
"popular_places_data":
env.paths.popularity_path,
"brands_data":
env.paths.food_paths,
"brands_translations_data":
env.paths.food_translations_path,
})
if is_accepted(env, StageCoastline):
extra.update({"emit_coasts": True})
if is_accepted(env, StageIsolinesInfo):
extra.update({"isolines_path": PathProvider.isolines_path()})
steps.step_features(env, **extra)
if os.path.exists(env.paths.packed_polygons_path):
shutil.copy2(env.paths.packed_polygons_path, env.paths.mwm_path)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
build_prolog_task = PythonOperator(
task_id="Build_prolog_task",
provide_context=True,
python_callable=MapsGenerationDAG.build_prolog,
dag=self,
)
build_epilog_task = PythonOperator(
task_id="Build_epilog_task",
provide_context=True,
python_callable=MapsGenerationDAG.build_epilog,
dag=self,
)
publish_maps_task = PythonOperator(
task_id="Publish_maps_task",
provide_context=True,
python_callable=MapsGenerationDAG.publish_maps,
dag=self,
)
rm_build_task = make_rm_build_task(self)
build_epilog_task >> publish_maps_task >> rm_build_task
for country in get_all_countries_list(PathProvider.borders_path()):
build_prolog_task >> self.make_mwm_operator(
country) >> build_epilog_task
def step_isolines_info(env: Env, country: AnyStr, **kwargs):
run_gen_tool_with_recovery_country(
env,
env.gen_tool,
out=env.get_subprocess_out(country),
err=env.get_subprocess_out(country),
data_path=env.paths.mwm_path,
intermediate_data_path=env.paths.intermediate_data_path,
user_resource_path=env.paths.user_resource_path,
generate_isolines_info=True,
isolines_path=PathProvider.isolines_path(),
output=country,
**kwargs,
)
def main():
root = logging.getLogger()
root.addHandler(logging.NullHandler())
options = parse_options()
# Processing of 'continue' option.
# If 'continue' is set maps generation is continued from the last build
# that is found automatically.
build_name = None
if options["continue"] is None or options["continue"]:
d = find_last_build_dir(options["continue"])
if d is None:
raise ContinueError(
"The build cannot continue: the last build " "directory was not found."
)
build_name = d
# Processing of 'countries' option.
# There is processing 'countries' and 'without_countries' options.
# Option 'without_countries' has more priority than 'countries'.
# Options 'countries' and 'without_countries' can include '*'.
# For example: '--countries="UK*, Japan"*' means
# '--countries="UK_England_East Midlands, UK_England_East of England_Essex, ...,
# Japan_Chubu Region_Aichi_Nagoya, Japan_Chubu Region_Aichi_Toyohashi, ..."'.
countries_line = ""
without_countries_line = ""
if "COUNTRIES" in os.environ:
countries_line = os.environ["COUNTRIES"]
if options["countries"]:
countries_line = options["countries"]
else:
countries_line = "*"
if options["without_countries"]:
without_countries_line = options["without_countries"]
all_countries = get_all_countries_list(PathProvider.borders_path())
def end_star_compare(prefix, full):
return full.startswith(prefix)
def compare(a, b):
return a == b
def get_countries_set_from_line(line):
countries = []
used_countries = set()
countries_list = []
if os.path.isfile(line):
with open(line) as f:
countries_list = [x.strip() for x in f]
elif line:
countries_list = [x.strip() for x in line.replace(";", ",").split(",")]
for country_item in countries_list:
cmp = compare
_raw_country = country_item[:]
if _raw_country and _raw_country[-1] == "*":
_raw_country = _raw_country.replace("*", "")
cmp = end_star_compare
for country in all_countries:
if cmp(_raw_country, country):
used_countries.add(country_item)
countries.append(country)
countries = unique(countries)
diff = set(countries_list) - used_countries
if diff:
raise ValidationError(f"Bad input countries {', '.join(diff)}")
return set(countries)
countries = get_countries_set_from_line(countries_line)
without_countries = get_countries_set_from_line(without_countries_line)
countries -= without_countries
countries = list(countries)
if not countries:
countries = all_countries
# Processing of 'order' option.
# It defines an order of countries generation using a file from 'order' path.
if options["order"]:
ordered_countries = []
countries = set(countries)
with open(options["order"]) as file:
for c in file:
if c.strip().startswith("#"):
continue
c = c.split("\t")[0].strip()
if c in countries:
ordered_countries.append(c)
countries.remove(c)
if countries:
raise ValueError(
f"{options['order']} does not have an order " f"for {countries}."
)
countries = ordered_countries
# Processing of 'skip' option.
skipped_stages = set()
if options["skip"]:
for s in options["skip"].replace(";", ",").split(","):
stage = s.strip()
if not stages.stages.is_valid_stage_name(stage):
raise SkipError(f"{stage} not found.")
skipped_stages.add(stages.get_stage_type(stage))
if settings.PLANET_URL != settings.DEFAULT_PLANET_URL:
skipped_stages.add(sd.StageUpdatePlanet)
if sd.StageCoastline in skipped_stages:
if any(x in WORLDS_NAMES for x in options["countries"]):
raise SkipError(
f"You can not skip {stages.get_stage_name(sd.StageCoastline)}"
f" if you want to generate {WORLDS_NAMES}."
f" You can exclude them with --without_countries option."
)
if not settings.NEED_PLANET_UPDATE:
skipped_stages.add(sd.StageUpdatePlanet)
# Make env and run maps generation.
env = Env(
countries=countries,
production=options["production"],
build_name=build_name,
skipped_stages=skipped_stages,
)
from_stage = None
if options["from_stage"]:
from_stage = f"{options['from_stage']}"
if options["coasts"]:
generate_coasts(env, from_stage)
else:
generate_maps(env, from_stage)
env.finish()