def reset_to_stage(self, stage_name: AnyStr):
"""
Resets generation state to stage_name.
Status files are overwritten new statuses according stage_name.
It supposes that stages have next representation:
stage1, ..., stage_mwm[country_stage_1, ..., country_stage_M], ..., stageN
"""
countries_statuses_paths = [
os.path.join(self.env.paths.status_path, f)
for f in os.listdir(self.env.paths.status_path)
if os.path.isfile(os.path.join(self.env.paths.status_path, f))
and os.path.join(self.env.paths.status_path, f)
!= self.env.paths.main_status_path
]
def set_countries_stage(st):
for path in countries_statuses_paths:
Status(path, st).update_status()
def finish_countries_stage():
for path in countries_statuses_paths:
Status(path).finish()
high_level_stages = [get_stage_name(s) for s in self.runnable_stages]
if not (
stage_name in high_level_stages
or any(stage_name == get_stage_name(s) for s in stages.countries_stages)
):
raise ContinueError(f"{stage_name} not in {', '.join(high_level_stages)}.")
if not os.path.exists(self.env.paths.main_status_path):
raise ContinueError(
f"Status file {self.env.paths.main_status_path} not found."
)
if not os.path.exists(self.env.paths.status_path):
raise ContinueError(f"Status path {self.env.paths.status_path} not found.")
mwm_stage_name = get_stage_name(stages.mwm_stage)
stage_mwm_index = high_level_stages.index(mwm_stage_name)
main_status = None
if stage_name in high_level_stages[: stage_mwm_index + 1]:
main_status = stage_name
set_countries_stage("")
elif stage_name in high_level_stages[stage_mwm_index + 1 :]:
main_status = stage_name
finish_countries_stage()
else:
main_status = get_stage_name(stages.mwm_stage)
set_countries_stage(stage_name)
Status(self.env.paths.main_status_path, main_status).update_status()
def apply(obj: Stage, env: "Env", country: AnyStr, *args, **kwargs):
name = get_stage_name(obj)
_logger = DummyObject()
countries_meta = env.countries_meta
if "logger" in countries_meta[country]:
_logger, _ = countries_meta[country]["logger"]
if not env.is_accepted_stage(stage):
_logger.info(f"{name} was not accepted.")
return
if "status" not in countries_meta[country]:
countries_meta[country]["status"] = Status()
status = countries_meta[country]["status"]
status_file = os.path.join(env.paths.status_path, f"{country}.status")
status.init(status_file, name)
if status.need_skip():
_logger.warning(f"{name} was skipped.")
return
status.update_status()
method(obj, env, country, *args, **kwargs)
def set_countries_stage(st):
for path in countries_statuses_paths:
Status(path, st).update_status()
def finish_countries_stage():
for path in countries_statuses_paths:
Status(path).finish()
def __init__(self,
countries: Optional[List[AnyStr]] = None,
production: bool = False,
build_name: Optional[AnyStr] = None,
build_suffix: AnyStr = "",
skipped_stages: Optional[Set[Type[Stage]]] = None,
force_download_files: bool = False):
self.setup_logging()
logger.info("Start setup ...")
os.environ["TMPDIR"] = PathProvider.tmp_dir()
for k, v in self.setup_osm_tools().items():
setattr(self, k, v)
self.gen_tool = self.setup_generator_tool()
self.production = production
self.force_download_files = force_download_files
self.countries = countries
self.skipped_stages = set(
) if skipped_stages is None else skipped_stages
if self.countries is None:
self.countries = get_all_countries_list(
PathProvider.borders_path())
self.node_storage = settings.NODE_STORAGE
version_format = "%Y_%m_%d__%H_%M_%S"
suffix_div = "-"
dt = None
if build_name is None:
dt = datetime.datetime.now()
build_name = dt.strftime(version_format)
if build_suffix:
build_name = f"{build_name}{suffix_div}{build_suffix}"
else:
s = build_name.split(suffix_div, maxsplit=1)
if len(s) == 1:
s.append("")
date_str, build_suffix = s
dt = datetime.datetime.strptime(date_str, version_format)
self.build_suffix = build_suffix
self.mwm_version = dt.strftime("%y%m%d")
self.planet_version = dt.strftime("%s")
self.build_path = os.path.join(settings.MAIN_OUT_PATH, build_name)
self.build_name = build_name
logger.info(f"Build name is {self.build_name}.")
logger.info(f"Build path is {self.build_path}.")
self.paths = PathProvider(self.build_path, self.mwm_version)
Version.write(self.build_path, self.planet_version)
self.setup_borders()
self.setup_osm2ft()
self.main_status = Status()
# self.countries_meta stores log files and statuses for each country.
self.countries_meta = collections.defaultdict(dict)
self.subprocess_out = None
self.subprocess_countries_out = {}
printed_countries = ", ".join(self.countries)
if len(self.countries) > 50:
printed_countries = (f"{', '.join(self.countries[:25])}, ..., "
f"{', '.join(self.countries[-25:])}")
logger.info(f"The following {len(self.countries)} maps will build: "
f"{printed_countries}.")
logger.info("Finish setup")
class Env:
"""
Env provides a generation environment. It sets up instruments and paths,
that are used for a maps generation. It stores state of the maps generation.
"""
def __init__(self,
countries: Optional[List[AnyStr]] = None,
production: bool = False,
build_name: Optional[AnyStr] = None,
build_suffix: AnyStr = "",
skipped_stages: Optional[Set[Type[Stage]]] = None,
force_download_files: bool = False):
self.setup_logging()
logger.info("Start setup ...")
os.environ["TMPDIR"] = PathProvider.tmp_dir()
for k, v in self.setup_osm_tools().items():
setattr(self, k, v)
self.gen_tool = self.setup_generator_tool()
self.production = production
self.force_download_files = force_download_files
self.countries = countries
self.skipped_stages = set(
) if skipped_stages is None else skipped_stages
if self.countries is None:
self.countries = get_all_countries_list(
PathProvider.borders_path())
self.node_storage = settings.NODE_STORAGE
version_format = "%Y_%m_%d__%H_%M_%S"
suffix_div = "-"
dt = None
if build_name is None:
dt = datetime.datetime.now()
build_name = dt.strftime(version_format)
if build_suffix:
build_name = f"{build_name}{suffix_div}{build_suffix}"
else:
s = build_name.split(suffix_div, maxsplit=1)
if len(s) == 1:
s.append("")
date_str, build_suffix = s
dt = datetime.datetime.strptime(date_str, version_format)
self.build_suffix = build_suffix
self.mwm_version = dt.strftime("%y%m%d")
self.planet_version = dt.strftime("%s")
self.build_path = os.path.join(settings.MAIN_OUT_PATH, build_name)
self.build_name = build_name
logger.info(f"Build name is {self.build_name}.")
logger.info(f"Build path is {self.build_path}.")
self.paths = PathProvider(self.build_path, self.mwm_version)
Version.write(self.build_path, self.planet_version)
self.setup_borders()
self.setup_osm2ft()
self.main_status = Status()
# self.countries_meta stores log files and statuses for each country.
self.countries_meta = collections.defaultdict(dict)
self.subprocess_out = None
self.subprocess_countries_out = {}
printed_countries = ", ".join(self.countries)
if len(self.countries) > 50:
printed_countries = (f"{', '.join(self.countries[:25])}, ..., "
f"{', '.join(self.countries[-25:])}")
logger.info(f"The following {len(self.countries)} maps will build: "
f"{printed_countries}.")
logger.info("Finish setup")
def __getitem__(self, item):
return self.__dict__[item]
def get_tmp_mwm_names(self) -> List[AnyStr]:
tmp_ext = ".mwm.tmp"
existing_names = set()
for f in os.listdir(self.paths.intermediate_tmp_path):
path = os.path.join(self.paths.intermediate_tmp_path, f)
if f.endswith(tmp_ext) and os.path.isfile(path):
name = f.replace(tmp_ext, "")
if name in self.countries:
existing_names.add(name)
return [c for c in self.countries if c in existing_names]
def add_skipped_stage(self, stage: Union[Type[Stage], Stage]):
if isinstance(stage, Stage):
stage = stage.__class__
self.skipped_stages.add(stage)
def is_accepted_stage(self, stage: Union[Type[Stage], Stage]) -> bool:
if isinstance(stage, Stage):
stage = stage.__class__
return stage not in self.skipped_stages
def finish(self):
self.main_status.finish()
def finish_mwm(self, mwm_name: AnyStr):
self.countries_meta[mwm_name]["status"].finish()
def set_subprocess_out(self,
subprocess_out: Any,
country: Optional[AnyStr] = None):
if country is None:
self.subprocess_out = subprocess_out
else:
self.subprocess_countries_out[country] = subprocess_out
def get_subprocess_out(self, country: Optional[AnyStr] = None):
if country is None:
return self.subprocess_out
else:
return self.subprocess_countries_out[country]
@staticmethod
def setup_logging():
def exception_handler(type, value, tb):
logger.exception(f"Uncaught exception: {str(value)}",
exc_info=(type, value, tb))
logging.config.dictConfig(settings.LOGGING)
sys.excepthook = exception_handler
@staticmethod
def setup_generator_tool() -> AnyStr:
logger.info("Check generator tool ...")
gen_tool_path = shutil.which(settings.GEN_TOOL)
if gen_tool_path is None:
logger.info(f"Find generator tool in {settings.BUILD_PATH} ...")
gen_tool_path = find_executable(settings.BUILD_PATH,
settings.GEN_TOOL)
logger.info(f"Generator found - {gen_tool_path}")
return gen_tool_path
@staticmethod
def setup_osm_tools() -> Dict[AnyStr, AnyStr]:
path = settings.OSM_TOOLS_PATH
osm_tool_names = [
settings.OSM_TOOL_CONVERT,
settings.OSM_TOOL_UPDATE,
settings.OSM_TOOL_FILTER,
]
logger.info("Check osm tools ...")
if not create_if_not_exist_path(path):
tmp_paths = [os.path.join(path, t) for t in osm_tool_names]
if all([is_executable(t) for t in tmp_paths]):
osm_tool_paths = dict(zip(osm_tool_names, tmp_paths))
logger.info(
f"Osm tools found - {', '.join(osm_tool_paths.values())}")
return osm_tool_paths
tmp_paths = [shutil.which(t) for t in osm_tool_names]
if all(tmp_paths):
osm_tool_paths = dict(zip(osm_tool_names, tmp_paths))
logger.info(
f"Osm tools found - {', '.join(osm_tool_paths.values())}")
return osm_tool_paths
logger.info("Build osm tools ...")
return build_osmtools(settings.OSM_TOOLS_SRC_PATH)
def setup_borders(self):
temp_borders = self.paths.generation_borders_path
# It is needed in case of rebuilding several mwms.
for filename in os.listdir(temp_borders):
file_path = os.path.join(temp_borders, filename)
if os.path.isfile(file_path) or os.path.islink(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
borders = PathProvider.borders_path()
for x in self.countries:
if x in WORLDS_NAMES:
continue
poly = f"{x}.poly"
os.symlink(os.path.join(borders, poly),
os.path.join(temp_borders, poly))
symlink_force(temp_borders,
os.path.join(self.paths.draft_path, "borders"))
def setup_osm2ft(self):
for x in os.listdir(self.paths.osm2ft_path):
p = os.path.join(self.paths.osm2ft_path, x)
if os.path.isfile(p) and x.endswith(".mwm.osm2ft"):
shutil.move(p, os.path.join(self.paths.mwm_path, x))
def reset_to_stage(self, stage_name: AnyStr):
"""
Resets generation state to stage_name.
Status files are overwritten new statuses according stage_name.
It supposes that stages have next representation:
stage1, ..., stage_mwm[country_stage_1, ..., country_stage_M], ..., stageN
"""
high_level_stages = [get_stage_name(s) for s in self.runnable_stages]
if not (stage_name in high_level_stages
or any(stage_name == get_stage_name(s)
for s in stages.countries_stages)):
raise ContinueError(
f"{stage_name} not in {', '.join(high_level_stages)}.")
if not os.path.exists(self.env.paths.status_path):
raise ContinueError(
f"Status path {self.env.paths.status_path} not found.")
if not os.path.exists(self.env.paths.main_status_path):
raise ContinueError(
f"Status file {self.env.paths.main_status_path} not found.")
countries_statuses_paths = []
countries = set(self.env.countries)
for f in os.listdir(self.env.paths.status_path):
full_name = os.path.join(self.env.paths.status_path, f)
if (os.path.isfile(full_name)
and full_name != self.env.paths.main_status_path
and without_stat_ext(f) in countries):
countries_statuses_paths.append(full_name)
def set_countries_stage(st):
for path in countries_statuses_paths:
Status(path, st).update_status()
def finish_countries_stage():
for path in countries_statuses_paths:
Status(path).finish()
def index(l: List, val):
try:
return l.index(val)
except ValueError:
return -1
mwm_stage_name = get_stage_name(stages.mwm_stage)
stage_mwm_index = index(high_level_stages, mwm_stage_name)
main_status = None
if (stage_mwm_index == -1
or stage_name in high_level_stages[:stage_mwm_index + 1]):
main_status = stage_name
set_countries_stage("")
elif stage_name in high_level_stages[stage_mwm_index + 1:]:
main_status = stage_name
finish_countries_stage()
else:
main_status = get_stage_name(stages.mwm_stage)
set_countries_stage(stage_name)
Status(self.env.paths.main_status_path, main_status).update_status()