def download_under_lock(env: "Env", urls: Dict[AnyStr, AnyStr],
stage_name: AnyStr):
lock_name = f"{os.path.join(env.paths.status_path, stage_name)}.lock"
status_name = f"{os.path.join(env.paths.status_path, stage_name)}.download"
with filelock.FileLock(lock_name):
s = status.Status(status_name)
if not s.is_finished():
urls = normalize_url_to_path_dict(urls)
download_files(urls, env.force_download_files)
s.finish()
def apply(obj: Stage, env: "Env", country: AnyStr, *args, **kwargs):
name = get_stage_name(obj)
_logger = DummyObject()
countries_meta = env.countries_meta
if "logger" in countries_meta[country]:
_logger, _ = countries_meta[country]["logger"]
if not env.is_accepted_stage(stage):
_logger.info(f"Stage {name} was not accepted.")
return
if "status" not in countries_meta[country]:
countries_meta[country]["status"] = status.Status()
country_status = countries_meta[country]["status"]
status_file = os.path.join(env.paths.status_path,
status.with_stat_ext(country))
country_status.init(status_file, name)
if country_status.need_skip():
_logger.warning(f"Stage {name} was skipped.")
return
country_status.update_status()
method(obj, env, country, *args, **kwargs)
def __init__(
self,
countries: Optional[List[AnyStr]] = None,
production: bool = False,
build_name: Optional[AnyStr] = None,
build_suffix: AnyStr = "",
skipped_stages: Optional[Set[Type[Stage]]] = None,
force_download_files: bool = False,
):
self.setup_logging()
logger.info("Start setup ...")
os.environ["TMPDIR"] = PathProvider.tmp_dir()
for k, v in self.setup_osm_tools().items():
setattr(self, k, v)
self.production = production
self.force_download_files = force_download_files
self.countries = countries
self.skipped_stages = set(
) if skipped_stages is None else skipped_stages
if self.countries is None:
self.countries = get_all_countries_list(
PathProvider.borders_path())
self.node_storage = settings.NODE_STORAGE
version_format = "%Y_%m_%d__%H_%M_%S"
suffix_div = "-"
self.dt = None
if build_name is None:
self.dt = datetime.datetime.now()
build_name = self.dt.strftime(version_format)
if build_suffix:
build_name = f"{build_name}{suffix_div}{build_suffix}"
else:
s = build_name.split(suffix_div, maxsplit=1)
if len(s) == 1:
s.append("")
date_str, build_suffix = s
self.dt = datetime.datetime.strptime(date_str, version_format)
self.build_suffix = build_suffix
self.mwm_version = self.dt.strftime("%y%m%d")
self.planet_version = self.dt.strftime("%s")
self.build_path = os.path.join(settings.MAIN_OUT_PATH, build_name)
self.build_name = build_name
self.gen_tool = self.setup_generator_tool()
if WORLD_NAME in self.countries:
self.world_roads_builder_tool = self.setup_world_roads_builder_tool(
)
logger.info(f"Build name is {self.build_name}.")
logger.info(f"Build path is {self.build_path}.")
self.paths = PathProvider(self.build_path, self.build_name,
self.mwm_version)
Version.write(self.build_path, self.planet_version)
self.setup_borders()
self.setup_osm2ft()
if self.force_download_files:
for item in os.listdir(self.paths.status_path):
if item.endswith(".download"):
os.remove(os.path.join(self.paths.status_path, item))
self.main_status = status.Status()
# self.countries_meta stores log files and statuses for each country.
self.countries_meta = collections.defaultdict(dict)
self.subprocess_out = None
self.subprocess_countries_out = {}
printed_countries = ", ".join(self.countries)
if len(self.countries) > 50:
printed_countries = (f"{', '.join(self.countries[:25])}, ..., "
f"{', '.join(self.countries[-25:])}")
logger.info(f"The following {len(self.countries)} maps will build: "
f"{printed_countries}.")
logger.info("Finish setup")