def apply(self, env: Env): with ThreadPool(settings.THREADS_COUNT) as pool: pool.map( lambda c: StageMwm.make_mwm(c, env), env.get_tmp_mwm_names(), chunksize=1, )
def apply(self, env: Env): steps_info = get_stages_info(env.paths.log_path, {"statistics"}) stats = defaultdict(lambda: defaultdict(dict)) stats["steps"] = steps_info["steps"] for country in env.get_tmp_mwm_names(): with open(os.path.join(env.paths.stats_path, f"{country}.json")) as f: stats["countries"][country] = { "types": json.load(f), "steps": steps_info["countries"][country] } def default(o): if isinstance(o, datetime.timedelta): return str(o) with open(os.path.join(env.paths.stats_path, "stats.json"), "w") as f: json.dump( stats, f, ensure_ascii=False, sort_keys=True, indent=2, default=default )
def build_mwm(**kwargs): build_name = kwargs["ti"].xcom_pull(key="build_name") params = MapsGenerationDAG.get_params(**kwargs) params.update({ "build_name": build_name, "countries": [ country, ] }) env = Env(**params) # We need to check existing of mwm.tmp. It is needed if we want to # build mwms from part of planet. tmp_mwm_name = env.get_tmp_mwm_names() assert len(tmp_mwm_name) <= 1 if not tmp_mwm_name: logger.warning(f"mwm.tmp does not exist for {country}.") return run_generation_from_first_stage(env, (sd.StageMwm(), ), build_lock=False)