def download_under_lock(env: "Env", urls: Dict[AnyStr, AnyStr],
stage_name: AnyStr):
lock_name = f"{os.path.join(env.paths.status_path, stage_name)}.lock"
status_name = f"{os.path.join(env.paths.status_path, stage_name)}.download"
with filelock.FileLock(lock_name):
s = status.Status(status_name)
if not s.is_finished():
urls = normalize_url_to_path_dict(urls)
download_files(urls, env.force_download_files)
s.finish()
def apply(self, env: Env, use_old_if_fail=True):
coasts_geom = "WorldCoasts.geom"
coasts_rawgeom = "WorldCoasts.rawgeom"
try:
coastline.make_coastline(env)
except BadExitStatusError as e:
if not use_old_if_fail:
raise e
logger.info("Build costs failed. Try to download the costs...")
download_files({
settings.PLANET_COASTS_GEOM_URL:
os.path.join(env.paths.coastline_path, coasts_geom),
settings.PLANET_COASTS_RAWGEOM_URL:
os.path.join(env.paths.coastline_path, coasts_rawgeom),
})
for f in [coasts_geom, coasts_rawgeom]:
path = os.path.join(env.paths.coastline_path, f)
shutil.copy2(path, env.paths.intermediate_data_path)
def step_download_and_convert_planet(env: Env, force_download: bool, **kwargs):
if force_download or not is_verified(env.paths.planet_osm_pbf):
download_files(
{
settings.PLANET_URL: env.paths.planet_osm_pbf,
settings.PLANET_MD5_URL: md5_ext(env.paths.planet_osm_pbf),
},
env.force_download_files,
)
if not is_verified(env.paths.planet_osm_pbf):
raise ValidationError(f"Wrong md5 sum for {env.paths.planet_osm_pbf}.")
convert_planet(
env[settings.OSM_TOOL_CONVERT],
env.paths.planet_osm_pbf,
env.paths.planet_o5m,
output=env.get_subprocess_out(),
error=env.get_subprocess_out(),
)
os.remove(env.paths.planet_osm_pbf)
os.remove(md5_ext(env.paths.planet_osm_pbf))
def apply(obj: Stage, env: "Env", *args, **kwargs):
if hasattr(obj,
"internal_dependencies") and obj.internal_dependencies:
with obj.depends_from_internal_lock:
if not obj.depends_from_internal_downloaded:
deps = {}
for d in obj.internal_dependencies:
if "p" in d.mode and not env.production:
continue
path = None
if type(d.path_method) is property:
path = d.path_method.__get__(env.paths)
assert path is not None, type(d.path_method)
deps[d.url] = path
if deps:
download_files(deps, env.force_download_files)
obj.depends_from_internal_downloaded = True
method(obj, env, *args, **kwargs)