def from_lamps(
dir_name,
n_bins,
params,
out_name,
x,
lop,
angles,
wav,
spct,
viirs,
refl,
bool_array,
):
print("Building inputs from discrete inventory.")
# lamps distribution
inv_name = params["lamps_inventory"]
lampsData = np.loadtxt(inv_name, usecols=list(range(7)), ndmin=2)
photometry = np.loadtxt(inv_name, usecols=[-2, -1], dtype=str, ndmin=2)
domain = MSD.from_domain("domain.ini")
sources = np.unique(photometry[:, 1])
print("Classifying points.")
points = dict()
for layer in range(len(domain)):
ysize, xsize = domain[layer].shape
col, row = domain._get_col_row(lampsData[:, :2].T, layer)
valid = (0 <= col) * (col < xsize) * (0 <= row) * (row < ysize) > 0
ind = np.where(valid)[0]
points[layer] = (col[ind], row[ind], ind)
print("Calculating the generalized lamps.")
for n in range(n_bins):
for s in sources:
np.savetxt(
dir_name + "fctem_wl_%g_lamp_%s.dat" % (x[n], s),
np.concatenate([lop[s], angles]).reshape((2, -1)).T,
)
with open(dir_name + "lamps.lst", "w") as zfile:
zfile.write("\n".join(sources) + "\n")
geometry = dict()
for geo in ["obsth", "obstd", "obstf", "altlp", "lights"]:
geometry[geo] = MSD.from_domain("domain.ini")
lumlp = dict()
for s in sources:
for wl in x:
lumlp[s, wl] = MSD.from_domain("domain.ini")
for layer, pts in points.items():
cols, rows, inds = pts
if len(inds):
for col, row in np.unique([cols, rows], axis=1).T:
ind = inds[np.logical_and(cols == col, rows == row)]
lumens = lampsData[:, 2][ind]
for n, geo in zip(range(3, 7),
["obsth", "obstd", "obstf", "altlp"]):
geometry[geo][layer][row,
col] = np.average(lampsData[:,
n][ind],
weights=lumens)
geometry["lights"][layer][row, col] = 1
local_sources = np.unique(photometry[ind][:, 1])
for s in local_sources:
mask = photometry[:, 1][ind] == s
fctem = np.array(
[spct[type] for type in photometry[:, 0][ind][mask]])
fctem = np.sum(fctem * lumens[mask, None], 0)
y = [np.mean(fctem[mask]) for mask in bool_array]
for i, wl in enumerate(x):
lumlp[s, wl][layer][row, col] = y[i]
print("Saving data.")
for geo, ds in geometry.items():
ds.save(dir_name + out_name + "_" + geo)
for key, ds in lumlp.items():
s, wl = key
ds.save(dir_name + "%s_%03d_lumlp_%s" % (out_name, wl, s))
def from_zones(
dir_name,
inv_name,
n_inv,
n_bins,
params,
out_name,
x,
lop,
angles,
wav,
spct,
viirs,
refl,
bool_array,
):
print("Building inputs from zones inventory.")
# lamps distribution
zonData = pt.parse_inventory(inv_name, n_inv)
sources = np.unique([lamp[2] for zd in zonData for lamp in zd])
for n in range(n_bins):
for s in sources:
np.savetxt(
dir_name + "fctem_wl_%g_lamp_%s.dat" % (x[n], s),
np.concatenate([lop[s], angles]).reshape((2, -1)).T,
)
with open(dir_name + "lamps.lst", "w") as zfile:
zfile.write("\n".join(sources) + "\n")
print("Making zone properties files.")
circles = MSD.from_domain("domain.ini") # Same geolocalisation
zonfile = np.loadtxt(params["zones_inventory"],
usecols=list(range(7)),
ndmin=2)
# zone number
for i, dat in enumerate(zonfile, 1):
circles.set_circle((dat[0], dat[1]), dat[2] * 1000, i)
circles.save(dir_name + out_name + "_zone")
weights = [sum(z[0] for z in zone) for zone in zonData]
for w, dat in zip(weights, zonfile):
circles.set_circle((dat[0], dat[1]), dat[2] * 1000, bool(w))
circles.save(dir_name + "origin")
for n, name in zip(range(3, 7), ["obsth", "obstd", "obstf", "altlp"]):
for i, dat in enumerate(zonfile, 1):
circles.set_circle((dat[0], dat[1]), dat[2] * 1000, dat[n])
circles.save(dir_name + out_name + "_" + name)
print("Inverting lamp intensity.")
viirs_dat = MSD.Open("stable_lights.hdf5")
for i in range(len(viirs_dat)):
viirs_dat[i] *= 1e-5 # nW/cm^2/sr -> W/m^2/sr
viirs_dat[i][viirs_dat[i] < 0] = 0.0
water_mask = MSD.Open("water_mask.hdf5")
for i, wm in enumerate(water_mask):
viirs_dat[i][wm == 0] = 0.0
circles = MSD.Open(dir_name + out_name + "_zone.hdf5")
zon_mask = np.empty(len(circles), dtype=object)
for i in range(len(zon_mask)):
zon_mask[i] = (np.arange(1,
len(zonfile) + 1)[:, None,
None] == circles[i])
a = np.deg2rad(angles)
mids = np.concatenate([[a[0]], np.mean([a[1:], a[:-1]], 0), [a[-1]]])
sinx = 2 * np.pi * (np.cos(mids[:-1]) - np.cos(mids[1:]))
# Pixel size in m^2
S = np.array([viirs_dat.pixel_size(i)**2 for i in range(len(viirs_dat))])
# Calculate zones lamps
zones = pt.make_zones(angles, lop, wav, spct, zonData, sources)
# phie = DNB * S / int( R ( rho/pi Gdown + Gup ) ) dlambda
Gdown = np.tensordot(zones[:, :, angles > 90],
sinx[angles > 90],
axes=([2], [0]))
Gup = (np.tensordot(
zones[:, :, angles < 70], sinx[angles < 70], axes=([2], [0])) /
sinx[angles < 70].sum())
integral = np.sum(viirs * (Gdown * refl / np.pi + Gup),
(1, 2)) * (wav[1] - wav[0])
phie = [
pt.safe_divide(
viirs_dat[i] * S[i],
np.sum(zon_mask[i] * integral[:, None, None], 0),
) for i in range(len(S))
]
ratio = [
np.tensordot(zones[..., ind], sinx, axes=([2], [0])).mean(-1)
for ind in bool_array
]
for n in range(n_bins):
r = [
np.sum(zon_mask[layer][:, None] * ratio[n][:, :, None, None], 0)
for layer in range(len(phie))
]
for i, s in enumerate(sources):
new = MSD.from_domain("domain.ini")
for layer in range(len(new)):
new[layer] = phie[layer] * r[layer][i]
new.save(dir_name + "%s_%g_lumlp_%s" % (out_name, x[n], s))
def inputs():
"""Prepares the executions inputs."""
print("Preparing the inputs for the experiment.")
dir_name = "Inputs/"
shutil.rmtree(dir_name, True)
os.makedirs(dir_name)
shutil.copy("inputs_params.in", dir_name + "inputs_params.in")
with open("inputs_params.in") as f:
params = yaml.safe_load(f)
if (params["zones_inventory"] is not None
and params["lamps_inventory"] is not None):
print("Validating the inventories.")
lamps = np.loadtxt(params["lamps_inventory"], usecols=[0, 1])
zones = np.loadtxt(params["zones_inventory"], usecols=[0, 1, 2])
zonData = pt.parse_inventory(params["zones_inventory"], 7)
hasLights = [sum(x[0] for x in z) != 0 for z in zonData]
circles = MSD.from_domain("domain.ini")
for dat, b in zip(zones, hasLights):
circles.set_circle((dat[0], dat[1]), dat[2] * 1000, b)
zones_ind = MSD.from_domain("domain.ini")
for i, dat in enumerate(zones, 1):
zones_ind.set_circle((dat[0], dat[1]), dat[2] * 1000, i)
failed = set()
for j, coords in enumerate(lamps, 1):
for i in range(len(circles)):
try:
col, row = circles._get_col_row(coords, i)
if circles[i][row, col] and col >= 0 and row >= 0:
zon_ind = zones_ind[i][row, col]
failed.add((j, coords[0], coords[1], zon_ind))
except IndexError:
continue
if len(failed):
for i, lat, lon, zon_ind in sorted(failed):
print(
"WARNING: Lamp #%d (%.06g,%.06g) falls within non-null zone #%d"
% (i, lat, lon, zon_ind))
raise SystemExit()
out_name = params["exp_name"]
if params["road_orientation"]:
print("Computing road orientation (Can be slow for large domains)")
from illum.street_orientation import street_orientation
with open("domain.ini") as f:
domain_params = yaml.safe_load(f)
srs = domain_params["srs"]
lats, lons = MSD.from_domain("domain.ini").get_obs_pos()
bearings = street_orientation(lats, lons, srs)
np.savetxt(dir_name + "/brng.lst", bearings, fmt="%g")
print("Loading photometry files.")
# Angular distribution (normalised to 1)
lop_files = glob("Lights/*.lop")
angles = np.arange(181, dtype=float)
lop = {
os.path.basename(s).rsplit(".", 1)[0].split("_", 1)[0]:
pt.load_lop(angles, s)
for s in lop_files
}
# Spectral distribution (normalised with scotopric vision to 1)
wav, viirs = np.loadtxt("Lights/viirs.dat", skiprows=1).T
viirs /= np.max(viirs)
wav, norm_spectrum = np.loadtxt("Lights/photopic.dat", skiprows=1).T
norm_spectrum /= np.max(norm_spectrum)
spct_files = glob("Lights/*.spct")
spct = {
os.path.basename(s).rsplit(".", 1)[0].split("_", 1)[0]:
pt.load_spct(wav, norm_spectrum, s)
for s in spct_files
}
print("Splitting in wavelengths bins.")
if os.path.isfile("spectral_bands.dat"):
bins = np.loadtxt("spectral_bands.dat", delimiter=",")
n_bins = bins.shape[0]
else:
n_bins = params["nb_bins"]
lmin = params["lambda_min"]
lmax = params["lambda_max"]
limits = np.linspace(lmin, lmax, n_bins + 1)
bins = np.stack([limits[:-1], limits[1:]], axis=1)
bool_array = (wav >= bins[:, 0:1]) & (wav < bins[:, 1:2])
x = bins.mean(1)
print("Interpolating reflectance.")
aster_files = glob("Lights/*.aster")
aster = {
os.path.basename(s).split(".", 1)[0]: np.loadtxt(s)
for s in aster_files
}
for type in aster:
wl, refl = aster[type].T
wl *= 1000.0
refl /= 100.0
aster[type] = np.interp(wav, wl, refl)
sum_coeffs = sum(params["reflectance"][type]
for type in params["reflectance"])
if sum_coeffs == 0:
sum_coeffs = 1.0
refl = sum(aster[type] * coeff / sum_coeffs
for type, coeff in params["reflectance"].items())
reflect = [np.mean(refl[mask]) for mask in bool_array]
with open(dir_name + "/refl.lst", "w") as zfile:
zfile.write("\n".join(["%.06g" % n for n in reflect]) + "\n")
print("Linking mie files.")
illumpath = os.path.dirname(illum.__path__[0])
shutil.copy2(
os.path.abspath(illumpath + "/Molecular_optics/MolecularAbs.txt"),
dir_name,
)
OPAC(x)
shutil.copy("srtm.hdf5", dir_name)
with open(dir_name + "/wav.lst", "w") as zfile:
zfile.write("".join("%g\n" % w for w in x))
if params["zones_inventory"] is not None:
dir_name = ".Inputs_zones/"
inv_name = params["zones_inventory"]
n_inv = 7
shutil.rmtree(dir_name, True)
os.makedirs(dir_name)
from_zones(
dir_name,
inv_name,
n_inv,
n_bins,
params,
out_name,
x,
lop,
angles,
wav,
spct,
viirs,
refl,
bool_array,
)
if params["lamps_inventory"] is not None:
dir_name = ".Inputs_lamps/"
shutil.rmtree(dir_name, True)
os.makedirs(dir_name)
from_lamps(
dir_name,
n_bins,
params,
out_name,
x,
lop,
angles,
wav,
spct,
viirs,
refl,
bool_array,
)
dir_name = "Inputs/"
print("Unifying inputs.")
lfiles = {fname.split(os.sep)[-1] for fname in glob(".Inputs_lamps/*")}
zfiles = {fname.split(os.sep)[-1] for fname in glob(".Inputs_zones/*")}
for fname in lfiles - zfiles:
shutil.move(os.path.join(".Inputs_lamps", fname), "Inputs")
for fname in zfiles - lfiles:
shutil.move(os.path.join(".Inputs_zones", fname), "Inputs")
for fname in zfiles & lfiles:
if "fctem" in fname:
shutil.move(os.path.join(".Inputs_lamps", fname), "Inputs")
elif fname.endswith(".lst"):
with open(os.path.join(".Inputs_lamps", fname)) as f:
ldat = f.readlines()
with open(os.path.join(".Inputs_zones", fname)) as f:
zdat = f.readlines()
with open(os.path.join("Inputs", fname), "w") as f:
f.write("".join(sorted(set(ldat + zdat))))
elif fname.endswith(".hdf5"):
ldat = MSD.Open(os.path.join(".Inputs_lamps", fname))
zdat = MSD.Open(os.path.join(".Inputs_zones", fname))
for i, dat in enumerate(ldat):
zdat[i][dat != 0] = dat[dat != 0]
zdat.save(os.path.join("Inputs", fname))
else:
print("WARNING: File %s not merged properly." % fname)
if "origin.hdf5" not in zfiles:
origin = MSD.from_domain("domain.ini")
origin.save("Inputs/origin")
shutil.rmtree(".Inputs_lamps", True)
shutil.rmtree(".Inputs_zones", True)
# Interpolation of the obstacles properties
defined = MSD.Open(dir_name + "origin.hdf5")
lights_file = dir_name + out_name + "_lights.hdf5"
if os.path.isfile(lights_file):
lights = MSD.Open(lights_file)
for i, layer in enumerate(lights):
defined[i] += layer
for geo in ["obsth", "obstd", "obstf", "altlp"]:
geometry = MSD.Open(dir_name + out_name + "_" + geo + ".hdf5")
for i, mask in enumerate(defined):
geometry[i] = (griddata(
points=np.where(mask),
values=geometry[i][mask.astype(bool)],
xi=tuple(np.ogrid[0:mask.shape[0], 0:mask.shape[1]]),
method="nearest",
) if mask.any() else np.zeros_like(geometry[i]))
geometry.save(dir_name + out_name + "_" + geo)
print("Done.")
def alternate(name, zones, lights):
"""Generates an alternate scenario at constant lumen.
This scenatio will be based on the content of the `Inputs` folder and
will be placed in a folder named `Inputs_NAME`.
"""
if zones is None and lights is None:
print("ERROR: At least one of 'zones' and 'lights' must be provided.")
raise SystemExit
dirname = "Inputs_%s/" % name
if os.path.exists(dirname):
shutil.rmtree(dirname)
os.makedirs(dirname)
with open("inputs_params.in") as f:
params = yaml.safe_load(f)
if zones is not None and lights is not None:
print("Validating the inventories.")
lamps = np.loadtxt(lights, usecols=[0, 1])
zones = np.loadtxt(params["zones_inventory"], usecols=[0, 1, 2])
zonData = pt.parse_inventory(zones, 0)
hasLights = [sum(x[0] for x in z) != 0 for z in zonData]
circles = MSD.from_domain("domain.ini")
for dat, b in zip(zones, hasLights):
circles.set_circle((dat[0], dat[1]), dat[2] * 1000, b)
zones_ind = MSD.from_domain("domain.ini")
for i, dat in enumerate(zones, 1):
zones_ind.set_circle((dat[0], dat[1]), dat[2] * 1000, i)
failed = set()
for j, coords in enumerate(lamps, 1):
for i in range(len(circles)):
try:
col, row = circles._get_col_row(coords, i)
if circles[i][row, col] and col >= 0 and row >= 0:
zon_ind = zones_ind[i][row, col]
failed.add((j, coords[0], coords[1], zon_ind))
except IndexError:
continue
if len(failed):
for i, lat, lon, zon_ind in sorted(failed):
print(
"WARNING: Lamp #%d (%.06g,%.06g) falls within non-null zone #%d"
% (i, lat, lon, zon_ind))
raise SystemExit()
shutil.copy("Inputs/inputs_params.in", dirname)
print("\nLoading data...")
# Angular distribution (normalised to 1)
lop_files = glob("Lights/*.lop")
angles = np.arange(181, dtype=float)
lop = {
os.path.basename(s).rsplit(".", 1)[0].split("_", 1)[0]:
pt.load_lop(angles, s)
for s in lop_files
}
# Spectral distribution (normalised with scotopric vision to 1)
wav, viirs = np.loadtxt("Lights/viirs.dat", skiprows=1).T
viirs = pt.spct_norm(wav, viirs)
scotopic = pt.load_spct(wav, np.ones(wav.shape), "Lights/scotopic.dat", 1)
photopic = pt.load_spct(wav, np.ones(wav.shape), "Lights/photopic.dat", 1)
ratio_ps = 1.0
norm_spectrum = ratio_ps * photopic + (1 - ratio_ps) * scotopic
spct_files = glob("Lights/*.spct")
spct = {
os.path.basename(s).rsplit(".", 1)[0].split("_", 1)[0]:
pt.load_spct(wav, norm_spectrum, s)
for s in spct_files
}
# Make bins
if os.path.isfile("spectral_bands.dat"):
bins = np.loadtxt("spectral_bands.dat", delimiter=",")
n_bins = bins.shape[0]
else:
n_bins = params["nb_bins"]
lmin = params["lambda_min"]
lmax = params["lambda_max"]
limits = np.linspace(lmin, lmax, n_bins + 1)
bins = np.stack([limits[:-1], limits[1:]], axis=1)
bool_array = (wav >= bins[:, 0:1]) & (wav < bins[:, 1:2])
x = bins.mean(1).tolist()
out_name = params["exp_name"]
asper_files = glob("Lights/*.aster")
asper = {
os.path.basename(s).split(".", 1)[0]: np.loadtxt(s)
for s in asper_files
}
for type in asper:
wl, refl = asper[type].T
wl *= 1000.0
refl /= 100.0
asper[type] = interp(wl, refl, bounds_error=False, fill_value=0.0)(wav)
sum_coeffs = sum(params["reflectance"][type]
for type in params["reflectance"])
if sum_coeffs == 0:
sum_coeffs = 1.0
refl = sum(asper[type] * coeff / sum_coeffs
for type, coeff in params["reflectance"].items())
reflect = [np.mean(refl[mask]) for mask in bool_array]
with open(dirname + "/refl.lst", "w") as zfile:
zfile.write("\n".join(["%.06g" % n for n in reflect]) + "\n")
# Photopic/scotopic spectrum
nspct = ratio_ps * photopic + (1 - ratio_ps) * scotopic
nspct = nspct / np.sum(nspct)
nspct = [np.mean(nspct[mask]) for mask in bool_array]
for aero_file in glob("Inputs/*.txt"):
shutil.copy(aero_file, aero_file.replace("Inputs", dirname))
shutil.copy("srtm.hdf5", dirname)
with open(dirname + "/wav.lst", "w") as zfile:
zfile.write("\n".join(map(str, x)) + "\n")
if params["zones_inventory"] is not None:
dir_name = ".Inputs_zones/"
inv_name = params["zones_inventory"]
n_inv = 7
shutil.rmtree(dir_name, True)
os.makedirs(dir_name)
from_zones(
dir_name,
inv_name,
n_inv,
n_bins,
params,
out_name,
x,
lop,
angles,
wav,
spct,
viirs,
refl,
bool_array,
)
oldlumlp = MSD.from_domain("domain.ini")
for fname in glob("Inputs/*lumlp*"):
ds = MSD.Open(fname)
wl = int(fname.split("_")[1])
for i, dat in enumerate(ds):
oldlumlp[i] += dat * nspct[x.index(wl)] * dl
newlumlp = MSD.from_domain("domain.ini")
for fname in glob(os.path.join(dir_name, "*lumlp*")):
ds = MSD.Open(fname)
wl = int(fname.split("_")[2])
for i, dat in enumerate(ds):
newlumlp[i] += dat * nspct[x.index(wl)] * dl
ratio = MSD.from_domain("domain.ini")
for i in range(len(ratio)):
ratio[i] = pt.safe_divide(oldlumlp[i], newlumlp[i])
for fname in glob(os.path.join(dir_name, "*lumlp*")):
ds = MSD.Open(fname)
for i, dat in enumerate(ratio):
ds[i] *= dat
ds.save(fname)
if params["lamps_inventory"] is not None:
dir_name = ".Inputs_lamps/"
shutil.rmtree(dir_name, True)
os.makedirs(dir_name)
from_lamps(
dir_name,
n_bins,
params,
out_name,
x,
lop,
angles,
wav,
spct,
viirs,
refl,
bool_array,
)
print("Unifying inputs.")
lfiles = {fname.split(os.sep)[-1] for fname in glob(".Inputs_lamps/*")}
zfiles = {fname.split(os.sep)[-1] for fname in glob(".Inputs_zones/*")}
for fname in lfiles - zfiles:
shutil.move(os.path.join(".Inputs_lamps", fname), dirname)
for fname in zfiles - lfiles:
shutil.move(os.path.join(".Inputs_zones", fname), dirname)
for fname in zfiles & lfiles:
if "fctem" in fname:
shutil.move(os.path.join(".Inputs_lamps", fname), dirname)
elif fname.endswith(".lst"):
with open(os.path.join(".Inputs_lamps", fname)) as f:
ldat = f.readlines()
with open(os.path.join(".Inputs_zones", fname)) as f:
zdat = f.readlines()
with open(os.path.join(dirname, fname), "w") as f:
f.write("".join(sorted(set(ldat + zdat))))
elif fname.endswith(".hdf5"):
ldat = MSD.Open(os.path.join(".Inputs_lamps", fname))
zdat = MSD.Open(os.path.join(".Inputs_zones", fname))
for i, dat in enumerate(ldat):
zdat[i][dat != 0] = dat[dat != 0]
zdat.save(os.path.join(dirname, fname))
else:
print("WARNING: File %s not merged properly." % fname)
if "origin.hdf5" not in zfiles:
origin = MSD.from_domain("domain.ini")
origin.save(dirname + "/origin")
shutil.rmtree(".Inputs_lamps", True)
shutil.rmtree(".Inputs_zones", True)
print("Done.")
def save(params, data, dstname, scale_factor=1.0):
scaled_data = [d * scale_factor for d in data]
ds = MSD.from_domain(params, scaled_data)
ds.save(dstname)