def process_ignitions(igns,bounds,time=None):
"""
Process ignitions the same way than satellite data.
:param igns: ([lons],[lats],[dates]) where lons and lats in degrees and dates in ESMF format
:param bounds: coordinate bounding box filtering to
:return ignitions: dictionary with all the information from each ignition similar to satellite data
Developed in Python 2.7.15 :: Anaconda 4.5.10, on MACINTOSH.
Angel Farguell ([email protected]), 2019-05-29
"""
# Time interval
if time:
interval_datetime = map(time_iso2datetime,time)
# prefix of the elements in the dictionary
prefix = "IGN"
# initializing dictionary
ignitions = dict({})
# scan and track dimensions of the observation (in km)
scan = 1.
track = 1.
# confidences
conf_fire = 100.
# for each ignition
for lon, lat, time_iso in zip(igns[0],igns[1],igns[2]):
try:
# take coordinates
lon = np.array(lon)
lat = np.array(lat)
# look if coordinates in bounding box
mask = np.logical_and(np.logical_and(np.logical_and(lon >= bounds[0], lon <= bounds[1]),lat >= bounds[2]),lat <= bounds[3])
if not mask.sum():
continue
lons = lon[mask]
lats = lat[mask]
# get time elements
time_num = time_iso2num(time_iso)
time_datetime = time_iso2datetime(time_iso)
# skip if not inside interval (only if time is determined)
if time and (time_datetime < interval_datetime[0] or time_datetime > interval_datetime[1]):
print 'Perimeter from %s skipped, not inside the simulation interval!' % file
continue
time_data = '_A%04d%03d_%02d%02d_%02d' % (time_datetime.year, time_datetime.timetuple().tm_yday,
time_datetime.hour, time_datetime.minute, time_datetime.second)
acq_date = '%04d-%02d-%02d' % (time_datetime.year, time_datetime.month, time_datetime.day)
acq_time = '%02d%02d' % (time_datetime.hour, time_datetime.minute)
except Exception as e:
print 'Error: bad ignition %s specified.' % igns
print 'Exception: %s.' % e
sys.exit(1)
# no nofire detection
lon_nofire = np.array([])
lat_nofire = np.array([])
conf_nofire = np.array([])
# update ignitions dictionary
ignitions.update({prefix + time_data: Dict({'lon': lons, 'lat': lats,
'fire': np.array(9*np.ones(lons.shape)), 'conf_fire': np.array(conf_fire*np.ones(lons.shape)),
'lon_fire': lons, 'lat_fire': lats, 'lon_nofire': lon_nofire, 'lat_nofire': lat_nofire,
'scan_fire': scan*np.ones(lons.shape), 'track_fire': track*np.ones(lons.shape),
'conf_nofire' : conf_nofire, 'scan_nofire': scan*np.ones(lon_nofire.shape),
'track_nofire': track*np.ones(lon_nofire.shape), 'time_iso': time_iso,
'time_num': time_num, 'acq_date': acq_date, 'acq_time': acq_time})})
return ignitions
def process_infrared_perimeters(dst,bounds,time=None,maxp=500,ngrid=50,plot=False,gen_polys=False):
"""
Process infrared perimeters the same way than satellite data.
:param dst: path to kml perimeter files
:param bounds: coordinate bounding box filtering to
:param time: optional, time interval in ISO
:param maxp: optional, maximum number of points for each perimeter
:param ngrid: optional, number of nodes for the grid of in/out nodes at each direction
:param plot: optional, boolean to plot or not the result at each perimeter iteration
:return perimeters: dictionary with all the information from each perimeter similar to satellite data
Developed in Python 2.7.15 :: Anaconda 4.5.10, on MACINTOSH.
Angel Farguell ([email protected]), 2019-05-29
"""
# Time interval
if time:
interval_datetime = map(time_iso2datetime,time)
# list of kml files in 'dst' path
files = glob.glob(osp.join(dst, '*.kml'))
# prefix of the elements in the dictionary
prefix = "PER"
# initializing dictionary
perimeters = Dict({})
# scan and track dimensions of the observation (in km)
scan = .5
track = .5
# confidences
conf_fire = 100.
conf_nofire = 70.
# Creating grid where to evaluate in/out of the perimeter
[X,Y] = np.meshgrid(np.linspace(bounds[0],bounds[1],ngrid),np.linspace(bounds[2],bounds[3],ngrid))
XX = np.c_[np.ravel(X),np.ravel(Y)]
# if any file
if files:
# for each file
for file in files:
print 'Retrieving perimeters from %s' % file
try:
# open the file
f = open(file,"r")
# read all the lines of the file
f_str = ''.join(f.readlines())
# close the file
f.close()
except Exception as e:
print 'Error: exception when opening file %s, %s' % (file,e.message)
sys.exit(1)
try:
# Read name and get time elements
# read name of the file
name = re.findall(r'<name>(.*?)</name>',f_str,re.DOTALL)[0]
# read date of the perimeter
date = re.match(r'.* ([0-9]+)-([0-9]+)-([0-9]+) ([0-9]{2})([0-9]{2})',name).groups()
date = (date[2],date[0],date[1],date[3],date[4])
# create ISO time of the date
time_iso = '%04d-%02d-%02dT%02d:%02d:00' % tuple([ int(d) for d in date ])
# create numerical time from the ISO time
time_num = time_iso2num(time_iso)
# create datetime element from the ISO time
time_datetime = time_iso2datetime(time_iso)
# skip if not inside interval (only if time is determined)
if time and (time_datetime < interval_datetime[0] or time_datetime > interval_datetime[1]):
print 'Perimeter from %s skipped, not inside the simulation interval!' % file
continue
# create time stamp
time_data = '_A%04d%03d_%02d%02d' % (time_datetime.year, time_datetime.timetuple().tm_yday,
time_datetime.hour, time_datetime.minute)
# create acquisition date
acq_date = '%04d-%02d-%02d' % (time_datetime.year, time_datetime.month, time_datetime.day)
# create acquisition time
acq_time = '%02d%02d' % (time_datetime.hour, time_datetime.minute)
# Get the coordinates of all the perimeters
# regex of the polygons (or perimeters)
polygons = re.findall(r'<Polygon>(.*?)</Polygon>',f_str,re.DOTALL)
# for each polygon, outer boundary
outer_buckets = [re.findall(r'<outerBoundaryIs>(.*?)</outerBoundaryIs>',p,re.DOTALL)[0] for p in polygons]
# for each outer polygon, regex of the coordinates
buckets = [re.split(r'\s',re.findall(r'<coordinates>(.*?)</coordinates>',p,re.DOTALL)[0])[1:] for p in outer_buckets]
# array of arrays with each outer polygon coordinates
outer_coordinates = [[np.array(re.split(',',b)[0:2]).astype(float) for b in bucket if b is not ''] for bucket in buckets]
# for each polygon, inner boundary
inner_buckets = [re.findall(r'<innerBoundaryIs>(.*?)</innerBoundaryIs>',p,re.DOTALL)[0] if re.findall(r'<innerBoundaryIs>(.*?)</innerBoundaryIs>',p,re.DOTALL) else '' for p in polygons]
# for each inner polygon, regex of the coordinates
buckets = [re.split(r'\s',re.findall(r'<coordinates>(.*?)</coordinates>',p,re.DOTALL)[0])[1:] if p != '' else '' for p in inner_buckets]
# array of arrays with each inner polygon coordinates
inner_coordinates = [[np.array(re.split(',',b)[0:2]).astype(float) for b in bucket if b is not ''] for bucket in buckets]
except Exception as e:
print 'Error: file %s has not proper structure.' % file
print 'Exception: %s.' % e
sys.exit(1)
# Plot perimeter
if plot:
plt.ion()
for outer in outer_coordinates:
if len(outer):
x = np.array(outer)[:,0]
y = np.array(outer)[:,1]
plt.plot(x,y,'gx')
for inner in inner_coordinates:
if len(inner):
x = np.array(outer)[:,0]
y = np.array(outer)[:,1]
plt.plot(x,y,'rx')
# Create paths for each polygon (outer and inner)
outer_paths,inner_paths = process_paths(outer_coordinates,inner_coordinates)
if len(outer_paths):
# compute mask of coordinates inside outer polygons
outer_mask = np.logical_or.reduce([path.contains_points(XX) for path in outer_paths if path])
# compute mask of coordinates inside inner polygons
inner_mask = np.logical_or.reduce([path.contains_points(XX) for path in inner_paths if path])
# mask inside outer polygons and outside inner polygons
inmask = outer_mask * ~inner_mask
# upper and lower bounds arifitial from in/out polygon
up_arti = XX[inmask]
lon_fire = np.array([up[0] for up in up_arti])
lat_fire = np.array([up[1] for up in up_arti])
low_arti = XX[~inmask]
lon_nofire = np.array([low[0] for low in low_arti])
lat_nofire = np.array([low[1] for low in low_arti])
# take a coarsening of the outer polygons
coordinates = []
for k,coord in enumerate(outer_coordinates):
if len(coord) > maxp:
coarse = len(coord)/maxp
if coarse > 0:
coordinates += [coord[ind] for ind in np.concatenate(([0],range(len(coord))[coarse:-coarse:coarse]))]
if coordinates:
# append perimeter nodes
lon_fire = np.append(lon_fire,np.array(coordinates)[:,0])
lat_fire = np.append(lat_fire,np.array(coordinates)[:,1])
# create general arrays
lon = np.concatenate((lon_nofire,lon_fire))
lat = np.concatenate((lat_nofire,lat_fire))
fire = np.concatenate((5*np.ones(lon_nofire.shape),9*np.ones(lon_fire.shape)))
# mask in bounds
mask = np.logical_and(np.logical_and(np.logical_and(lon >= bounds[0],lon <= bounds[1]),lat >= bounds[2]),lat <= bounds[3])
if not mask.sum():
break
lons = lon[mask]
lats = lat[mask]
fires = fire[mask]
# plot results
if plot:
plt.plot(lons[fires==5],lats[fires==5],'g.')
plt.plot(lons[fires==9],lats[fires==9],'r.')
plt.show()
plt.pause(.5)
plt.cla()
else:
lons = np.array([])
lats = np.array([])
fires = np.array([])
if gen_polys:
from shapely.geometry import Polygon
from shapely.ops import transform
from functools import partial
import pyproj
proj4_wgs84 = '+proj=latlong +ellps=WGS84 +datum=WGS84 +units=degree +no_defs'
proj4_moll = '+proj=moll +lon_0=0 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m +no_defs'
proj = partial(pyproj.transform, pyproj.Proj(init='epsg:4326'),
pyproj.Proj(proj4_moll))
polys = []
for n,outer in enumerate(outer_coordinates):
x = np.array(outer)[:,0]
y = np.array(outer)[:,1]
shape = Polygon([(l[0],l[1]) for l in zip(x,y)]).buffer(0)
inner = np.array(inner_coordinates[n])
if len(inner):
if len(inner.shape) > 2:
for h in inner_coordinates[n]:
xh = np.array(h)[:,0]
yh = np.array(h)[:,1]
else:
xh = np.array(inner)[:,0]
yh = np.array(inner)[:,1]
shapeh = Polygon([(l[0],l[1]) for l in zip(xh,yh)]).buffer(0)
shape.difference(shapeh)
poly = transform(proj,shape)
polys.append(poly)
idn = prefix + time_data
if idn in perimeters.keys():
idn = idn + '_2'
# update perimeters dictionary
perimeters.update({idn: Dict({'file': file, 'lon': lons, 'lat': lats,
'fire': fires, 'conf_fire': np.array(conf_fire*np.ones(lons[fires==9].shape)),
'lon_fire': lons[fires==9], 'lat_fire': lats[fires==9], 'lon_nofire': lons[fires==5], 'lat_nofire': lats[fires==5],
'scan_fire': scan*np.ones(lons[fires==9].shape), 'track_fire': track*np.ones(lons[fires==9].shape),
'conf_nofire': np.array(conf_nofire*np.ones(lons[fires==5].shape)),
'scan_nofire': scan*np.ones(lons[fires==5].shape), 'track_nofire': track*np.ones(lons[fires==9].shape),
'time_iso': time_iso, 'time_num': time_num, 'acq_date': acq_date, 'acq_time': acq_time})})
if gen_polys:
perimeters[idn].update({'polys': polys})
else:
print 'Warning: No KML files in the path specified'
perimeters = []
return perimeters
def process_tign_g(lon,
lat,
tign_g,
ctime,
scale,
time_num,
epsilon=5,
plot=False):
"""
Process forecast from lon, lat, and tign_g as 3D data points
:param lon: array of longitudes
:param lat: array of latitudes
:param tign_g: array of fire arrival time
:param ctime: time char in wrfout of the last fire arrival time
:param scale: numerical time scale in seconds of start and end of simulation
:param time_num: numerical time interval in seconds of start and end of simulation
:param epsilon: optional, epsilon in seconds to define the separation of artificial data
:param plot: optional, plotting results
Developed in Python 2.7.15 :: Anaconda 4.5.10, on MACINTOSH.
Angel Farguell ([email protected]) and James Haley, 2019-06-05
"""
# confidences
conf_ground = 25.
conf_fire = 25.
# ctime transformations
ctime_iso = ctime.replace('_', 'T')
ctime_datetime = time_iso2datetime(ctime_iso)
tscale = 3600. * 24.
# tign_g as data points smaller than maximum tign_g
t1d = np.ravel(tign_g)
etime = t1d.max()
mask = t1d < etime
x1d = np.ravel(lon)[mask]
y1d = np.ravel(lat)[mask]
t1d = t1d[mask]
# compute time as days from start of the simulation
tt = (np.array([
time_iso2num(
time_datetime2iso(ctime_datetime -
timedelta(seconds=float(etime - T))))
for T in t1d
]) - scale[0]) / tscale
# define forecast artificial data points
fground = np.c_[x1d.ravel(), y1d.ravel(), tt.ravel() - epsilon / tscale]
ffire = np.c_[x1d.ravel(), y1d.ravel(), tt.ravel() + epsilon / tscale]
# coarsening
maxsize = 5e3
coarsening = np.int(1 + len(ffire) / maxsize)
fground = fground[::coarsening, :]
ffire = ffire[::coarsening, :]
# define output data
X = np.concatenate((fground, ffire))
y = np.concatenate((-np.ones(len(fground)), np.ones(len(ffire))))
c = np.concatenate(
(conf_ground * np.ones(len(fground)), conf_fire * np.ones(len(ffire))))
# plot results
if plot:
from contline import get_contour_verts
from contour2kml import contour2kml
tt = np.array([
time_iso2num(
time_datetime2iso(ctime_datetime -
timedelta(seconds=float(etime - T))))
for T in np.ravel(tign_g)
])
data = get_contour_verts(lon,
lat,
np.reshape(tt, tign_g.shape),
time_num,
contour_dt_hours=6,
contour_dt_init=24,
contour_dt_final=12)
contour2kml(data, 'perimeters_forecast.kml')
col = [(0, .5, 0), (.5, 0, 0)]
cm_GR = colors.LinearSegmentedColormap.from_list('GrRd', col, N=2)
fig = plt.figure()
ax = fig.gca(projection='3d')
ax.scatter(X[:, 0],
X[:, 1],
X[:, 2],
c=y,
cmap=cm_GR,
s=1,
alpha=.5,
vmin=y.min(),
vmax=y.max())
ax.set_xlabel("Longitude")
ax.set_ylabel("Latitude")
ax.set_zlabel("Fire arrival time [days]")
plt.show()
return X, y, c
def process_tign_g_slices(lon,
lat,
tign_g,
bounds,
ctime,
dx,
dy,
wrfout_file='',
dt_for=3600.,
plot=False):
"""
Process forecast as slices on time from lon, lat, and tign_g
:param lon: array of longitudes
:param lat: array of latitudes
:param tign_g: array of fire arrival time
:param bounds: coordinate bounding box filtering to
:param ctime: time char in wrfout of the last fire arrival time
:param dx,dy: data resolution in meters
:param dt_for: optional, temporal resolution to get the fire arrival time from in seconds
:param wrfout_file: optional, to get the name of the file in the dictionary
Developed in Python 2.7.15 :: Anaconda 4.5.10, on MACINTOSH.
Angel Farguell ([email protected]) and James Haley, 2019-06-05
"""
if plot:
fig = plt.figure()
# prefix of the elements in the dictionary
prefix = "FOR"
# confidences
conf_fire = 50
conf_nofire = 10
# margin percentage
margin = .5
# scan and track dimensions of the observation (in km)
scan = dx / 1000.
track = dy / 1000.
# initializing dictionary
forecast = Dict({})
# ctime transformations
ctime_iso = ctime.replace('_', 'T')
ctime_datetime = time_iso2datetime(ctime_iso)
# mask coordinates to bounding box
mask = np.logical_and(
np.logical_and(np.logical_and(lon >= bounds[0], lon <= bounds[1]),
lat >= bounds[2]), lat <= bounds[3])
# create a square subset of fire arrival time less than the maximum
mtign = np.logical_and(mask, tign_g < tign_g.max())
mlon = lon[mtign]
mlat = lat[mtign]
mlenlon = margin * (mlon.max() - mlon.min())
mlenlat = margin * (mlat.max() - mlat.min())
sbounds = (mlon.min() - mlenlon, mlon.max() + mlenlon,
mlat.min() - mlenlat, mlat.max() + mlenlat)
smask = np.logical_and(
np.logical_and(np.logical_and(lon >= sbounds[0], lon <= sbounds[1]),
lat >= sbounds[2]), lat <= sbounds[3])
# times to get fire arrival time from
TT = np.arange(tign_g.min() - 3 * dt_for, tign_g.max(), dt_for)[0:-1]
for T in TT:
# fire arrival time to datetime
time_datetime = ctime_datetime - timedelta(
seconds=float(tign_g.max() - T)
) # there is an error of about 10 seconds (wrfout not ending at exact time of simulation)
# create ISO time of the date
time_iso = time_datetime2iso(time_datetime)
# create numerical time from the ISO time
time_num = time_iso2num(time_iso)
# create time stamp
time_data = '_A%04d%03d_%02d%02d_%02d' % (
time_datetime.year, time_datetime.timetuple().tm_yday,
time_datetime.hour, time_datetime.minute, time_datetime.second)
# create acquisition date
acq_date = '%04d-%02d-%02d' % (time_datetime.year, time_datetime.month,
time_datetime.day)
# create acquisition time
acq_time = '%02d%02d' % (time_datetime.hour, time_datetime.minute)
print 'Retrieving forecast from %s' % time_iso
# masks
fire = tign_g <= T
nofire = np.logical_and(tign_g > T,
np.logical_or(tign_g != tign_g.max(), smask))
# coordinates masked
lon_fire = lon[np.logical_and(mask, fire)]
lat_fire = lat[np.logical_and(mask, fire)]
lon_nofire = lon[np.logical_and(mask, nofire)]
lat_nofire = lat[np.logical_and(mask, nofire)]
# create general arrays
lons = np.concatenate((lon_nofire, lon_fire))
lats = np.concatenate((lat_nofire, lat_fire))
fires = np.concatenate(
(5 * np.ones(lon_nofire.shape), 9 * np.ones(lon_fire.shape)))
# plot results
if plot:
plt.ion()
plt.cla()
plt.plot(lons[fires == 5], lats[fires == 5], 'g.')
plt.plot(lons[fires == 9], lats[fires == 9], 'r.')
plt.pause(.0001)
plt.show()
# update perimeters dictionary
forecast.update({
prefix + time_data:
Dict({
'file':
wrfout_file,
'time_tign_g':
T,
'lon':
lons,
'lat':
lats,
'fire':
fires,
'conf_fire':
np.array(conf_fire * np.ones(lons[fires == 9].shape)),
'lon_fire':
lons[fires == 9],
'lat_fire':
lats[fires == 9],
'lon_nofire':
lons[fires == 5],
'lat_nofire':
lats[fires == 5],
'scan_fire':
scan * np.ones(lons[fires == 9].shape),
'track_fire':
track * np.ones(lons[fires == 9].shape),
'conf_nofire':
np.array(conf_nofire * np.ones(lons[fires == 5].shape)),
'scan_nofire':
scan * np.ones(lons[fires == 5].shape),
'track_nofire':
track * np.ones(lons[fires == 9].shape),
'time_iso':
time_iso,
'time_num':
time_num,
'acq_date':
acq_date,
'acq_time':
acq_time
})
})
return forecast
sl.save(f, 'forecast')
else:
from infrared_perimeters import process_ignitions
from setup import process_detections
dst = 'ideal_test'
plot = False
ideal = sl.load(dst)
kk = 4
data = process_tign_g_slices(ideal['lon'][::kk, ::kk],
ideal['lat'][::kk, ::kk],
ideal['tign_g'][::kk, ::kk],
ideal['bounds'],
ideal['ctime'],
ideal['dx'],
ideal['dy'],
wrfout_file='ideal',
dt_for=ideal['dt'],
plot=plot)
if 'point' in ideal.keys():
p = [[ideal['point'][0]], [ideal['point'][1]], [ideal['point'][2]]]
data.update(process_ignitions(p, ideal['bounds']))
elif 'points' in ideal.keys():
p = [[point[0] for point in ideal['points']],
[point[1] for point in ideal['points']],
[point[2] for point in ideal['points']]]
data.update(process_ignitions(p, ideal['bounds']))
etime = time_iso2num(ideal['ctime'].replace('_', 'T'))
time_num_int = (etime - ideal['tign_g'].max(), etime)
sl.save((data, ideal['lon'], ideal['lat'], time_num_int), 'data')
process_detections(data, ideal['lon'], ideal['lat'], time_num_int)