def process_ignitions(igns,bounds,time=None): """ Process ignitions the same way than satellite data. :param igns: ([lons],[lats],[dates]) where lons and lats in degrees and dates in ESMF format :param bounds: coordinate bounding box filtering to :return ignitions: dictionary with all the information from each ignition similar to satellite data Developed in Python 2.7.15 :: Anaconda 4.5.10, on MACINTOSH. Angel Farguell ([email protected]), 2019-05-29 """ # Time interval if time: interval_datetime = map(time_iso2datetime,time) # prefix of the elements in the dictionary prefix = "IGN" # initializing dictionary ignitions = dict({}) # scan and track dimensions of the observation (in km) scan = 1. track = 1. # confidences conf_fire = 100. # for each ignition for lon, lat, time_iso in zip(igns[0],igns[1],igns[2]): try: # take coordinates lon = np.array(lon) lat = np.array(lat) # look if coordinates in bounding box mask = np.logical_and(np.logical_and(np.logical_and(lon >= bounds[0], lon <= bounds[1]),lat >= bounds[2]),lat <= bounds[3]) if not mask.sum(): continue lons = lon[mask] lats = lat[mask] # get time elements time_num = time_iso2num(time_iso) time_datetime = time_iso2datetime(time_iso) # skip if not inside interval (only if time is determined) if time and (time_datetime < interval_datetime[0] or time_datetime > interval_datetime[1]): print 'Perimeter from %s skipped, not inside the simulation interval!' % file continue time_data = '_A%04d%03d_%02d%02d_%02d' % (time_datetime.year, time_datetime.timetuple().tm_yday, time_datetime.hour, time_datetime.minute, time_datetime.second) acq_date = '%04d-%02d-%02d' % (time_datetime.year, time_datetime.month, time_datetime.day) acq_time = '%02d%02d' % (time_datetime.hour, time_datetime.minute) except Exception as e: print 'Error: bad ignition %s specified.' % igns print 'Exception: %s.' % e sys.exit(1) # no nofire detection lon_nofire = np.array([]) lat_nofire = np.array([]) conf_nofire = np.array([]) # update ignitions dictionary ignitions.update({prefix + time_data: Dict({'lon': lons, 'lat': lats, 'fire': np.array(9*np.ones(lons.shape)), 'conf_fire': np.array(conf_fire*np.ones(lons.shape)), 'lon_fire': lons, 'lat_fire': lats, 'lon_nofire': lon_nofire, 'lat_nofire': lat_nofire, 'scan_fire': scan*np.ones(lons.shape), 'track_fire': track*np.ones(lons.shape), 'conf_nofire' : conf_nofire, 'scan_nofire': scan*np.ones(lon_nofire.shape), 'track_nofire': track*np.ones(lon_nofire.shape), 'time_iso': time_iso, 'time_num': time_num, 'acq_date': acq_date, 'acq_time': acq_time})}) return ignitions
def process_infrared_perimeters(dst,bounds,time=None,maxp=500,ngrid=50,plot=False,gen_polys=False): """ Process infrared perimeters the same way than satellite data. :param dst: path to kml perimeter files :param bounds: coordinate bounding box filtering to :param time: optional, time interval in ISO :param maxp: optional, maximum number of points for each perimeter :param ngrid: optional, number of nodes for the grid of in/out nodes at each direction :param plot: optional, boolean to plot or not the result at each perimeter iteration :return perimeters: dictionary with all the information from each perimeter similar to satellite data Developed in Python 2.7.15 :: Anaconda 4.5.10, on MACINTOSH. Angel Farguell ([email protected]), 2019-05-29 """ # Time interval if time: interval_datetime = map(time_iso2datetime,time) # list of kml files in 'dst' path files = glob.glob(osp.join(dst, '*.kml')) # prefix of the elements in the dictionary prefix = "PER" # initializing dictionary perimeters = Dict({}) # scan and track dimensions of the observation (in km) scan = .5 track = .5 # confidences conf_fire = 100. conf_nofire = 70. # Creating grid where to evaluate in/out of the perimeter [X,Y] = np.meshgrid(np.linspace(bounds[0],bounds[1],ngrid),np.linspace(bounds[2],bounds[3],ngrid)) XX = np.c_[np.ravel(X),np.ravel(Y)] # if any file if files: # for each file for file in files: print 'Retrieving perimeters from %s' % file try: # open the file f = open(file,"r") # read all the lines of the file f_str = ''.join(f.readlines()) # close the file f.close() except Exception as e: print 'Error: exception when opening file %s, %s' % (file,e.message) sys.exit(1) try: # Read name and get time elements # read name of the file name = re.findall(r'<name>(.*?)</name>',f_str,re.DOTALL)[0] # read date of the perimeter date = re.match(r'.* ([0-9]+)-([0-9]+)-([0-9]+) ([0-9]{2})([0-9]{2})',name).groups() date = (date[2],date[0],date[1],date[3],date[4]) # create ISO time of the date time_iso = '%04d-%02d-%02dT%02d:%02d:00' % tuple([ int(d) for d in date ]) # create numerical time from the ISO time time_num = time_iso2num(time_iso) # create datetime element from the ISO time time_datetime = time_iso2datetime(time_iso) # skip if not inside interval (only if time is determined) if time and (time_datetime < interval_datetime[0] or time_datetime > interval_datetime[1]): print 'Perimeter from %s skipped, not inside the simulation interval!' % file continue # create time stamp time_data = '_A%04d%03d_%02d%02d' % (time_datetime.year, time_datetime.timetuple().tm_yday, time_datetime.hour, time_datetime.minute) # create acquisition date acq_date = '%04d-%02d-%02d' % (time_datetime.year, time_datetime.month, time_datetime.day) # create acquisition time acq_time = '%02d%02d' % (time_datetime.hour, time_datetime.minute) # Get the coordinates of all the perimeters # regex of the polygons (or perimeters) polygons = re.findall(r'<Polygon>(.*?)</Polygon>',f_str,re.DOTALL) # for each polygon, outer boundary outer_buckets = [re.findall(r'<outerBoundaryIs>(.*?)</outerBoundaryIs>',p,re.DOTALL)[0] for p in polygons] # for each outer polygon, regex of the coordinates buckets = [re.split(r'\s',re.findall(r'<coordinates>(.*?)</coordinates>',p,re.DOTALL)[0])[1:] for p in outer_buckets] # array of arrays with each outer polygon coordinates outer_coordinates = [[np.array(re.split(',',b)[0:2]).astype(float) for b in bucket if b is not ''] for bucket in buckets] # for each polygon, inner boundary inner_buckets = [re.findall(r'<innerBoundaryIs>(.*?)</innerBoundaryIs>',p,re.DOTALL)[0] if re.findall(r'<innerBoundaryIs>(.*?)</innerBoundaryIs>',p,re.DOTALL) else '' for p in polygons] # for each inner polygon, regex of the coordinates buckets = [re.split(r'\s',re.findall(r'<coordinates>(.*?)</coordinates>',p,re.DOTALL)[0])[1:] if p != '' else '' for p in inner_buckets] # array of arrays with each inner polygon coordinates inner_coordinates = [[np.array(re.split(',',b)[0:2]).astype(float) for b in bucket if b is not ''] for bucket in buckets] except Exception as e: print 'Error: file %s has not proper structure.' % file print 'Exception: %s.' % e sys.exit(1) # Plot perimeter if plot: plt.ion() for outer in outer_coordinates: if len(outer): x = np.array(outer)[:,0] y = np.array(outer)[:,1] plt.plot(x,y,'gx') for inner in inner_coordinates: if len(inner): x = np.array(outer)[:,0] y = np.array(outer)[:,1] plt.plot(x,y,'rx') # Create paths for each polygon (outer and inner) outer_paths,inner_paths = process_paths(outer_coordinates,inner_coordinates) if len(outer_paths): # compute mask of coordinates inside outer polygons outer_mask = np.logical_or.reduce([path.contains_points(XX) for path in outer_paths if path]) # compute mask of coordinates inside inner polygons inner_mask = np.logical_or.reduce([path.contains_points(XX) for path in inner_paths if path]) # mask inside outer polygons and outside inner polygons inmask = outer_mask * ~inner_mask # upper and lower bounds arifitial from in/out polygon up_arti = XX[inmask] lon_fire = np.array([up[0] for up in up_arti]) lat_fire = np.array([up[1] for up in up_arti]) low_arti = XX[~inmask] lon_nofire = np.array([low[0] for low in low_arti]) lat_nofire = np.array([low[1] for low in low_arti]) # take a coarsening of the outer polygons coordinates = [] for k,coord in enumerate(outer_coordinates): if len(coord) > maxp: coarse = len(coord)/maxp if coarse > 0: coordinates += [coord[ind] for ind in np.concatenate(([0],range(len(coord))[coarse:-coarse:coarse]))] if coordinates: # append perimeter nodes lon_fire = np.append(lon_fire,np.array(coordinates)[:,0]) lat_fire = np.append(lat_fire,np.array(coordinates)[:,1]) # create general arrays lon = np.concatenate((lon_nofire,lon_fire)) lat = np.concatenate((lat_nofire,lat_fire)) fire = np.concatenate((5*np.ones(lon_nofire.shape),9*np.ones(lon_fire.shape))) # mask in bounds mask = np.logical_and(np.logical_and(np.logical_and(lon >= bounds[0],lon <= bounds[1]),lat >= bounds[2]),lat <= bounds[3]) if not mask.sum(): break lons = lon[mask] lats = lat[mask] fires = fire[mask] # plot results if plot: plt.plot(lons[fires==5],lats[fires==5],'g.') plt.plot(lons[fires==9],lats[fires==9],'r.') plt.show() plt.pause(.5) plt.cla() else: lons = np.array([]) lats = np.array([]) fires = np.array([]) if gen_polys: from shapely.geometry import Polygon from shapely.ops import transform from functools import partial import pyproj proj4_wgs84 = '+proj=latlong +ellps=WGS84 +datum=WGS84 +units=degree +no_defs' proj4_moll = '+proj=moll +lon_0=0 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m +no_defs' proj = partial(pyproj.transform, pyproj.Proj(init='epsg:4326'), pyproj.Proj(proj4_moll)) polys = [] for n,outer in enumerate(outer_coordinates): x = np.array(outer)[:,0] y = np.array(outer)[:,1] shape = Polygon([(l[0],l[1]) for l in zip(x,y)]).buffer(0) inner = np.array(inner_coordinates[n]) if len(inner): if len(inner.shape) > 2: for h in inner_coordinates[n]: xh = np.array(h)[:,0] yh = np.array(h)[:,1] else: xh = np.array(inner)[:,0] yh = np.array(inner)[:,1] shapeh = Polygon([(l[0],l[1]) for l in zip(xh,yh)]).buffer(0) shape.difference(shapeh) poly = transform(proj,shape) polys.append(poly) idn = prefix + time_data if idn in perimeters.keys(): idn = idn + '_2' # update perimeters dictionary perimeters.update({idn: Dict({'file': file, 'lon': lons, 'lat': lats, 'fire': fires, 'conf_fire': np.array(conf_fire*np.ones(lons[fires==9].shape)), 'lon_fire': lons[fires==9], 'lat_fire': lats[fires==9], 'lon_nofire': lons[fires==5], 'lat_nofire': lats[fires==5], 'scan_fire': scan*np.ones(lons[fires==9].shape), 'track_fire': track*np.ones(lons[fires==9].shape), 'conf_nofire': np.array(conf_nofire*np.ones(lons[fires==5].shape)), 'scan_nofire': scan*np.ones(lons[fires==5].shape), 'track_nofire': track*np.ones(lons[fires==9].shape), 'time_iso': time_iso, 'time_num': time_num, 'acq_date': acq_date, 'acq_time': acq_time})}) if gen_polys: perimeters[idn].update({'polys': polys}) else: print 'Warning: No KML files in the path specified' perimeters = [] return perimeters
def process_tign_g(lon, lat, tign_g, ctime, scale, time_num, epsilon=5, plot=False): """ Process forecast from lon, lat, and tign_g as 3D data points :param lon: array of longitudes :param lat: array of latitudes :param tign_g: array of fire arrival time :param ctime: time char in wrfout of the last fire arrival time :param scale: numerical time scale in seconds of start and end of simulation :param time_num: numerical time interval in seconds of start and end of simulation :param epsilon: optional, epsilon in seconds to define the separation of artificial data :param plot: optional, plotting results Developed in Python 2.7.15 :: Anaconda 4.5.10, on MACINTOSH. Angel Farguell ([email protected]) and James Haley, 2019-06-05 """ # confidences conf_ground = 25. conf_fire = 25. # ctime transformations ctime_iso = ctime.replace('_', 'T') ctime_datetime = time_iso2datetime(ctime_iso) tscale = 3600. * 24. # tign_g as data points smaller than maximum tign_g t1d = np.ravel(tign_g) etime = t1d.max() mask = t1d < etime x1d = np.ravel(lon)[mask] y1d = np.ravel(lat)[mask] t1d = t1d[mask] # compute time as days from start of the simulation tt = (np.array([ time_iso2num( time_datetime2iso(ctime_datetime - timedelta(seconds=float(etime - T)))) for T in t1d ]) - scale[0]) / tscale # define forecast artificial data points fground = np.c_[x1d.ravel(), y1d.ravel(), tt.ravel() - epsilon / tscale] ffire = np.c_[x1d.ravel(), y1d.ravel(), tt.ravel() + epsilon / tscale] # coarsening maxsize = 5e3 coarsening = np.int(1 + len(ffire) / maxsize) fground = fground[::coarsening, :] ffire = ffire[::coarsening, :] # define output data X = np.concatenate((fground, ffire)) y = np.concatenate((-np.ones(len(fground)), np.ones(len(ffire)))) c = np.concatenate( (conf_ground * np.ones(len(fground)), conf_fire * np.ones(len(ffire)))) # plot results if plot: from contline import get_contour_verts from contour2kml import contour2kml tt = np.array([ time_iso2num( time_datetime2iso(ctime_datetime - timedelta(seconds=float(etime - T)))) for T in np.ravel(tign_g) ]) data = get_contour_verts(lon, lat, np.reshape(tt, tign_g.shape), time_num, contour_dt_hours=6, contour_dt_init=24, contour_dt_final=12) contour2kml(data, 'perimeters_forecast.kml') col = [(0, .5, 0), (.5, 0, 0)] cm_GR = colors.LinearSegmentedColormap.from_list('GrRd', col, N=2) fig = plt.figure() ax = fig.gca(projection='3d') ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=y, cmap=cm_GR, s=1, alpha=.5, vmin=y.min(), vmax=y.max()) ax.set_xlabel("Longitude") ax.set_ylabel("Latitude") ax.set_zlabel("Fire arrival time [days]") plt.show() return X, y, c
def process_tign_g_slices(lon, lat, tign_g, bounds, ctime, dx, dy, wrfout_file='', dt_for=3600., plot=False): """ Process forecast as slices on time from lon, lat, and tign_g :param lon: array of longitudes :param lat: array of latitudes :param tign_g: array of fire arrival time :param bounds: coordinate bounding box filtering to :param ctime: time char in wrfout of the last fire arrival time :param dx,dy: data resolution in meters :param dt_for: optional, temporal resolution to get the fire arrival time from in seconds :param wrfout_file: optional, to get the name of the file in the dictionary Developed in Python 2.7.15 :: Anaconda 4.5.10, on MACINTOSH. Angel Farguell ([email protected]) and James Haley, 2019-06-05 """ if plot: fig = plt.figure() # prefix of the elements in the dictionary prefix = "FOR" # confidences conf_fire = 50 conf_nofire = 10 # margin percentage margin = .5 # scan and track dimensions of the observation (in km) scan = dx / 1000. track = dy / 1000. # initializing dictionary forecast = Dict({}) # ctime transformations ctime_iso = ctime.replace('_', 'T') ctime_datetime = time_iso2datetime(ctime_iso) # mask coordinates to bounding box mask = np.logical_and( np.logical_and(np.logical_and(lon >= bounds[0], lon <= bounds[1]), lat >= bounds[2]), lat <= bounds[3]) # create a square subset of fire arrival time less than the maximum mtign = np.logical_and(mask, tign_g < tign_g.max()) mlon = lon[mtign] mlat = lat[mtign] mlenlon = margin * (mlon.max() - mlon.min()) mlenlat = margin * (mlat.max() - mlat.min()) sbounds = (mlon.min() - mlenlon, mlon.max() + mlenlon, mlat.min() - mlenlat, mlat.max() + mlenlat) smask = np.logical_and( np.logical_and(np.logical_and(lon >= sbounds[0], lon <= sbounds[1]), lat >= sbounds[2]), lat <= sbounds[3]) # times to get fire arrival time from TT = np.arange(tign_g.min() - 3 * dt_for, tign_g.max(), dt_for)[0:-1] for T in TT: # fire arrival time to datetime time_datetime = ctime_datetime - timedelta( seconds=float(tign_g.max() - T) ) # there is an error of about 10 seconds (wrfout not ending at exact time of simulation) # create ISO time of the date time_iso = time_datetime2iso(time_datetime) # create numerical time from the ISO time time_num = time_iso2num(time_iso) # create time stamp time_data = '_A%04d%03d_%02d%02d_%02d' % ( time_datetime.year, time_datetime.timetuple().tm_yday, time_datetime.hour, time_datetime.minute, time_datetime.second) # create acquisition date acq_date = '%04d-%02d-%02d' % (time_datetime.year, time_datetime.month, time_datetime.day) # create acquisition time acq_time = '%02d%02d' % (time_datetime.hour, time_datetime.minute) print 'Retrieving forecast from %s' % time_iso # masks fire = tign_g <= T nofire = np.logical_and(tign_g > T, np.logical_or(tign_g != tign_g.max(), smask)) # coordinates masked lon_fire = lon[np.logical_and(mask, fire)] lat_fire = lat[np.logical_and(mask, fire)] lon_nofire = lon[np.logical_and(mask, nofire)] lat_nofire = lat[np.logical_and(mask, nofire)] # create general arrays lons = np.concatenate((lon_nofire, lon_fire)) lats = np.concatenate((lat_nofire, lat_fire)) fires = np.concatenate( (5 * np.ones(lon_nofire.shape), 9 * np.ones(lon_fire.shape))) # plot results if plot: plt.ion() plt.cla() plt.plot(lons[fires == 5], lats[fires == 5], 'g.') plt.plot(lons[fires == 9], lats[fires == 9], 'r.') plt.pause(.0001) plt.show() # update perimeters dictionary forecast.update({ prefix + time_data: Dict({ 'file': wrfout_file, 'time_tign_g': T, 'lon': lons, 'lat': lats, 'fire': fires, 'conf_fire': np.array(conf_fire * np.ones(lons[fires == 9].shape)), 'lon_fire': lons[fires == 9], 'lat_fire': lats[fires == 9], 'lon_nofire': lons[fires == 5], 'lat_nofire': lats[fires == 5], 'scan_fire': scan * np.ones(lons[fires == 9].shape), 'track_fire': track * np.ones(lons[fires == 9].shape), 'conf_nofire': np.array(conf_nofire * np.ones(lons[fires == 5].shape)), 'scan_nofire': scan * np.ones(lons[fires == 5].shape), 'track_nofire': track * np.ones(lons[fires == 9].shape), 'time_iso': time_iso, 'time_num': time_num, 'acq_date': acq_date, 'acq_time': acq_time }) }) return forecast
sl.save(f, 'forecast') else: from infrared_perimeters import process_ignitions from setup import process_detections dst = 'ideal_test' plot = False ideal = sl.load(dst) kk = 4 data = process_tign_g_slices(ideal['lon'][::kk, ::kk], ideal['lat'][::kk, ::kk], ideal['tign_g'][::kk, ::kk], ideal['bounds'], ideal['ctime'], ideal['dx'], ideal['dy'], wrfout_file='ideal', dt_for=ideal['dt'], plot=plot) if 'point' in ideal.keys(): p = [[ideal['point'][0]], [ideal['point'][1]], [ideal['point'][2]]] data.update(process_ignitions(p, ideal['bounds'])) elif 'points' in ideal.keys(): p = [[point[0] for point in ideal['points']], [point[1] for point in ideal['points']], [point[2] for point in ideal['points']]] data.update(process_ignitions(p, ideal['bounds'])) etime = time_iso2num(ideal['ctime'].replace('_', 'T')) time_num_int = (etime - ideal['tign_g'].max(), etime) sl.save((data, ideal['lon'], ideal['lat'], time_num_int), 'data') process_detections(data, ideal['lon'], ideal['lat'], time_num_int)