def get_projection(self): """ Returns GeographicSystem and MapProjection instances from lmatools.coordinateSystems corresponding to the coordinate system specified by the metadata of the first NetCDF file in self._filenames. """ from lmatools.coordinateSystems import GeographicSystem, MapProjection geosys = GeographicSystem() f = NetCDFFile(self._filenames[0]) # Surely someone has written an automated library to parse coordinate # reference data from CF-compliant files. if 'Lambert_Azimuthal_Equal_Area' in list(f.variables.keys()): nc_proj = f.variables['Lambert_Azimuthal_Equal_Area'] proj_name = 'laea' ctrlon, ctrlat = (nc_proj.longitude_of_projection_origin, nc_proj.latitude_of_projection_origin,) try: ctralt = nc_proj.altitude_of_projection_origin except AttributeError: print("No altitude attribute in NetCDF projection data, setting to 0.0") ctralt = 0.0 mapproj = MapProjection(proj_name, ctrLat = ctrlat, ctrLon=ctrlon, lat_0=ctrlat, lon_0=ctrlon) # print geosys.fromECEF(*mapproj.toECEF((0,0), (0,0), (0,0))) return geosys, mapproj else: print("projection not found, assuming lat, lon grid") return geosys, geosys f.close()
def cluster(a_file, output_path, outfile, params, logger, min_points=1, **kwargs): """ There is no intermediate ASCII output or temporary file for this code, since all data remains as native Python objects. """ logger = logging.getLogger('FlashAutorunLogger') if 'mask_length' in params: mask_length = params['mask_length'] else: mask_length = 4 lma=LMAdataFile(a_file, mask_length = mask_length) # for line in lma.header: # print line ctr_lat, ctr_lon, ctr_alt = params['ctr_lat'], params['ctr_lon'], 0.0 good = (lma.stations >= params['stations'][0]) & (lma.chi2 <= params['chi2'][1]) if 'alt' in params: good = good & (lma.alt < params['alt'][1]) data = lma.data[good] geoCS = GeographicSystem() X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt']) Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt) X, Y, Z = X-Xc, Y-Yc, Z-Zc print "sorting {0} total points".format(data.shape[0]) D_max, t_max = 3.0e3, 0.15 # m, s IDs = np.arange(X.shape[0]) X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max T_vector = data['time'][:,None] / t_max XYZT = np.hstack((X_vector, T_vector)) lma.sort_status = 'in process' # Maximum 3 s flash length, normalized to the time separation scale flash_object_maker = create_flash_objs(lma, data) label_aggregator = aggregate_ids(flash_object_maker) clusterer = cluster_chunk_pairs(label_aggregator, min_points=min_points) chunker = chunk(XYZT[:,-1].min(), 3.0/.15, clusterer) stream(XYZT.astype('float64'), IDs,chunker) flash_object_maker.close() # These are handled by target.close in each coroutine's GeneratorExit handler # clusterer.close() # label_aggregator.close() # flash_object_maker.close() print lma.sort_status print len(lma.flash_objects) return lma, lma.flash_objects
def cluster(a_file, output_path, outfile, params, logger, min_points=1, **kwargs): """ There is no intermediate ASCII output or temporary file for this code, since all data remains as native Python objects. """ logger = logging.getLogger('FlashAutorunLogger') lma=LMAdataFile(a_file) # for line in lma.header: # print line ctr_lat, ctr_lon, ctr_alt = params['ctr_lat'], params['ctr_lon'], 0.0 good = (lma.stations >= params['stations'][0]) & (lma.chi2 <= params['chi2'][1]) if 'alt' in params: good = good & (lma.alt < params['alt'][1]) data = lma.data[good] geoCS = GeographicSystem() X,Y,Z = geoCS.toECEF(data['lon'], data['lat'], data['alt']) Xc, Yc, Zc = geoCS.toECEF( ctr_lon, ctr_lat, ctr_alt) X, Y, Z = X-Xc, Y-Yc, Z-Zc print("sorting {0} total points".format(data.shape[0])) D_max, t_max = params['distance'], params['thresh_critical_time'] # m, s duration_max = params['thresh_duration'] IDs = np.arange(X.shape[0]) X_vector = np.hstack((X[:,None],Y[:,None],Z[:,None])) / D_max T_vector = data['time'][:,None] / t_max XYZT = np.hstack((X_vector, T_vector)) # Maximum 3 s flash length, normalized to the time separation scale flash_object_maker = create_flash_objs(lma, data) label_aggregator = aggregate_ids(flash_object_maker) clusterer = cluster_chunk_pairs(label_aggregator, min_points=min_points) if XYZT.shape[0] < 1: # no data, so minimum time is zero. Assume nothing is done with the data, # so that time doesn't matter. No flashes can result. chunker = chunk(0, duration_max/t_max, clusterer) else: chunker = chunk(XYZT[:,-1].min(), duration_max/t_max, clusterer) stream(XYZT.astype('float64'), IDs,chunker) flash_object_maker.close() # These are handled by target.close in each coroutine's GeneratorExit handler # clusterer.close() # label_aggregator.close() # flash_object_maker.close() print(len(lma.flash_objects)) return lma, lma.flash_objects
def geo_to_cartesisan(self, lon, lat, alt): """ Convert lat, lon in degrees and altitude in meters to Earth-centered, Earth-fixed cartesian coordinates. Translate to coordinate center location. Returns X,Y,Z in meters. """ geoCS = GeographicSystem() X, Y, Z = geoCS.toECEF(lon, lat, alt) Xc, Yc, Zc = geoCS.toECEF(self.ctr_lon, self.ctr_lat, self.ctr_alt) X, Y, Z = X - Xc, Y - Yc, Z - Zc return (X, Y, Z)
def get_GOESR_coordsys(sat_lon_nadir=-75.0): """ Values from the GOES-R PUG Volume 3, L1b data Returns geofixcs, grs80lla: the fixed grid coordinate system and the latitude, longitude, altitude coordinate system referenced to the GRS80 ellipsoid used by GOES-R as its earth reference. """ goes_sweep = 'x' # Meteosat is 'y' ellipse = 'GRS80' datum = 'WGS84' sat_ecef_height = 35786023.0 geofixcs = GeostationaryFixedGridSystem(subsat_lon=sat_lon_nadir, ellipse=ellipse, datum=datum, sweep_axis=goes_sweep, sat_ecef_height=sat_ecef_height) grs80lla = GeographicSystem(ellipse='GRS80', datum='WGS84') return geofixcs, grs80lla
def test_fixed_grid_GOESR(): """ Tests from the GOES-R PUG Volume 3, L1b data """ sat_lon_nadir = -75.0 goes_sweep = 'x' # Meteosat is 'y' ellipse = 'GRS80' datum = 'WGS84' sat_ecef_height = 35786023.0 geofixcs = GeostationaryFixedGridSystem(subsat_lon=sat_lon_nadir, ellipse=ellipse, datum=datum, sweep_axis=goes_sweep, sat_ecef_height=sat_ecef_height) latloncs = GeographicSystem(ellipse=ellipse, datum=datum) test_lat = 33.846162 test_lon = -84.690932 test_alt = 0.0 test_fixx = -0.024052 test_fixy = 0.095340 test_fixz = 0.0 atol = 1e-6 # Test forward from geodetic X, Y, Z = latloncs.toECEF(test_lon, test_lat, test_alt) x, y, z = geofixcs.fromECEF(X, Y, Z) assert_allclose(test_fixx, x, rtol=atol) assert_allclose(test_fixy, y, rtol=atol) assert_allclose(test_fixz, z, rtol=atol) # print(test_fixx, test_fixy, test_fixz) # print(x,y,z) # Test inverse from fixed grid angle X, Y, Z = geofixcs.toECEF(test_fixx, test_fixy, test_fixz) x, y, z = latloncs.fromECEF(X, Y, Z) assert_allclose(test_lon, x, atol=atol) assert_allclose(test_lat, y, atol=atol) assert_allclose(test_alt, z, atol=atol)
import netCDF4 import numpy as np import pandas as pd import scipy as sci # from scipy.spatial import * import matplotlib import matplotlib.pyplot as plt from matplotlib.lines import Line2D from matplotlib.widgets import Widget from matplotlib.colors import LogNorm, Normalize from lmatools.coordinateSystems import GeographicSystem, MapProjection geosys = GeographicSystem() from ipywidgets import widgets # from IPython.display import HTML # from IPython.display import Javascript # from IPython.display import display from lmatools.vis import ctables as ct import logging, json # from http://stackoverflow.com/questions/3488934/simplejson-and-numpy-array/24375113#24375113 class NumpyAwareJSONEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, np.ndarray): # and obj.ndim == 1: return obj.tolist() return json.JSONEncoder.default(self, obj)
except GeneratorExit: print(total) #lma=LMAdataFile('/Users/ebruning/Documents/Lightning\ interpretation/Flash-length/Thomas/LYLOUT_120412_01817.exported.dat.gz') #ctr_lat, ctr_lon, ctr_alt = 40.4463980, -104.6368130, 1000.00 lma = LMAdataFile('/data/20040526/LMA/LYLOUT_040526_224000_0600.dat.gz') # for line in lma.header: # print line ctr_lat, ctr_lon, ctr_alt = 35.2791257, -97.9178678, 417.90 # OKLMA #ctr_lat, ctr_lon, ctr_alt = 40.4463980, -104.6368130, 1000.00 # COLMA good = (lma.stations >= 6) & (lma.chi2 <= 2.0) & (lma.alt < 20e3) data = lma.data[good] geoCS = GeographicSystem() X, Y, Z = geoCS.toECEF(data['lon'], data['lat'], data['alt']) Xc, Yc, Zc = geoCS.toECEF(ctr_lon, ctr_lat, ctr_alt) X, Y, Z = X - Xc, Y - Yc, Z - Zc D_max, t_max = 3.0e3, 0.15 # m, s X_vector = np.hstack((X[:, None], Y[:, None], Z[:, None])) / D_max T_vector = data['time'][:, None] / t_max XYZT = np.hstack((X_vector, T_vector - T_vector.min())) # Maximum 3 s flash length, normalized to the time separation scale chunker = chunk(XYZT[:, -1].min(), 3.0 / .15, cluster_chunk_pairs(cluster_printer())) stream(XYZT.astype('float32'), chunker)
def dlonlat_at_grid_center(ctr_lat, ctr_lon, dx=4.0e3, dy=4.0e3, x_bnd=(-100e3, 100e3), y_bnd=(-100e3, 100e3), proj_datum='WGS84', proj_ellipse='WGS84'): """ Utility function useful for producing a regular grid of lat/lon data, where an approximate spacing (dx, dy) and total span of the grid (x_bnd, y_bnd) is desired. Units are in meters. There is guaranteed to be distortion away from the grid center, i.e., only the grid cells adjacent to the center location will have area dx * dy. Likewise, the lat, lon range is calculated naively using dlat, dlon multiplied by the number of grid cells implied by x_bnd/dx, y_bnd/dy. This is the naive approach, but probably what's expected when specifying distances in kilometers for an inherently distorted lat/lon grid. Returns: (dlon, dlat, lon_bnd, lat_bnd) corresponding to (dx, dy, x_range, y_range) """ # Use the Azimuthal equidistant projection as the method for converting to kilometers. proj_name = 'aeqd' mapProj = MapProjection(projection=proj_name, ctrLat=ctr_lat, ctrLon=ctr_lon, lat_ts=ctr_lat, lon_0=ctr_lon, lat_0=ctr_lat, lat_1=ctr_lat, ellipse=proj_ellipse, datum=proj_datum) geoProj = GeographicSystem() # Get dlat lon_n, lat_n, z_n = geoProj.fromECEF(*mapProj.toECEF(0, dy, 0)) dlat = lat_n - ctr_lat # Get dlon lon_e, lat_e, z_e = geoProj.fromECEF(*mapProj.toECEF(dx, 0, 0)) dlon = lon_e - ctr_lon lon_min = ctr_lon + dlon * (x_bnd[0] / dx) lon_max = ctr_lon + dlon * (x_bnd[1] / dx) lat_min = ctr_lat + dlat * (y_bnd[0] / dy) lat_max = ctr_lat + dlat * (y_bnd[1] / dy) # Alternate method: lat lon for the actual distance to the NSEW in the projection #lon_range_n, lat_range_n, z_range_n = geoProj.fromECEF(*mapProj.toECEF(0,y_bnd,0)) #lon_range_e, lat_range_e, z_range_e = geoProj.fromECEF(*mapProj.toECEF(x_bnd,0,0)) return dlon, dlat, (lon_min, lon_max), (lat_min, lat_max)
def __init__( self, start_time, end_time, do_3d=True, frame_interval=120.0, dx=4.0e3, dy=4.0e3, dz=1.0e3, base_date=None, x_bnd=(-100e3, 100e3), y_bnd=(-100e3, 100e3), z_bnd=(0e3, 20e3), ctr_lat=35.23833, ctr_lon=-97.46028, ctr_alt=0.0, proj_name='aeqd', proj_datum='WGS84', proj_ellipse='WGS84', pixel_coords=None, flash_count_logfile=None, energy_grids=None, event_grid_area_fraction_key=None, spatial_scale_factor=1.0 / 1000.0, subdivide=False, ): """ Class to support gridding of flash and event data. On init, specify the grid If proj_name = 'pixel_grid' then pixel_coords must be an instance of lmatools.coordinateSystems.PixelGrid If proj_name = 'geos' then pixel_coords must be an instance of lmatools.coordinateSystems.GeostationaryFixedGridSystem energy_grids controls which energy grids are saved, default None. energy_grids may be True, which will calculate all energy grid types, or it may be one of 'specific_energy', 'total_energy', or a list of one or more of these. event_grid_area_fraction_key specifies the name of the variable in the events array that gives the fraction of each grid cell covered by each event. Used only for pixel-based event detectors. """ if energy_grids == True: energy_grids = ('specific_energy', 'total_energy') self.energy_grids = energy_grids self.spatial_scale_factor = spatial_scale_factor self.event_grid_area_fraction_key = event_grid_area_fraction_key # args, kwargs that are saved for the future self.do_3d = do_3d self.start_time = start_time self.dx, self.dy, self.dz = dx, dy, dz self.end_time = end_time self.frame_interval = frame_interval self.base_date = base_date self.min_points_per_flash = 1 self.proj_name = proj_name self.ctr_lat, self.ctr_lon, self.ctr_alt = ctr_lat, ctr_lon, ctr_alt if flash_count_logfile is None: flash_count_logfile = log self.flash_count_logfile = flash_count_logfile t_edges, duration = time_edges(self.start_time, self.end_time, self.frame_interval) # reference time is the date part of the start_time, unless the user provides a different date. if self.base_date is None: t_ref, t_edges_seconds = seconds_since_start_of_day( self.start_time, t_edges) else: t_ref, t_edges_seconds = seconds_since_start_of_day( self.base_date, t_edges) self.n_frames = len(t_edges) - 1 xedge = np.arange(x_bnd[0], x_bnd[1] + dx, dx) yedge = np.arange(y_bnd[0], y_bnd[1] + dy, dy) zedge = np.arange(z_bnd[0], z_bnd[1] + dz, dz) if self.proj_name == 'latlong': dx_units = '{0:6.4f}deg'.format(float(dx)) mapProj = GeographicSystem() elif self.proj_name == 'pixel_grid': dx_units = 'pixel' mapProj = pixel_coords elif self.proj_name == 'geos': dx_units = '{0:03d}urad'.format(int(dx * 1e6)) mapProj = pixel_coords else: dx_units = '{0:5.1f}m'.format(float(dx)) mapProj = MapProjection(projection=self.proj_name, ctrLat=ctr_lat, ctrLon=ctr_lon, lat_ts=ctr_lat, lon_0=ctr_lon, lat_0=ctr_lat, lat_1=ctr_lat, ellipse=proj_ellipse, datum=proj_datum) geoProj = GeographicSystem() self.t_ref = t_ref self.t_edges = t_edges self.t_edges_seconds = t_edges_seconds self.duration = duration self.xedge = xedge self.yedge = yedge self.zedge = zedge self.mapProj = mapProj self.geoProj = geoProj self.dx_units = dx_units self.pipeline_setup() self.output_setup()
def get_geometry_hrrr(glm, nadir, hrrr, ltgellipsever=1): # We don't use X, Y, Z below. # x, y are the fixed grid 2D coord arrays (from meshgrid) # X, Y, Z are the ECEF coords of each pixel intersected at the ltg ellipsoid # lon_ltg, lat_ltg are the parallax-corrected lon lat at the earth's surface # below the lightning position on the lightning ellipsoid. # outside_glm_full_disk is a boolean mask for the positions that GLM can't # observe. # All of the above are 2D arrays corrsponding the center positions of the 2 # km fixed grid pixels in the GLM gridded products. ((x, y), (X, Y, Z), (lon_ltg, lat_ltg, alt_ltg), outside_glm_full_disk) = get_glm_earth_geometry(glm, nadir, ltgellipsever) ### HRRR interpolation ### # Everything below presumes LCC. assert hrrr.MAP_PROJ_CHAR == 'Lambert Conformal' corner_0_lla = (hrrr.XLONG[0, 0, 0].data, hrrr.XLAT[0, 0, 0].data, np.asarray(0.0, dtype=hrrr.XLAT[0, 0, 0].dtype)) corner_1_lla = (hrrr.XLONG[0, -1, -1].data, hrrr.XLAT[0, -1, -1].data, np.asarray(0.0, dtype=hrrr.XLAT[0, 1, -1].dtype)) hrrr_dx, hrrr_dy = hrrr.DX, hrrr.DX hrrr_Nx, hrrr_Ny = hrrr.dims['west_east'], hrrr.dims['south_north'] hrrrproj = { 'lat_0': hrrr.CEN_LAT, 'lon_0': hrrr.CEN_LON + 360.0, 'lat_1': hrrr.TRUELAT1, 'lat_2': hrrr.TRUELAT2, # 'R':hrrr.LambertConformal_Projection.earth_radius, # 'a':6371229, # 'b':6371229, 'R': 6371229, } lcc = MapProjection(projection='lcc', ctrLat=hrrrproj['lat_0'], ctrLon=hrrrproj['lon_0'], **hrrrproj) hrrr_lla = GeographicSystem(r_equator=hrrrproj['R'], r_pole=hrrrproj['R']) lcc_cornerx_0, lcc_cornery_0, lcc_cornerz_0 = lcc.fromECEF( *hrrr_lla.toECEF(*corner_0_lla)) lcc_cornerx_1, lcc_cornery_1, lcc_cornerz_1 = lcc.fromECEF( *hrrr_lla.toECEF(*corner_1_lla)) # def grid_idx(x, y, x0, y0, dx, dy): # """ # Convert x, y returned by [projection].fromECEF to the grid index in the # NetCDF file. x0 and y0 are the [projection] coordinates of the center of # the zero-index position in the NetCDF grid. dx and dy are the grid spacing # in meters. # # returns (xidx, yidx) # Taking int(xidx) will give the zero-based grid cell index. # """ # # To get the correct behavior with int(xidx), add a half # # since x0 is the center. # xidx = (x-x0)/dx + 0.5 # yidx = (y-y0)/dy + 0.5 # return xidx, yidx # The 3D position (X,Y,Z) defines an implicit lon, lat, alt with respect to the # spherical earth we specified for the HRRR and its associated Lambert # conformal projection. We let proj4 handle the mapping from the ECEF # coordinates (an absolute position) directly to LCC. lccx2, lccy2, lccz2 = lcc.fromECEF( *hrrr_lla.toECEF(lon_ltg, lat_ltg, np.zeros_like(lon_ltg))) lccx2.shape = x.shape lccy2.shape = x.shape lccx = lccx2 lccy = lccy2 # Set up the model grid, since the hrrr file doesn't include those values. hrrrx_1d = np.arange(hrrr_Nx, dtype='f4') * hrrr_dx + lcc_cornerx_0 hrrry_1d = np.arange(hrrr_Ny, dtype='f4') * hrrr_dy + lcc_cornery_0 hrrrx, hrrry = np.meshgrid(hrrrx_1d, hrrry_1d) interp_loc = np.vstack((hrrrx.flatten(), hrrry.flatten())).T # GLM variables are filled with nan everywhere there is no lightning, # so set those locations corresponding to valid earth locations to zero. lcc_glm_x_flat = lccx[:, :].flatten() lcc_glm_y_flat = lccy[:, :].flatten() good = np.isfinite(lcc_glm_x_flat) & np.isfinite(lcc_glm_y_flat) good = good & (~outside_glm_full_disk.flatten()) data_loc = np.vstack((lcc_glm_x_flat[good], lcc_glm_y_flat[good])).T return (data_loc, good, interp_loc, hrrrx.shape)