def test_sun_earth_distance(self): path_to_data = glob.glob("../../data/test_data/*.N1")[0] product = epr.Product(path_to_data) HotspotDetector = ATXDetector(product) target = 0.9877038273760421 result = HotspotDetector._compute_sun_earth_distance() self.assertAlmostEqual(target, result)
def test_compute_frp(self): path_to_data = glob.glob("../../data/test_data/*.N1")[0] product = epr.Product(path_to_data) HotspotDetector = ATXDetector(product) HotspotDetector.run_detector(flares_or_sampling=True) path_to_target = "../../data/test_data/atx_frp.npy" target = np.load(path_to_target) result = HotspotDetector.frp self.assertEqual(True, (target == result).all())
def test_detect_hotspots_atx(self): path_to_data = glob.glob("../../data/test_data/*.N1")[0] path_to_target = "../../data/test_data/atx_detect_hotspots.npy" target = np.load(path_to_target) product = epr.Product(path_to_data) HotspotDetector = ATXDetector(product) HotspotDetector.run_detector() self.assertEqual(True, (target == HotspotDetector.hotspots).all())
def test_radiance_from_BT(self): path_to_data = glob.glob("../../data/test_data/*.N1")[0] product = epr.Product(path_to_data) HotspotDetector = ATXDetector(product) brightness_temp = 1500 wavelength = 1.6 result = HotspotDetector._rad_from_BT(wavelength, brightness_temp) target = 28200.577465487077 self.assertAlmostEqual(target, result)
def test_radiance_from_reflectance(self): path_to_target = "../../data/test_data/atx_radiance_from_reflectance.npy" target = np.load(path_to_target) path_to_data = glob.glob("../../data/test_data/*.N1")[0] product = epr.Product(path_to_data) HotspotDetector = ATXDetector(product) reflectance = product.get_band('reflec_nadir_1600').read_as_array() result = HotspotDetector._rad_from_ref(reflectance) self.assertEqual(True, (target == result).all())
def test_night_mask_atx(self): path_to_data = glob.glob("../../data/test_data/*.N1")[0] path_to_target = "../../data/test_data/atx_nightmask.npy" target = np.load(path_to_target) target_mean = np.mean(target) product = epr.Product(path_to_data) HotspotDetector = ATXDetector(product) HotspotDetector.run_detector() self.assertAlmostEqual(target_mean, np.mean(HotspotDetector.night_mask))
def test_get_arcmin_int(self): coords = np.array([ -150.53434, -100.13425, -50.20493, 0.34982, 50.43562, 100.12343, 150.56443 ]) target = np.array([-15032, -10008, -5012, 21, 5026, 10007, 15034]) path_to_data = glob.glob("../../data/test_data/*.N1")[0] product = epr.Product(path_to_data) HotspotDetector = ATXDetector(product) result = HotspotDetector._find_arcmin_gridcell(coords) self.assertEqual(True, (target == result).all())
def test_run_atx(self): target = pd.read_csv(glob.glob("../../data/test_data/ATS*.csv")[0]) path_to_data = glob.glob("../../data/test_data/*.N1")[0] product = epr.Product(path_to_data) HotspotDetector = ATXDetector(product) HotspotDetector.run_detector() result = HotspotDetector.to_dataframe(keys=['latitude', 'longitude']) # TODO determine why floating point errors are causing issues in testing here target = target.astype(int) result = result.astype(int) are_equal = target.equals(result) self.assertEqual(True, are_equal)
def main(): file_to_process = sys.argv[1] sensor = sys.argv[2] if sensor != 'sls': product = epr.Product(file_to_process) HotspotDetector = ATXDetector(product) keys = ['latitude', 'longitude'] else: product = utils.extract_zip(file_to_process, fp.slstr_extract_temp) HotspotDetector = SLSDetector(product) keys = ['latitude', 'longitude'] HotspotDetector.run_detector() df = HotspotDetector.to_dataframe(keys=keys) df.to_csv(utils.build_outpath(sensor, file_to_process, 'hotspots'))
def create_asar_image(iPath, oPath_4326, oPath_3413, fileName): #plt.close("all") product = epr.Product(os.path.join(iPath, fileName)) band = product.get_band('proc_data') sc_w = product.get_scene_width() sc_h = product.get_scene_height() raw_counts = band.read_as_array( sc_w, sc_h) #, xoffset=100, yoffset=6500, xstep=2, ystep=2) lat = product.get_band('latitude').read_as_array(sc_w, sc_h) lon = product.get_band('longitude').read_as_array(sc_w, sc_h) incident_angle = product.get_band('incident_angle').read_as_array( sc_w, sc_h) raw_counts_trmmd = raw_counts # Trimming the array by removing zero values from rows and cols msk = [] for m in range(raw_counts_trmmd.shape[0]): if raw_counts_trmmd[m, :].sum() == 0: msk.append(m) raw_counts_trmmd = numpy.delete(raw_counts_trmmd, msk, axis=0) lat = numpy.delete(lat, msk, axis=0) lon = numpy.delete(lon, msk, axis=0) incident_angle = numpy.delete(incident_angle, msk, axis=0) msk = [] for n in range(raw_counts_trmmd.shape[1]): if raw_counts_trmmd[:, n].sum() == 0: msk.append(n) raw_counts_trmmd = numpy.delete(raw_counts_trmmd, msk, axis=1) lat = numpy.delete(lat, msk, axis=1) lon = numpy.delete(lon, msk, axis=1) incident_angle = numpy.delete(incident_angle, msk, axis=1) raw_counts = raw_counts_trmmd # Adding Sigma_0 calibration_constant = \ product.get_dataset('MAIN_PROCESSING_PARAMS_ADS').read_record(0).get_field('calibration_factors.1.ext_cal_fact').get_elems() # sigma0 = 10*log10( raw_counts**2*sin(incident_angle*pi/180)/calibration_constant ) sigma0 = raw_counts**2 * sin( incident_angle * pi / 180) / calibration_constant print " start filter" from scipy.signal import wiener sigma0w = wiener(sigma0, mysize=(7, 7), noise=None) # sigma0w = sigma0 pol = product.get_sph().get_field('MDS1_TX_RX_POLAR').get_elem() if pol == 'H/H': ph = (2.20495, -14.3561e-2, 11.28e-4) sigma0_hh_ref = exp( (ph[0] + incident_angle * ph[1] + incident_angle**2 * ph[2]) * log(10)) roughness = sigma0w / sigma0_hh_ref elif pol == 'V/V': pv = (2.29373, -15.393e-2, 15.1762e-4) sigma0_vv_ref = exp( (pv[0] + incident_angle * pv[1] + incident_angle**2 * pv[2]) * log(10)) roughness = sigma0w / sigma0_vv_ref # masking the arrays raw_counts = ma.masked_where(raw_counts == 0, raw_counts) roughness = ma.masked_where(raw_counts == 0, roughness) sigma0 = ma.masked_where(raw_counts == 0, sigma0) sigma0w = ma.masked_where(raw_counts == 0, sigma0w) default_fill_value = double(ma.default_fill_value(raw_counts)) scale = 30 pr.kd_tree.which_kdtree() pr.get_capabilities() scale = 1 ln = lon[::scale, ::scale] lt = lat[::scale, ::scale] data = roughness[::scale, ::scale] for proj in ['EPSG:4326', 'EPSG:3413']: print " start projection %s" % proj if proj == 'EPSG:4326': oPath = oPath_4326 area_def = swath_area_def( name='Temporal SWATH EPSG Projection 4326', proj='eqc', lonlim=(lon.min(), lon.max()), latlim=(lat.min(), lat.max()), ellps="WGS84", res=150) elif proj == 'EPSG:3413': oPath = oPath_3413 area_def = swath_area_def( name='Temporal SWATH EPSG Projection 3413', proj='stere', lonlim=(lon.min(), lon.max()), latlim=(lat.min(), lat.max()), ellps="WGS84", res=150, lat_ts=70, lat_0=90, lon_0=-45) swath_def = pr.geometry.SwathDefinition(lons=ln, lats=lt) # result = pr.kd_tree.resample_nearest(swath_def, data.ravel(), area_def, radius_of_influence=30000, nprocs=2) # result = pr.kd_tree.resample_nearest(swath_def, data, area_def, radius_of_influence=50000, epsilon=0.5, fill_value=default_fill_value) result = pr.kd_tree.resample_nearest(swath_def, data.ravel(), area_def, radius_of_influence=300, epsilon=0.5, nprocs=2, fill_value=None) oFileName = os.path.join(oPath, fileName + '.png') gray() imsave(oFileName, result, vmin=0, vmax=2) create_KML_asar(area_def.area_extent, os.path.join(oPath, fileName + '.kml')) close()
def read_atsr(path_to_ats_data): return epr.Product(path_to_ats_data)
def main(): file_to_process = sys.argv[1] sensor = sys.argv[2] if sensor != 'sls': product = epr.Product(file_to_process) HotspotDetector = ATXDetector(product) flare_keys = [ 'latitude', 'longitude', 'local_cloudiness', 'swir_16', 'frp', 'pixel_size', 'mwir', 'background_mwir' ] flare_aggregator = { 'frp': np.sum, 'swir_16': np.mean, 'mwir': np.mean, 'background_mwir': np.mean, 'pixel_size': np.sum, 'latitude': np.mean, 'longitude': np.mean, 'local_cloudiness': np.mean, 'year': 'first', 'month': 'first', 'day': 'first', 'hhmm': 'first' } sampling_keys = ['latitude', 'longitude', 'local_cloudiness'] sampling_aggregator = { 'local_cloudiness': np.mean, 'year': 'first', 'month': 'first', 'day': 'first', 'hhmm': 'first' } atx_persistent_fp = os.path.join(fp.output_l3, 'all_sensors', 'all_flare_locations_ats.csv') persistent_df = pd.read_csv(atx_persistent_fp) else: product = utils.extract_zip(file_to_process, fp.slstr_extract_temp) HotspotDetector = SLSDetector(product) flare_keys = [ 'latitude', 'longitude', 'local_cloudiness', 'swir_16', 'swir_22', 'frp', 'pixel_size' ] flare_aggregator = { 'frp': np.sum, 'swir_16': np.mean, 'swir_22': np.mean, 'pixel_size': np.sum, 'latitude': np.mean, 'longitude': np.mean, 'local_cloudiness': np.mean, 'year': 'first', 'month': 'first', 'day': 'first', 'hhmm': 'first' } sampling_keys = [ 'latitude', 'longitude', 'local_cloudiness', ] sampling_aggregator = { 'local_cloudiness': np.mean, 'year': 'first', 'month': 'first', 'day': 'first', 'hhmm': 'first' } # merge persistent dataframes for SLSTR atx_persistent_fp = os.path.join(fp.output_l3, 'all_sensors', 'all_flare_locations_atx.csv') atx_persistent_df = pd.read_csv(atx_persistent_fp) sls_persistent_fp = os.path.join(fp.output_l3, 'all_sensors', 'all_flare_locations_sls.csv') sls_persistent_df = pd.read_csv(sls_persistent_fp) persistent_df = merge_hotspot_dataframes(atx_persistent_df, sls_persistent_df) # find persistent hotspots (i.e. flares) HotspotDetector.run_detector(flares_or_sampling=True) flare_df = HotspotDetector.to_dataframe(keys=flare_keys, joining_df=persistent_df) aggregated_flare_df = aggregate(flare_df, flare_aggregator) aggregated_flare_df.to_csv( utils.build_outpath(sensor, file_to_process, 'flares')) # get sampling associated with persistent hotspots sampling_df = HotspotDetector.to_dataframe(keys=sampling_keys, joining_df=persistent_df) aggregated_sampling_df = aggregate(sampling_df, sampling_aggregator) aggregated_sampling_df.to_csv( utils.build_outpath(sensor, file_to_process, 'samples'))
def readASAR(iPath, fileName, pxlRes=800.0, image_size_skip=40 * 1e6, area_bbox=45): logger.info(os.path.join(iPath, fileName)) try: product = epr.Product(os.path.join(iPath, fileName)) except: logger.error('unable to read file') return False try: band = product.get_band('proc_data') except epr.EPRValueError: logger.error( 'unable to get band "proc_data": epr_get_band_id: band not found') return False sc_w = double(product.get_scene_width()) sc_h = double(product.get_scene_height()) # Skipping if image too large logger.debug('sc_w*sc_h = %s MPs', str(round(sc_w * sc_h / 1e6))) if sc_w * sc_h > image_size_skip: logger.debug("ASAR Image too large, skipping...") return False # Get lat/lon from geolocation grid dataset = product.get_dataset('GEOLOCATION_GRID_ADS') fltp_lats = map( lambda x: dataset.read_record(x).get_field('first_line_tie_points.lats' ).get_elems(), range(dataset.get_num_records())) lltp_lats = map( lambda x: dataset.read_record(x).get_field('last_line_tie_points.lats') .get_elems(), range(dataset.get_num_records())) fltp_lons = map( lambda x: dataset.read_record(x).get_field( 'first_line_tie_points.longs').get_elems(), range(dataset.get_num_records())) lltp_lons = map( lambda x: dataset.read_record(x).get_field('last_line_tie_points.longs' ).get_elems(), range(dataset.get_num_records())) fltp_lats = asarray(double(fltp_lats)) / 1e6 lltp_lats = asarray(double(lltp_lats)) / 1e6 fltp_lons = asarray(double(fltp_lons)) / 1e6 lltp_lons = asarray(double(lltp_lons)) / 1e6 lats = row_stack((fltp_lats, lltp_lats[-1, :])) lons = row_stack((fltp_lons, lltp_lons[-1, :])) # Skipping if no area overlap if mean(lats[:]) <= area_bbox: logger.debug("No area overlap. Skipping...") return False lats = fliplr(lats) lons = fliplr(lons) # Find scale to reduce image to the specified resolution arrShape = asarray([sc_w, sc_h]) _lats = asarray([lats[0, 0], lats[-1, -1], lats[0, -1], lats[-1, 0]]) _lons = asarray([lons[0, 0], lons[-1, -1], lons[0, -1], lons[-1, 0]]) imageRes = round( mean( asarray( distancelib.getPixelResolution(_lats, _lons, arrShape, 'km')) * 1e3)) scale = pxlRes / imageRes extMax = (0., 0., arrShape[0] - 1, arrShape[1] - 1) ext = (0., 0., arrShape[0] - 1, arrShape[1] - 1) # Format extent/spacing ext, spa = format_extent_spacing(extent=ext, spacing=scale, extmax=extMax) # Read data with stepping=spacing try: raw_counts = band.read_as_array(sc_w, sc_h, xstep=spa[0], ystep=spa[1]) incident_angle = product.get_band('incident_angle').read_as_array( sc_w, sc_h, xstep=spa[0], ystep=spa[1]) except epr.EPRValueError: logger.error("EPRValueError") return False lats_2 = imresize(lats, raw_counts.shape) lons_2 = imresize(lons, raw_counts.shape) # if lats.max() <= 35: # logger.debug("skipping no area overlap") # return False # Trimming the array by removing zero values from rows and cols msk = [] for m in range(raw_counts.shape[0]): if raw_counts[m, :].sum() == 0: msk.append(m) raw_counts = delete(raw_counts, msk, axis=0) lats_2 = delete(lats_2, msk, axis=0) lons_2 = delete(lons_2, msk, axis=0) incident_angle = delete(incident_angle, msk, axis=0) polarization = product.get_sph().get_field('MDS1_TX_RX_POLAR').get_elem() msk = [] for n in range(raw_counts.shape[1]): if raw_counts[:, n].sum() == 0: msk.append(n) raw_counts = delete(raw_counts, msk, axis=1) lats_2 = delete(lats_2, msk, axis=1) lons_2 = delete(lons_2, msk, axis=1) incident_angle = delete(incident_angle, msk, axis=1) # Adding Sigma_0 calibration_constant = product.get_dataset( 'MAIN_PROCESSING_PARAMS_ADS').read_record(0).get_field( 'calibration_factors.1.ext_cal_fact').get_elems() # sigma0 = 10*log10( raw_counts**2*sin(incident_angle*pi/180)/calibration_constant ) sigma0 = raw_counts**2 * sin( incident_angle * pi / 180) / calibration_constant return sigma0, lats_2, lons_2, incident_angle, polarization, lats, lons