def s2_pre_processing(s2_dir, cams_dir, dem): s2_dir = os.path.abspath(s2_dir) scihub = [] aws = [] for (dirpath, dirnames, filenames) in os.walk(s2_dir): if len(filenames) > 0: temp = [dirpath + '/' + i for i in filenames] for j in temp: if ('MTD' in j) & ('TL' in j) & ('xml' in j): scihub.append(j) if 'metadata.xml' in j: aws.append(j) s2_tiles = [] for metafile in scihub + aws: with open(metafile) as f: for i in f.readlines(): if 'TILE_ID' in i: tile = i.split('</')[0].split('>')[-1] if 'SENSING_TIME' in i: sensing_time = i.split('</')[0].split('>')[-1] obs_time = datetime.datetime.strptime( sensing_time, u'%Y-%m-%dT%H:%M:%S.%fZ') log_file = os.path.dirname(metafile) + '/SIAC_S2.log' if os.path.exists(log_file): os.remove(log_file) logger = create_logger(log_file) logger.propagate = False logger.info('Preprocessing for %s' % tile) logger.info('Doing per pixel angle resampling.') sun_ang_name, view_ang_names, toa_refs, cloud_name = resample_s2_angles( metafile) cloud_bands = np.array(toa_refs)[[0, 1, 3, 4, 7, 8, 9, 10, 11, 12]] logger.info('Getting cloud mask.') cloud = do_cloud(cloud_bands, cloud_name) cloud_mask = cloud > 0.6 # 0.4 is the sentinel hub default clean_pixel = ((cloud >= 0).sum() - cloud_mask.sum()) / 3348900. * 100. valid_pixel = (cloud >= 0).sum() / 3348900. * 100. logger.info('Valid pixel percentage: %.02f' % (valid_pixel)) logger.info('Clean pixel percentage: %.02f' % (clean_pixel)) cloud_mask = binary_dilation(binary_erosion(cloud_mask, disk(2)), disk(3)) | (cloud < 0) #logger.info('Getting first guess AOT and TCWV.') #tcwv_bands = np.array(toa_refs)[[9,8]] #aot_bands = np.array(toa_refs) #view_ang_name = view_ang_names[8] #aot, tcwv = do_aot_tcwv(aot_bands, tcwv_bands, cams_dir, obs_time, sun_ang_name, view_ang_name, dem, tcwv_name = None, aot_name = None) aot = None tcwv = None s2_tiles.append([ sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, aot, tcwv, metafile ]) handlers = logger.handlers[:] for handler in handlers: handler.close() logger.removeHandler(handler) return s2_tiles
def __init__(self, sensor_sat, toa_bands, band_index, view_angles, sun_angles, aoi = None, aot = None, tcwv = None, tco3 = None, aot_unc = None, tcwv_unc = None, tco3_unc = None, cloud_mask = None, obs_time = None, log_file = None, a_z_order = 1, ref_scale = 0.0001, ref_off = 0, ang_scale = 0.01, ele_scale = 0.001, atmo_scale = [1., 1., 1., 1., 1., 1.], global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt', cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/', emus_dir = 'SIAC/emus/', cams_scale = [1., 0.1, 46.698, 1., 1., 1.], block_size = 600, rgb = [None, None, None] ): self.sensor = sensor_sat[0] self.satellite = sensor_sat[1] self.toa_bands = toa_bands self.band_index = band_index self.view_angles = view_angles self.sun_angles = sun_angles self.aoi = aoi self.aot = aot self.tcwv = tcwv self.tco3 = tco3 self.aot_unc = aot_unc self.tcwv_unc = tcwv_unc self.tco3_unc = tco3_unc self.cloud_mask = cloud_mask self.obs_time = obs_time self.a_z_order = a_z_order self.ref_scale = ref_scale self.ref_off = ref_off self.ang_scale = ang_scale self.ele_scale = ele_scale self.atmo_scale = atmo_scale self.global_dem = global_dem self.emus_dir = emus_dir self.cams_dir = cams_dir self.cams_scale = cams_scale self.block_size = block_size self.toa_dir = os.path.abspath('/'.join(toa_bands[0].split('/')[:-1])) self.rgb = rgb r, g, b = self.rgb self._do_rgb = False self.ri, self.gi, self.bi = None, None, None if (r is not None) & (g is not None) & (b is not None): self.ri, self.gi, self.bi = self.toa_bands.index(r), self.toa_bands.index(g), self.toa_bands.index(b) self._do_rgb = True self.logger = create_logger(log_file)
def get_mcd43(aoi, obs_time, mcd43_dir='./MCD43/', vrt_dir='./MCD43_VRT/', log_file=None, jasmin=False): mcd43_dir = os.path.expanduser(mcd43_dir) # incase ~ used vrt_dir = os.path.expanduser(vrt_dir) logger = create_logger(log_file) logger.propagate = False logger.info('Querying MCD43 files...') ret = find_files(aoi, obs_time, mcd43_dir, temporal_window=16, jasmin=jasmin) urls = [granule for granule in ret if granule.find("http") >= 0] url_fnames = [granule for granule in ret if granule.find("http") < 0] flist = url_fnames logger.info("Will need to download {:d} files, ".format(len(urls)) + "{:d} are already present".format(len(url_fnames))) if len(urls) > 0: logger.info( 'Start downloading MCD43 for the AOI, this may take some time.') url_fnames_to_get = [[i, mcd43_dir + '/' + i.split('/')[-1]] for i in urls] p = Pool(5) logger.info('Start downloading...') auth = get_auth(logger) par = partial(downloader, auth=auth) ret = p.map(par, url_fnames_to_get) p.close() p.join() flist.extend([x[1] for x in url_fnames_to_get]) flist = np.array(flist) all_dates = np.array([i.split('/')[-1].split('.')[1][1:9] for i in flist]) udates = np.unique(all_dates) fnames_dates = [[flist[all_dates == date].tolist(), date] for date in udates] logger.info('Creating daily VRT...') if jasmin: vrt_dir = tempfile.TemporaryDirectory(dir=vrt_dir).name + '/' if not os.path.exists(vrt_dir): os.mkdir(vrt_dir) par = partial(daily_vrt_jasmin, vrt_dir=vrt_dir) p = Pool(len(fnames_dates)) p.map(par, fnames_dates) p.close() p.join() else: par = partial(daily_vrt, vrt_dir=vrt_dir) p = Pool(len(fnames_dates)) p.map(par, fnames_dates) p.close() p.join() #par(fnames_dates[0]) handlers = logger.handlers[:] for handler in handlers: handler.close() logger.removeHandler(handler) return vrt_dir
def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \ cloud_mask, aot, tcwv, metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi=None, \ global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt', \ cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/', jasmin = False): if jasmin: global_dem = '/work/scratch/marcyin/DEM/global_dem.vrt' cams_dir = '/work/scratch/marcyin/CAMS/' os.environ['jasmin_memory_limit'] = '6.4e+10' #vrt_dir = '/work/scratch/marcyin/MCD43_VRT/' if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'): if not os.path.exists(home + '/MCD43/'): os.mkdir(home + '/MCD43/') if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT/'): if not os.path.exists(home + '/MCD43_VRT/'): os.mkdir(home + '/MCD43_VRT/') base = os.path.dirname(toa_refs[0]) base = toa_refs[0].replace('B01.jp2', '') with open(metafile) as f: for i in f.readlines(): if 'SENSING_TIME' in i: sensing_time = i.split('</')[0].split('>')[-1] obs_time = datetime.strptime(sensing_time, u'%Y-%m-%dT%H:%M:%S.%fZ') if 'TILE_ID' in i: sat = i.split('</')[0].split('>')[-1].split('_')[0] tile = i.split('</')[0].split('>')[-1] log_file = os.path.dirname(metafile) + '/SIAC_S2.log' logger = create_logger(log_file) logger.info('Starting atmospheric corretion for %s'%tile) if not np.all(cloud_mask): handlers = logger.handlers[:] for handler in handlers: handler.close() logger.removeHandler(handler) #if not jasmin: vrt_dir = get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir, log_file = log_file, jasmin = jasmin) #logger = create_logger(log_file) else: logger.info('No clean pixel in this scene and no MCD43 is downloaded.') sensor_sat = 'MSI', sat band_index = [1,2,3,7,11,12] band_wv = [469, 555, 645, 859, 1640, 2130] toa_bands = (np.array(toa_refs)[band_index,]).tolist() view_angles = (np.array(view_ang_names)[band_index,]).tolist() sun_angles = sun_ang_name logger.info('First pass AOT and TCWV: %.02f, %.02f'%(aot.mean(), tcwv.mean())) #logger.info('Running SIAC for tile: %s on %s'%(tile, obs_time.strftime('%Y-%M-%d'))) aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,\ sun_angles,obs_time,cloud_mask, gamma=10., spec_m_dir= \ file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', wv_prior=tcwv, wv_unc=0.1,\ mcd43_dir=vrt_dir, aoi=aoi, log_file = log_file, global_dem = global_dem, cams_dir = cams_dir, \ prior_scale = [1., 1, 46.698, 1., 1., 1.]) aero._solving() toa_bands = toa_refs view_angles = view_ang_names aot = base + 'aot.tif' tcwv = base + 'tcwv.tif' tco3 = base + 'tco3.tif' aot_unc = base + 'aot_unc.tif' tcwv_unc = base + 'tcwv_unc.tif' tco3_unc = base + 'tco3_unc.tif' rgb = [toa_bands[3], toa_bands[2], toa_bands[1]] band_index = [0,1,2,3,4,5,6,7,8,9,10,11,12] atmo = atmospheric_correction(sensor_sat,toa_bands, band_index,view_angles,\ sun_angles, aot = aot, cloud_mask = cloud_mask,\ tcwv = tcwv, tco3 = tco3, aot_unc = aot_unc, \ tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, rgb = \ rgb, emus_dir=file_path+'/emus/', log_file = log_file, global_dem = global_dem, cams_dir = cams_dir) atmo._doing_correction() if jasmin: shutil.rmtree(vrt_dir) return aero, atmo
import getpass import logging import requests import numpy as np from glob import glob from six.moves import input from functools import partial from os.path import expanduser from multiprocessing import Pool from datetime import datetime, timedelta from SIAC.create_logger import create_logger from SIAC.modis_tile_cal import get_vector_hv, get_raster_hv home = expanduser("~") file_path = os.path.dirname(os.path.realpath(__file__)) logger = create_logger() test_url = 'https://e4ftl01.cr.usgs.gov/MOTA/MCD43A1.006/2000.02.24/MCD43A1.A2000055.h32v08.006.2016101152216.hdf.xml' try: auth = tuple([os.environ['Earthdata_user'], os.environ['Earthdata_pass']]) with requests.Session() as s: s.auth = auth r1 = s.get(test_url) r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'}) if r.status_code == 401: logger.error('Wrong username and password are set for Earthdata_user and Earthdata_pass Environment variables.') else: with open(file_path + '/data/.earthdata_auth', 'wb') as f: for i in auth: f.write((i + '\n').encode()) except:
def __init__( self, sensor_sat, toa_bands, band_wv, band_index, view_angles, sun_angles, obs_time, cloud_mask, gamma=10., a_z_order=1, pixel_res=None, aoi=None, aot_prior=None, wv_prior=None, o3_prior=None, aot_unc=None, wv_unc=None, o3_unc=None, log_file=None, ref_scale=0.0001, ref_off=0, ang_scale=0.01, ele_scale=0.001, prior_scale=[1., 0.1, 46.698, 1., 1., 1.], emus_dir='SIAC/emus/', mcd43_dir='~/DATA/Multiply/MCD43/', global_dem='/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt', cams_dir='/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/', spec_m_dir='SIAC/spectral_mapping', aero_res=1000): self.sensor = sensor_sat[0] self.satellite = sensor_sat[1] self.toa_bands = toa_bands self.band_wv = band_wv self.band_index = band_index self.view_angles = view_angles self.sun_angles = sun_angles self.obs_time = obs_time self.cloud_mask = cloud_mask self.gamma = gamma self.a_z_order = a_z_order self.pixel_res = pixel_res self.aoi = aoi self.aot_prior = aot_prior self.wv_prior = wv_prior self.o3_prior = o3_prior self.aot_unc = aot_unc self.wv_unc = wv_unc self.o3_unc = o3_unc self.log_file = log_file self.ref_scale = ref_scale self.ref_off = ref_off self.ang_scale = ang_scale self.ele_scale = ele_scale self.prior_scale = prior_scale self.mcd43_dir = mcd43_dir self.emus_dir = emus_dir self.global_dem = global_dem self.cams_dir = cams_dir self.boa_wv = [645, 859, 469, 555, 1640, 2130] self.aero_res = aero_res self.mcd43_tmp = '%s/MCD43A1.A%d%03d.%s.006.*.hdf' self.toa_dir = os.path.abspath('/'.join(toa_bands[0].split('/')[:-1])) try: #spec_map = np.loadtxt(spec_m_dir + '/Aqua_%s_spectral_mapping.txt'%self.satellite).T self.spec_map = Two_NN( np_model_file=spec_m_dir + '/Aqua_%s_spectral_mapping.npz' % self.satellite) except: #spec_map = np.loadtxt(spec_m_dir + '/TERRA_%s_spectral_mapping.txt'%self.sensor).T self.spec_map = Two_NN( np_model_file=spec_m_dir + '/Aqua_%s_spectral_mapping.npz' % self.sensor) #self.spec_slope = spec_map[0] #self.spec_off = spec_map[1] self.logger = create_logger(self.log_file)
def do_correction(sun_ang_name, view_ang_names, toa_refs, qa_name, cloud_mask, \ metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi = None,\ global_dem = None, cams_dir = None, jasmin = False): if jasmin: if global_dem is None: global_dem = '/work/scratch-pw/marcyin/DEM/global_dem.vrt' if cams_dir is None: cams_dir = '/work/scratch-pw/marcyin/CAMS/' os.environ['jasmin_memory_limit'] = '6.4e+10' else: if global_dem is None: global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt' if cams_dir is None: cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/' if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'): if not os.path.exists(home + '/MCD43/'): os.mkdir(home + '/MCD43/') if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT/'): if not os.path.exists(home + '/MCD43_VRT/'): os.mkdir(home + '/MCD43_VRT/') base = os.path.dirname(toa_refs[0]) with open(metafile) as f: for line in f: if 'REFLECTANCE_MULT_BAND' in line: scale = float(line.split()[-1]) elif 'REFLECTANCE_ADD_BAND' in line: off = float(line.split()[-1]) elif 'DATE_ACQUIRED' in line: date = line.split()[-1] elif 'SCENE_CENTER_TIME' in line: time = line.split()[-1] log_file = os.path.dirname(metafile) + '/SIAC_L8.log' logger = create_logger(log_file) logger.info('Starting atmospheric corretion for %s' % base) datetime_str = date + time obs_time = datetime.strptime( datetime_str.split('.')[0], '%Y-%m-%d"%H:%M:%S') if not np.all(cloud_mask): # handlers = logger.handlers[:] # for handler in handlers: # handler.close() # logger.removeHandler(handler) #if not jasmin: vrt_dir = get_mcd43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir, logger=logger, jasmin=jasmin) #logger = create_logger(log_file) else: logger.info('No clean pixel in this scene and no MCD43 is downloaded.') #get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir) sensor_sat = 'OLI', 'L8' band_index = [1, 2, 3, 4, 5, 6] band_wv = [469, 555, 645, 859, 1640, 2130] toa_bands = (np.array(toa_refs)[band_index, ]).tolist() view_angles = (np.array(view_ang_names)[band_index, ]).tolist() sun_angles = sun_ang_name # sza = gdal.Open(sun_angles).ReadAsArray()[1] * 0.01 pixel_res = abs(gdal.Open(str(sun_angles)).GetGeoTransform()[1]) g = gdal.Warp('', str(sun_angles), format = 'MEM', xRes = pixel_res, yRes = pixel_res, warpOptions = ['NUM_THREADS=ALL_CPUS'],\ srcNodata = 0, dstNodata=0, cutlineDSName= aoi, cropToCutline=True, resampleAlg = 0) sza = g.ReadAsArray()[1] * 0.01 scale = scale / np.cos(np.deg2rad(sza)) off = off / np.cos(np.deg2rad(sza)) aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,sun_angles,\ obs_time,cloud_mask, gamma=10., ref_scale = scale, ref_off = off, global_dem = global_dem, cams_dir = cams_dir,\ spec_m_dir=file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir, aoi=aoi, log_file = log_file) aero._solving() toa_bands = toa_refs view_angles = view_ang_names base = base + '/' + 'B'.join(toa_bands[0].split('/')[-1].split('B')[:-1]) aot = base + 'aot.tif' tcwv = base + 'tcwv.tif' tco3 = base + 'tco3.tif' aot_unc = base + 'aot_unc.tif' tcwv_unc = base + 'tcwv_unc.tif' tco3_unc = base + 'tco3_unc.tif' rgb = [toa_bands[3], toa_bands[2], toa_bands[1]] band_index = [0, 1, 2, 3, 4, 5, 6] atmo = atmospheric_correction(sensor_sat,toa_bands, band_index,view_angles,sun_angles, \ aot = aot, cloud_mask = cloud_mask,tcwv = tcwv, tco3 = tco3, \ aot_unc = aot_unc, tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, global_dem = global_dem, cams_dir = cams_dir,\ rgb = rgb, ref_scale = scale, ref_off = off, emus_dir=file_path+'/emus/', aoi=aoi, log_file = log_file) atmo._doing_correction() return aero, atmo