def s2_pre_processing(s2_dir, cams_dir, dem):
s2_dir = os.path.abspath(s2_dir)
scihub = []
aws = []
for (dirpath, dirnames, filenames) in os.walk(s2_dir):
if len(filenames) > 0:
temp = [dirpath + '/' + i for i in filenames]
for j in temp:
if ('MTD' in j) & ('TL' in j) & ('xml' in j):
scihub.append(j)
if 'metadata.xml' in j:
aws.append(j)
s2_tiles = []
for metafile in scihub + aws:
with open(metafile) as f:
for i in f.readlines():
if 'TILE_ID' in i:
tile = i.split('</')[0].split('>')[-1]
if 'SENSING_TIME' in i:
sensing_time = i.split('</')[0].split('>')[-1]
obs_time = datetime.datetime.strptime(
sensing_time, u'%Y-%m-%dT%H:%M:%S.%fZ')
log_file = os.path.dirname(metafile) + '/SIAC_S2.log'
if os.path.exists(log_file):
os.remove(log_file)
logger = create_logger(log_file)
logger.propagate = False
logger.info('Preprocessing for %s' % tile)
logger.info('Doing per pixel angle resampling.')
sun_ang_name, view_ang_names, toa_refs, cloud_name = resample_s2_angles(
metafile)
cloud_bands = np.array(toa_refs)[[0, 1, 3, 4, 7, 8, 9, 10, 11, 12]]
logger.info('Getting cloud mask.')
cloud = do_cloud(cloud_bands, cloud_name)
cloud_mask = cloud > 0.6 # 0.4 is the sentinel hub default
clean_pixel = ((cloud >= 0).sum() - cloud_mask.sum()) / 3348900. * 100.
valid_pixel = (cloud >= 0).sum() / 3348900. * 100.
logger.info('Valid pixel percentage: %.02f' % (valid_pixel))
logger.info('Clean pixel percentage: %.02f' % (clean_pixel))
cloud_mask = binary_dilation(binary_erosion(cloud_mask, disk(2)),
disk(3)) | (cloud < 0)
#logger.info('Getting first guess AOT and TCWV.')
#tcwv_bands = np.array(toa_refs)[[9,8]]
#aot_bands = np.array(toa_refs)
#view_ang_name = view_ang_names[8]
#aot, tcwv = do_aot_tcwv(aot_bands, tcwv_bands, cams_dir, obs_time, sun_ang_name, view_ang_name, dem, tcwv_name = None, aot_name = None)
aot = None
tcwv = None
s2_tiles.append([
sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask,
aot, tcwv, metafile
])
handlers = logger.handlers[:]
for handler in handlers:
handler.close()
logger.removeHandler(handler)
return s2_tiles
def __init__(self,
sensor_sat,
toa_bands,
band_index,
view_angles,
sun_angles,
aoi = None,
aot = None,
tcwv = None,
tco3 = None,
aot_unc = None,
tcwv_unc = None,
tco3_unc = None,
cloud_mask = None,
obs_time = None,
log_file = None,
a_z_order = 1,
ref_scale = 0.0001,
ref_off = 0,
ang_scale = 0.01,
ele_scale = 0.001,
atmo_scale = [1., 1., 1., 1., 1., 1.],
global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt',
cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/',
emus_dir = 'SIAC/emus/',
cams_scale = [1., 0.1, 46.698, 1., 1., 1.],
block_size = 600,
rgb = [None, None, None]
):
self.sensor = sensor_sat[0]
self.satellite = sensor_sat[1]
self.toa_bands = toa_bands
self.band_index = band_index
self.view_angles = view_angles
self.sun_angles = sun_angles
self.aoi = aoi
self.aot = aot
self.tcwv = tcwv
self.tco3 = tco3
self.aot_unc = aot_unc
self.tcwv_unc = tcwv_unc
self.tco3_unc = tco3_unc
self.cloud_mask = cloud_mask
self.obs_time = obs_time
self.a_z_order = a_z_order
self.ref_scale = ref_scale
self.ref_off = ref_off
self.ang_scale = ang_scale
self.ele_scale = ele_scale
self.atmo_scale = atmo_scale
self.global_dem = global_dem
self.emus_dir = emus_dir
self.cams_dir = cams_dir
self.cams_scale = cams_scale
self.block_size = block_size
self.toa_dir = os.path.abspath('/'.join(toa_bands[0].split('/')[:-1]))
self.rgb = rgb
r, g, b = self.rgb
self._do_rgb = False
self.ri, self.gi, self.bi = None, None, None
if (r is not None) & (g is not None) & (b is not None):
self.ri, self.gi, self.bi = self.toa_bands.index(r), self.toa_bands.index(g), self.toa_bands.index(b)
self._do_rgb = True
self.logger = create_logger(log_file)
def get_mcd43(aoi,
obs_time,
mcd43_dir='./MCD43/',
vrt_dir='./MCD43_VRT/',
log_file=None,
jasmin=False):
mcd43_dir = os.path.expanduser(mcd43_dir) # incase ~ used
vrt_dir = os.path.expanduser(vrt_dir)
logger = create_logger(log_file)
logger.propagate = False
logger.info('Querying MCD43 files...')
ret = find_files(aoi,
obs_time,
mcd43_dir,
temporal_window=16,
jasmin=jasmin)
urls = [granule for granule in ret if granule.find("http") >= 0]
url_fnames = [granule for granule in ret if granule.find("http") < 0]
flist = url_fnames
logger.info("Will need to download {:d} files, ".format(len(urls)) +
"{:d} are already present".format(len(url_fnames)))
if len(urls) > 0:
logger.info(
'Start downloading MCD43 for the AOI, this may take some time.')
url_fnames_to_get = [[i, mcd43_dir + '/' + i.split('/')[-1]]
for i in urls]
p = Pool(5)
logger.info('Start downloading...')
auth = get_auth(logger)
par = partial(downloader, auth=auth)
ret = p.map(par, url_fnames_to_get)
p.close()
p.join()
flist.extend([x[1] for x in url_fnames_to_get])
flist = np.array(flist)
all_dates = np.array([i.split('/')[-1].split('.')[1][1:9] for i in flist])
udates = np.unique(all_dates)
fnames_dates = [[flist[all_dates == date].tolist(), date]
for date in udates]
logger.info('Creating daily VRT...')
if jasmin:
vrt_dir = tempfile.TemporaryDirectory(dir=vrt_dir).name + '/'
if not os.path.exists(vrt_dir):
os.mkdir(vrt_dir)
par = partial(daily_vrt_jasmin, vrt_dir=vrt_dir)
p = Pool(len(fnames_dates))
p.map(par, fnames_dates)
p.close()
p.join()
else:
par = partial(daily_vrt, vrt_dir=vrt_dir)
p = Pool(len(fnames_dates))
p.map(par, fnames_dates)
p.close()
p.join()
#par(fnames_dates[0])
handlers = logger.handlers[:]
for handler in handlers:
handler.close()
logger.removeHandler(handler)
return vrt_dir
def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \
cloud_mask, aot, tcwv, metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi=None, \
global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt', \
cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/', jasmin = False):
if jasmin:
global_dem = '/work/scratch/marcyin/DEM/global_dem.vrt'
cams_dir = '/work/scratch/marcyin/CAMS/'
os.environ['jasmin_memory_limit'] = '6.4e+10'
#vrt_dir = '/work/scratch/marcyin/MCD43_VRT/'
if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'):
if not os.path.exists(home + '/MCD43/'):
os.mkdir(home + '/MCD43/')
if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT/'):
if not os.path.exists(home + '/MCD43_VRT/'):
os.mkdir(home + '/MCD43_VRT/')
base = os.path.dirname(toa_refs[0])
base = toa_refs[0].replace('B01.jp2', '')
with open(metafile) as f:
for i in f.readlines():
if 'SENSING_TIME' in i:
sensing_time = i.split('</')[0].split('>')[-1]
obs_time = datetime.strptime(sensing_time, u'%Y-%m-%dT%H:%M:%S.%fZ')
if 'TILE_ID' in i:
sat = i.split('</')[0].split('>')[-1].split('_')[0]
tile = i.split('</')[0].split('>')[-1]
log_file = os.path.dirname(metafile) + '/SIAC_S2.log'
logger = create_logger(log_file)
logger.info('Starting atmospheric corretion for %s'%tile)
if not np.all(cloud_mask):
handlers = logger.handlers[:]
for handler in handlers:
handler.close()
logger.removeHandler(handler)
#if not jasmin:
vrt_dir = get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir, log_file = log_file, jasmin = jasmin)
#logger = create_logger(log_file)
else:
logger.info('No clean pixel in this scene and no MCD43 is downloaded.')
sensor_sat = 'MSI', sat
band_index = [1,2,3,7,11,12]
band_wv = [469, 555, 645, 859, 1640, 2130]
toa_bands = (np.array(toa_refs)[band_index,]).tolist()
view_angles = (np.array(view_ang_names)[band_index,]).tolist()
sun_angles = sun_ang_name
logger.info('First pass AOT and TCWV: %.02f, %.02f'%(aot.mean(), tcwv.mean()))
#logger.info('Running SIAC for tile: %s on %s'%(tile, obs_time.strftime('%Y-%M-%d')))
aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,\
sun_angles,obs_time,cloud_mask, gamma=10., spec_m_dir= \
file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', wv_prior=tcwv, wv_unc=0.1,\
mcd43_dir=vrt_dir, aoi=aoi, log_file = log_file, global_dem = global_dem, cams_dir = cams_dir, \
prior_scale = [1., 1, 46.698, 1., 1., 1.])
aero._solving()
toa_bands = toa_refs
view_angles = view_ang_names
aot = base + 'aot.tif'
tcwv = base + 'tcwv.tif'
tco3 = base + 'tco3.tif'
aot_unc = base + 'aot_unc.tif'
tcwv_unc = base + 'tcwv_unc.tif'
tco3_unc = base + 'tco3_unc.tif'
rgb = [toa_bands[3], toa_bands[2], toa_bands[1]]
band_index = [0,1,2,3,4,5,6,7,8,9,10,11,12]
atmo = atmospheric_correction(sensor_sat,toa_bands, band_index,view_angles,\
sun_angles, aot = aot, cloud_mask = cloud_mask,\
tcwv = tcwv, tco3 = tco3, aot_unc = aot_unc, \
tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, rgb = \
rgb, emus_dir=file_path+'/emus/', log_file = log_file, global_dem = global_dem, cams_dir = cams_dir)
atmo._doing_correction()
if jasmin:
shutil.rmtree(vrt_dir)
return aero, atmo
import getpass
import logging
import requests
import numpy as np
from glob import glob
from six.moves import input
from functools import partial
from os.path import expanduser
from multiprocessing import Pool
from datetime import datetime, timedelta
from SIAC.create_logger import create_logger
from SIAC.modis_tile_cal import get_vector_hv, get_raster_hv
home = expanduser("~")
file_path = os.path.dirname(os.path.realpath(__file__))
logger = create_logger()
test_url = 'https://e4ftl01.cr.usgs.gov/MOTA/MCD43A1.006/2000.02.24/MCD43A1.A2000055.h32v08.006.2016101152216.hdf.xml'
try:
auth = tuple([os.environ['Earthdata_user'], os.environ['Earthdata_pass']])
with requests.Session() as s:
s.auth = auth
r1 = s.get(test_url)
r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'})
if r.status_code == 401:
logger.error('Wrong username and password are set for Earthdata_user and Earthdata_pass Environment variables.')
else:
with open(file_path + '/data/.earthdata_auth', 'wb') as f:
for i in auth:
f.write((i + '\n').encode())
except:
def __init__(
self,
sensor_sat,
toa_bands,
band_wv,
band_index,
view_angles,
sun_angles,
obs_time,
cloud_mask,
gamma=10.,
a_z_order=1,
pixel_res=None,
aoi=None,
aot_prior=None,
wv_prior=None,
o3_prior=None,
aot_unc=None,
wv_unc=None,
o3_unc=None,
log_file=None,
ref_scale=0.0001,
ref_off=0,
ang_scale=0.01,
ele_scale=0.001,
prior_scale=[1., 0.1, 46.698, 1., 1., 1.],
emus_dir='SIAC/emus/',
mcd43_dir='~/DATA/Multiply/MCD43/',
global_dem='/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt',
cams_dir='/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/',
spec_m_dir='SIAC/spectral_mapping',
aero_res=1000):
self.sensor = sensor_sat[0]
self.satellite = sensor_sat[1]
self.toa_bands = toa_bands
self.band_wv = band_wv
self.band_index = band_index
self.view_angles = view_angles
self.sun_angles = sun_angles
self.obs_time = obs_time
self.cloud_mask = cloud_mask
self.gamma = gamma
self.a_z_order = a_z_order
self.pixel_res = pixel_res
self.aoi = aoi
self.aot_prior = aot_prior
self.wv_prior = wv_prior
self.o3_prior = o3_prior
self.aot_unc = aot_unc
self.wv_unc = wv_unc
self.o3_unc = o3_unc
self.log_file = log_file
self.ref_scale = ref_scale
self.ref_off = ref_off
self.ang_scale = ang_scale
self.ele_scale = ele_scale
self.prior_scale = prior_scale
self.mcd43_dir = mcd43_dir
self.emus_dir = emus_dir
self.global_dem = global_dem
self.cams_dir = cams_dir
self.boa_wv = [645, 859, 469, 555, 1640, 2130]
self.aero_res = aero_res
self.mcd43_tmp = '%s/MCD43A1.A%d%03d.%s.006.*.hdf'
self.toa_dir = os.path.abspath('/'.join(toa_bands[0].split('/')[:-1]))
try:
#spec_map = np.loadtxt(spec_m_dir + '/Aqua_%s_spectral_mapping.txt'%self.satellite).T
self.spec_map = Two_NN(
np_model_file=spec_m_dir +
'/Aqua_%s_spectral_mapping.npz' % self.satellite)
except:
#spec_map = np.loadtxt(spec_m_dir + '/TERRA_%s_spectral_mapping.txt'%self.sensor).T
self.spec_map = Two_NN(
np_model_file=spec_m_dir +
'/Aqua_%s_spectral_mapping.npz' % self.sensor)
#self.spec_slope = spec_map[0]
#self.spec_off = spec_map[1]
self.logger = create_logger(self.log_file)
def do_correction(sun_ang_name, view_ang_names, toa_refs, qa_name, cloud_mask, \
metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi = None,\
global_dem = None, cams_dir = None, jasmin = False):
if jasmin:
if global_dem is None:
global_dem = '/work/scratch-pw/marcyin/DEM/global_dem.vrt'
if cams_dir is None:
cams_dir = '/work/scratch-pw/marcyin/CAMS/'
os.environ['jasmin_memory_limit'] = '6.4e+10'
else:
if global_dem is None:
global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt'
if cams_dir is None:
cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/'
if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'):
if not os.path.exists(home + '/MCD43/'):
os.mkdir(home + '/MCD43/')
if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT/'):
if not os.path.exists(home + '/MCD43_VRT/'):
os.mkdir(home + '/MCD43_VRT/')
base = os.path.dirname(toa_refs[0])
with open(metafile) as f:
for line in f:
if 'REFLECTANCE_MULT_BAND' in line:
scale = float(line.split()[-1])
elif 'REFLECTANCE_ADD_BAND' in line:
off = float(line.split()[-1])
elif 'DATE_ACQUIRED' in line:
date = line.split()[-1]
elif 'SCENE_CENTER_TIME' in line:
time = line.split()[-1]
log_file = os.path.dirname(metafile) + '/SIAC_L8.log'
logger = create_logger(log_file)
logger.info('Starting atmospheric corretion for %s' % base)
datetime_str = date + time
obs_time = datetime.strptime(
datetime_str.split('.')[0], '%Y-%m-%d"%H:%M:%S')
if not np.all(cloud_mask):
# handlers = logger.handlers[:]
# for handler in handlers:
# handler.close()
# logger.removeHandler(handler)
#if not jasmin:
vrt_dir = get_mcd43(toa_refs[0],
obs_time,
mcd43_dir=mcd43,
vrt_dir=vrt_dir,
logger=logger,
jasmin=jasmin)
#logger = create_logger(log_file)
else:
logger.info('No clean pixel in this scene and no MCD43 is downloaded.')
#get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir)
sensor_sat = 'OLI', 'L8'
band_index = [1, 2, 3, 4, 5, 6]
band_wv = [469, 555, 645, 859, 1640, 2130]
toa_bands = (np.array(toa_refs)[band_index, ]).tolist()
view_angles = (np.array(view_ang_names)[band_index, ]).tolist()
sun_angles = sun_ang_name
# sza = gdal.Open(sun_angles).ReadAsArray()[1] * 0.01
pixel_res = abs(gdal.Open(str(sun_angles)).GetGeoTransform()[1])
g = gdal.Warp('', str(sun_angles), format = 'MEM', xRes = pixel_res, yRes = pixel_res, warpOptions = ['NUM_THREADS=ALL_CPUS'],\
srcNodata = 0, dstNodata=0, cutlineDSName= aoi, cropToCutline=True, resampleAlg = 0)
sza = g.ReadAsArray()[1] * 0.01
scale = scale / np.cos(np.deg2rad(sza))
off = off / np.cos(np.deg2rad(sza))
aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,sun_angles,\
obs_time,cloud_mask, gamma=10., ref_scale = scale, ref_off = off, global_dem = global_dem, cams_dir = cams_dir,\
spec_m_dir=file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir, aoi=aoi, log_file = log_file)
aero._solving()
toa_bands = toa_refs
view_angles = view_ang_names
base = base + '/' + 'B'.join(toa_bands[0].split('/')[-1].split('B')[:-1])
aot = base + 'aot.tif'
tcwv = base + 'tcwv.tif'
tco3 = base + 'tco3.tif'
aot_unc = base + 'aot_unc.tif'
tcwv_unc = base + 'tcwv_unc.tif'
tco3_unc = base + 'tco3_unc.tif'
rgb = [toa_bands[3], toa_bands[2], toa_bands[1]]
band_index = [0, 1, 2, 3, 4, 5, 6]
atmo = atmospheric_correction(sensor_sat,toa_bands, band_index,view_angles,sun_angles, \
aot = aot, cloud_mask = cloud_mask,tcwv = tcwv, tco3 = tco3, \
aot_unc = aot_unc, tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, global_dem = global_dem, cams_dir = cams_dir,\
rgb = rgb, ref_scale = scale, ref_off = off, emus_dir=file_path+'/emus/', aoi=aoi, log_file = log_file)
atmo._doing_correction()
return aero, atmo
