def s2_pre_processing(s2_dir, cams_dir, dem):
s2_dir = os.path.abspath(s2_dir)
scihub = []
aws = []
for (dirpath, dirnames, filenames) in os.walk(s2_dir):
if len(filenames) > 0:
temp = [dirpath + '/' + i for i in filenames]
for j in temp:
if ('MTD' in j) & ('TL' in j) & ('xml' in j):
scihub.append(j)
if 'metadata.xml' in j:
aws.append(j)
s2_tiles = []
for metafile in scihub + aws:
with open(metafile) as f:
for i in f.readlines():
if 'TILE_ID' in i:
tile = i.split('</')[0].split('>')[-1]
if 'SENSING_TIME' in i:
sensing_time = i.split('</')[0].split('>')[-1]
obs_time = datetime.datetime.strptime(
sensing_time, u'%Y-%m-%dT%H:%M:%S.%fZ')
log_file = os.path.dirname(metafile) + '/zac_s2.log'
if os.path.exists(log_file):
os.remove(log_file)
logger = create_logger(log_file)
logger.propagate = False
logger.info('Preprocessing for %s' % tile)
logger.info('Doing per pixel angle resampling.')
sun_ang_name, view_ang_names, toa_refs, cloud_name = resample_s2_angles(
metafile)
cloud_bands = np.array(toa_refs)[[0, 1, 3, 4, 7, 8, 9, 10, 11, 12]]
logger.info('Getting cloud mask.')
cloud = do_cloud(cloud_bands, cloud_name)
cloud_mask = cloud > 0.6 # 0.4 is the sentinel hub default
clean_pixel = ((cloud >= 0).sum() - cloud_mask.sum()) / 3348900. * 100.
valid_pixel = (cloud >= 0).sum() / 3348900. * 100.
logger.info('Valid pixel percentage: %.02f' % (valid_pixel))
logger.info('Clean pixel percentage: %.02f' % (clean_pixel))
cloud_mask = binary_dilation(binary_erosion(cloud_mask, disk(2)),
disk(3)) | (cloud < 0)
#logger.info('Getting first guess AOT and TCWV.')
#tcwv_bands = np.array(toa_refs)[[9,8]]
#aot_bands = np.array(toa_refs)
#view_ang_name = view_ang_names[8]
#aot, tcwv = do_aot_tcwv(aot_bands, tcwv_bands, cams_dir, obs_time, sun_ang_name, view_ang_name, dem, tcwv_name = None, aot_name = None)
aot = None
tcwv = None
s2_tiles.append([
sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask,
aot, tcwv, metafile
])
handlers = logger.handlers[:]
for handler in handlers:
handler.close()
logger.removeHandler(handler)
return s2_tiles
def get_mcd43(aoi, obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT/', log_file = None, jasmin = False):
mcd43_dir = os.path.expanduser(mcd43_dir) # incase ~ used
vrt_dir = os.path.expanduser(vrt_dir)
logger = create_logger(log_file)
logger.propagate = False
logger.info('Querying MCD43 files...')
ret = find_files(aoi, obs_time, mcd43_dir, temporal_window = 16, jasmin=jasmin)
urls = [granule for granule in ret if granule.find("http") >= 0]
url_fnames= [granule for granule in ret if granule.find("http") < 0]
flist = url_fnames
logger.info("Will need to download {:d} files, ".format(len(urls)) +
"{:d} are already present".format(len(url_fnames)))
if len(urls) > 0:
logger.info('Start downloading MCD43 for the AOI, this may take some time.')
url_fnames_to_get = [[i, mcd43_dir + '/' + i.split('/')[-1]] for i in urls]
p = Pool(5)
logger.info('Start downloading...')
auth = get_auth(logger)
par = partial(downloader, auth = auth)
ret = p.map(par, url_fnames_to_get)
p.close()
p.join()
flist.extend([x[1] for x in url_fnames_to_get])
flist = np.array(flist)
all_dates = np.array([i.split('/')[-1] .split('.')[1][1:9] for i in flist])
udates = np.unique(all_dates)
fnames_dates = [[flist[all_dates==date].tolist(),date] for date in udates]
logger.info('Creating daily VRT...')
if jasmin:
vrt_dir = tempfile.TemporaryDirectory(dir = vrt_dir).name + '/'
if not os.path.exists(vrt_dir):
os.mkdir(vrt_dir)
par = partial(daily_vrt_jasmin, vrt_dir = vrt_dir)
p = Pool(len(fnames_dates))
p.map(par, fnames_dates)
p.close()
p.join()
else:
par = partial(daily_vrt, vrt_dir = vrt_dir)
p = Pool(len(fnames_dates))
p.map(par, fnames_dates)
p.close()
p.join()
#par(fnames_dates[0])
handlers = logger.handlers[:]
for handler in handlers:
handler.close()
logger.removeHandler(handler)
return vrt_dir
import os
import requests
import logging
from zac.create_logger import create_logger
'''
logger = logging.getLogger('zac')
logger.setLevel(logging.INFO)
if not logger.handlers:
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
'''
logger = create_logger()
def downloader(fname, url_root, file_dir):
logger.propagate = False
new_url = url_root + fname
new_req = requests.get(new_url)
if new_req.ok:
logger.info('downloading %s and save it at %s' % (fname, os.path.join(file_dir, fname)))
with open(os.path.join(file_dir, fname), 'wb') as fp:
#for chunk in new_req.iter_content(chunk_size=1024):
# if chunk:
fp.write(new_req.content)
else:
logger.error(new_req.content)
def __init__(
self,
sensor_sat,
toa_bands,
band_wv,
band_index,
view_angles,
sun_angles,
obs_time,
cloud_mask,
gamma=10.0,
a_z_order=1,
pixel_res=None,
aoi=None,
aot_prior=None,
wv_prior=None,
o3_prior=None,
aot_unc=None,
wv_unc=None,
o3_unc=None,
log_file=None,
ref_scale=0.0001,
ref_off=0,
ang_scale=0.01,
ele_scale=0.001,
prior_scale=[1.0, 0.1, 46.698, 1.0, 1.0, 1.0],
emus_dir="zac/emus/",
mcd43_dir="~/DATA/Multiply/MCD43/",
global_dem="/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt",
cams_dir="/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/",
spec_m_dir="zac/spectral_mapping",
aero_res=1000,
):
self.sensor = sensor_sat[0]
self.satellite = sensor_sat[1]
self.toa_bands = toa_bands
self.band_wv = band_wv
self.band_index = band_index
self.view_angles = view_angles
self.sun_angles = sun_angles
self.obs_time = obs_time
self.cloud_mask = cloud_mask
self.gamma = gamma
self.a_z_order = a_z_order
self.pixel_res = pixel_res
self.aoi = aoi
self.aot_prior = aot_prior
self.wv_prior = wv_prior
self.o3_prior = o3_prior
self.aot_unc = aot_unc
self.wv_unc = wv_unc
self.o3_unc = o3_unc
self.log_file = log_file
self.ref_scale = ref_scale
self.ref_off = ref_off
self.ang_scale = ang_scale
self.ele_scale = ele_scale
self.prior_scale = prior_scale
self.mcd43_dir = mcd43_dir
self.emus_dir = emus_dir
self.global_dem = global_dem
self.cams_dir = cams_dir
self.boa_wv = [645, 859, 469, 555, 1640, 2130]
self.aero_res = aero_res
self.mcd43_tmp = "%s/MCD43A1.A%d%03d.%s.006.*.hdf"
self.toa_dir = os.path.abspath("/".join(toa_bands[0].split("/")[:-1]))
try:
# spec_map = np.loadtxt(spec_m_dir + '/Aqua_%s_spectral_mapping.txt'%self.satellite).T
self.spec_map = Two_NN(
np_model_file=spec_m_dir +
"/Aqua_%s_spectral_mapping.npz" % self.satellite)
except:
# spec_map = np.loadtxt(spec_m_dir + '/TERRA_%s_spectral_mapping.txt'%self.sensor).T
self.spec_map = Two_NN(
np_model_file=spec_m_dir +
"/Aqua_%s_spectral_mapping.npz" % self.sensor)
# self.spec_slope = spec_map[0]
# self.spec_off = spec_map[1]
self.logger = create_logger(self.log_file)
def __init__(self,
sensor_sat,
toa_bands,
band_index,
view_angles,
sun_angles,
aoi = None,
aot = None,
tcwv = None,
tco3 = None,
aot_unc = None,
tcwv_unc = None,
tco3_unc = None,
cloud_mask = None,
obs_time = None,
log_file = None,
a_z_order = 1,
ref_scale = 0.0001,
ref_off = 0,
ang_scale = 0.01,
ele_scale = 0.001,
atmo_scale = [1., 1., 1., 1., 1., 1.],
global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt',
cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/',
emus_dir = 'zac/emus/',
cams_scale = [1., 0.1, 46.698, 1., 1., 1.],
block_size = 600,
rgb = [None, None, None]
):
self.sensor = sensor_sat[0]
self.satellite = sensor_sat[1]
self.toa_bands = toa_bands
self.band_index = band_index
self.view_angles = view_angles
self.sun_angles = sun_angles
self.aoi = aoi
self.aot = aot
self.tcwv = tcwv
self.tco3 = tco3
self.aot_unc = aot_unc
self.tcwv_unc = tcwv_unc
self.tco3_unc = tco3_unc
self.cloud_mask = cloud_mask
self.obs_time = obs_time
self.a_z_order = a_z_order
self.ref_scale = ref_scale
self.ref_off = ref_off
self.ang_scale = ang_scale
self.ele_scale = ele_scale
self.atmo_scale = atmo_scale
self.global_dem = global_dem
self.emus_dir = emus_dir
self.cams_dir = cams_dir
self.cams_scale = cams_scale
self.block_size = block_size
self.toa_dir = os.path.abspath('/'.join(toa_bands[0].split('/')[:-1]))
self.rgb = rgb
r, g, b = self.rgb
self._do_rgb = False
self.ri, self.gi, self.bi = None, None, None
if (r is not None) & (g is not None) & (b is not None):
self.ri, self.gi, self.bi = self.toa_bands.index(r), self.toa_bands.index(g), self.toa_bands.index(b)
self._do_rgb = True
self.logger = create_logger(log_file)
def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \
cloud_mask, aot, tcwv, metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi=None, \
global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt', \
cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/', jasmin = False):
if jasmin:
global_dem = '/work/scratch/marcyin/DEM/global_dem.vrt'
cams_dir = '/work/scratch/marcyin/CAMS/'
os.environ['jasmin_memory_limit'] = '6.4e+10'
#vrt_dir = '/work/scratch/marcyin/MCD43_VRT/'
if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'):
if not os.path.exists(home + '/MCD43/'):
os.mkdir(home + '/MCD43/')
if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT/'):
if not os.path.exists(home + '/MCD43_VRT/'):
os.mkdir(home + '/MCD43_VRT/')
base = os.path.dirname(toa_refs[0])
base = toa_refs[0].replace('B01.jp2', '')
with open(metafile) as f:
for i in f.readlines():
if 'SENSING_TIME' in i:
sensing_time = i.split('</')[0].split('>')[-1]
obs_time = datetime.strptime(sensing_time,
u'%Y-%m-%dT%H:%M:%S.%fZ')
if 'TILE_ID' in i:
sat = i.split('</')[0].split('>')[-1].split('_')[0]
tile = i.split('</')[0].split('>')[-1]
log_file = os.path.dirname(metafile) + '/zac_s2.log'
logger = create_logger(log_file)
logger.info('Starting atmospheric corretion for %s' % tile)
if not np.all(cloud_mask):
handlers = logger.handlers[:]
for handler in handlers:
handler.close()
logger.removeHandler(handler)
#if not jasmin:
vrt_dir = get_mcd43(toa_refs[0],
obs_time,
mcd43_dir=mcd43,
vrt_dir=vrt_dir,
log_file=log_file,
jasmin=jasmin)
#logger = create_logger(log_file)
else:
logger.info('No clean pixel in this scene and no MCD43 is downloaded.')
sensor_sat = 'MSI', sat
band_index = [1, 2, 3, 7, 11, 12]
band_wv = [469, 555, 645, 859, 1640, 2130]
toa_bands = (np.array(toa_refs)[band_index, ]).tolist()
view_angles = (np.array(view_ang_names)[band_index, ]).tolist()
sun_angles = sun_ang_name
#logger.info('First pass AOT and TCWV: %.02f, %.02f'%(aot.mean(), tcwv.mean()))
#logger.info('Running zac for tile: %s on %s'%(tile, obs_time.strftime('%Y-%M-%d')))
aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,\
sun_angles,obs_time,cloud_mask, gamma=10., spec_m_dir= \
file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/',\
mcd43_dir=vrt_dir, aoi=aoi, log_file = log_file, global_dem = global_dem, cams_dir = cams_dir, \
prior_scale = [1., 0.1, 46.698, 1., 1., 1.])
aero._solving()
toa_bands = toa_refs
view_angles = view_ang_names
aot = base + 'aot.tif'
tcwv = base + 'tcwv.tif'
tco3 = base + 'tco3.tif'
aot_unc = base + 'aot_unc.tif'
tcwv_unc = base + 'tcwv_unc.tif'
tco3_unc = base + 'tco3_unc.tif'
rgb = [toa_bands[3], toa_bands[2], toa_bands[1]]
band_index = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
atmo = atmospheric_correction(sensor_sat,toa_bands, band_index,view_angles,\
sun_angles, aot = aot, cloud_mask = cloud_mask,\
tcwv = tcwv, tco3 = tco3, aot_unc = aot_unc, \
tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, rgb = \
rgb, emus_dir=file_path+'/emus/', log_file = log_file, global_dem = global_dem, cams_dir = cams_dir)
atmo._doing_correction()
if jasmin:
shutil.rmtree(vrt_dir)
return aero, atmo
def do_correction(
sun_ang_name,
view_ang_names,
toa_refs,
qa_name,
cloud_mask,
metafile,
mcd43=DEFAULT_MCD43_DIR,
vrt_dir=DEFAULT_MCD43_VRT_DIR,
aoi=None,
global_dem="/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt",
cams_dir="/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/",
):
if os.path.realpath(mcd43) in os.path.realpath(DEFAULT_MCD43_DIR)\
and not os.path.exists(DEFAULT_MCD43_DIR):
os.mkdir(DEFAULT_MCD43_DIR)
if os.path.realpath(vrt_dir) in os.path.realpath(DEFAULT_MCD43_VRT_DIR)\
and not os.path.exists(DEFAULT_MCD43_VRT_DIR):
os.mkdir(DEFAULT_MCD43_VRT_DIR)
base = os.path.dirname(toa_refs[0])
with open(metafile) as f:
for line in f:
if "REFLECTANCE_MULT_BAND" in line:
scale = float(line.split()[-1])
elif "REFLECTANCE_ADD_BAND" in line:
off = float(line.split()[-1])
elif "DATE_ACQUIRED" in line:
date = line.split()[-1]
elif "SCENE_CENTER_TIME" in line:
time = line.split()[-1]
log_file = os.path.join(os.path.dirname(metafile), "zac_l8.log")
logger = create_logger(log_file)
logger.info("Starting atmospheric corretion for %s" % base)
datetime_str = date + time
obs_time = datetime.strptime(datetime_str.split(".")[0], '%Y-%m-%d"%H:%M:%S')
if not np.all(cloud_mask):
handlers = logger.handlers[:]
for handler in handlers:
handler.close()
logger.removeHandler(handler)
vrt_dir = get_mcd43(
toa_refs[0],
obs_time,
mcd43_dir=mcd43,
vrt_dir=vrt_dir,
log_file=log_file,
)
else:
logger.info("No clean pixel in this scene and no MCD43 is downloaded.")
sensor_sat = "OLI", "L8"
band_index = [1, 2, 3, 4, 5, 6]
band_wv = [469, 555, 645, 859, 1640, 2130]
toa_bands = (np.array(toa_refs)[band_index,]).tolist()
view_angles = (np.array(view_ang_names)[band_index,]).tolist()
sun_angles = sun_ang_name
sza = gdal.Open(sun_angles).ReadAsArray()[1] * 0.01
scale = scale / np.cos(np.deg2rad(sza))
off = off / np.cos(np.deg2rad(sza))
aero = solve_aerosol(
sensor_sat,
toa_bands,
band_wv,
band_index,
view_angles,
sun_angles,
obs_time,
cloud_mask,
gamma=10.0,
ref_scale=scale,
ref_off=off,
global_dem=global_dem,
cams_dir=cams_dir,
spec_m_dir=os.path.join(os.path.dirname(file_path), "spectral_mapping", ""),
emus_dir=os.path.join(os.path.dirname(file_path), "emus", ""),
mcd43_dir=vrt_dir,
)
aero._solving()
toa_bands = toa_refs
view_angles = view_ang_names
base = os.path.join(base, "B".join(toa_bands[0].split("/")[-1].split("B")[:-1]))
aot = base + "aot.tif"
tcwv = base + "tcwv.tif"
tco3 = base + "tco3.tif"
aot_unc = base + "aot_unc.tif"
tcwv_unc = base + "tcwv_unc.tif"
tco3_unc = base + "tco3_unc.tif"
rgb = [toa_bands[3], toa_bands[2], toa_bands[1]]
band_index = [0, 1, 2, 3, 4, 5, 6]
atmo = atmospheric_correction(
sensor_sat,
toa_bands,
band_index,
view_angles,
sun_angles,
aot=aot,
cloud_mask=cloud_mask,
tcwv=tcwv,
tco3=tco3,
aot_unc=aot_unc,
tcwv_unc=tcwv_unc,
tco3_unc=tco3_unc,
global_dem=global_dem,
cams_dir=cams_dir,
rgb=rgb,
ref_scale=scale,
ref_off=off,
emus_dir=os.path.join(file_path, "emus", ""),
aoi=aoi,
)
atmo._doing_correction()
return aero, atmo