Esempio n. 1
0
def do_correction(sun_ang_name, view_ang_names, toa_refs, qa_name, cloud_mask, \
                  metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi = None):

    if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'):
        if not os.path.exists(home + '/MCD43/'):
            os.mkdir(home + '/MCD43/')

    if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT/'):
        if not os.path.exists(home + '/MCD43_VRT/'):
            os.mkdir(home + '/MCD43_VRT/')

    base = os.path.dirname(toa_refs[0])
    with open(metafile) as f:
        for line in f:
            if 'REFLECTANCE_MULT_BAND' in line:
                scale = float(line.split()[-1])
            elif 'REFLECTANCE_ADD_BAND' in line:
                off = float(line.split()[-1])
            elif 'DATE_ACQUIRED' in line:
                date = line.split()[-1]
            elif 'SCENE_CENTER_TIME' in line:
                time = line.split()[-1]

    datetime_str = date + time
    obs_time = datetime.strptime(
        datetime_str.split('.')[0], '%Y-%m-%d"%H:%M:%S')
    get_mcd43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir)
    sensor_sat = 'OLI', 'L8'
    band_index = [1, 2, 3, 4, 5, 6]
    band_wv = [469, 555, 645, 859, 1640, 2130]
    toa_bands = (np.array(toa_refs)[band_index, ]).tolist()
    view_angles = (np.array(view_ang_names)[band_index, ]).tolist()
    sun_angles = sun_ang_name
    sza = gdal.Open(sun_angles).ReadAsArray()[1] * 0.01
    scale = scale / np.cos(np.deg2rad(sza))
    off = off / np.cos(np.deg2rad(sza))
    aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,sun_angles,\
                         obs_time,cloud_mask, gamma=10., ref_scale = scale, ref_off = off, \
                         spec_m_dir=file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir)
    aero._solving()
    toa_bands = toa_refs
    view_angles = view_ang_names
    base = base + '/' + 'B'.join(toa_bands[0].split('/')[-1].split('B')[:-1])
    aot = base + 'aot.tif'
    tcwv = base + 'tcwv.tif'
    tco3 = base + 'tco3.tif'
    aot_unc = base + 'aot_unc.tif'
    tcwv_unc = base + 'tcwv_unc.tif'
    tco3_unc = base + 'tco3_unc.tif'
    rgb = [toa_bands[3], toa_bands[2], toa_bands[1]]
    band_index = [0, 1, 2, 3, 4, 5, 6]
    atmo = atmospheric_correction(sensor_sat,toa_bands, band_index,view_angles,sun_angles, \
                                  aot = aot, cloud_mask = cloud_mask,tcwv = tcwv, tco3 = tco3, \
                                  aot_unc = aot_unc, tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, \
                                  rgb = rgb, ref_scale = scale, ref_off = off, emus_dir=file_path+'/emus/', aoi=aoi)
    atmo._doing_correction()
    return aero, atmo
Esempio n. 2
0
def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \
                  cloud_mask, metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi=None):

    if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'):
        if not os.path.exists(home + '/MCD43/'):
            os.mkdir(home + '/MCD43/')

    if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT/'):
        if not os.path.exists(home + '/MCD43_VRT/'):
            os.mkdir(home + '/MCD43_VRT/')

    base = os.path.dirname(toa_refs[0])
    base = toa_refs[0].replace('B01.jp2', '')
    with open(metafile) as f:
        for i in f.readlines():
            if 'SENSING_TIME' in i:
                sensing_time = i.split('</')[0].split('>')[-1]
                obs_time = datetime.strptime(sensing_time,
                                             u'%Y-%m-%dT%H:%M:%S.%fZ')
            if 'TILE_ID' in i:
                sat = i.split('</')[0].split('>')[-1].split('_')[0]

    get_mcd43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir)
    sensor_sat = 'MSI', sat
    band_index = [1, 2, 3, 7, 11, 12]
    band_wv = [469, 555, 645, 859, 1640, 2130]
    toa_bands = (np.array(toa_refs)[band_index, ]).tolist()
    view_angles = (np.array(view_ang_names)[band_index, ]).tolist()
    sun_angles = sun_ang_name
    aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,\
                         sun_angles,obs_time,cloud_mask, gamma=10., spec_m_dir= \
                         file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir, aoi=aoi)
    aero._solving()
    toa_bands = toa_refs
    view_angles = view_ang_names
    aot = base + 'aot.tif'
    tcwv = base + 'tcwv.tif'
    tco3 = base + 'tco3.tif'
    aot_unc = base + 'aot_unc.tif'
    tcwv_unc = base + 'tcwv_unc.tif'
    tco3_unc = base + 'tco3_unc.tif'
    rgb = [toa_bands[3], toa_bands[2], toa_bands[1]]
    band_index = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
    atmo = atmospheric_correction(sensor_sat,toa_bands, band_index,view_angles,\
                                  sun_angles, aot = aot, cloud_mask = cloud_mask,\
                                  tcwv = tcwv, tco3 = tco3, aot_unc = aot_unc, \
                                  tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, rgb = \
                                  rgb, emus_dir=file_path+'/emus/')
    atmo._doing_correction()
    return aero, atmo
Esempio n. 3
0
def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \
                  cloud_mask, aot, tcwv, metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi=None, \
                  global_dem  = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt', \
                  cams_dir    = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/', jasmin = False):
    if jasmin:
        global_dem  = '/work/scratch/marcyin/DEM/global_dem.vrt'
        cams_dir    = '/work/scratch/marcyin/CAMS/'
        os.environ['jasmin_memory_limit'] = '6.4e+10'
        #vrt_dir     = '/work/scratch/marcyin/MCD43_VRT/'
    if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'):
        if not os.path.exists(home + '/MCD43/'):
            os.mkdir(home + '/MCD43/')

    if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT/'):
        if not os.path.exists(home + '/MCD43_VRT/'):
            os.mkdir(home + '/MCD43_VRT/')

    base = os.path.dirname(toa_refs[0])
    base = toa_refs[0].replace('B01.jp2', '')
    with open(metafile) as f:
        for i in f.readlines():
            if 'SENSING_TIME' in i:
                sensing_time = i.split('</')[0].split('>')[-1]
                obs_time = datetime.strptime(sensing_time, u'%Y-%m-%dT%H:%M:%S.%fZ')
            if 'TILE_ID' in i:
                sat  = i.split('</')[0].split('>')[-1].split('_')[0]
                tile = i.split('</')[0].split('>')[-1]
    log_file = os.path.dirname(metafile) + '/SIAC_S2.log'
    logger = create_logger(log_file)
    logger.info('Starting atmospheric corretion for %s'%tile)
    if not np.all(cloud_mask):
        handlers = logger.handlers[:]
        for handler in handlers:
            handler.close()
            logger.removeHandler(handler)
        #if not jasmin:
        vrt_dir = get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir, log_file = log_file, jasmin = jasmin)
        #logger = create_logger(log_file)
    else:
        logger.info('No clean pixel in this scene and no MCD43 is downloaded.')
    sensor_sat = 'MSI', sat
    band_index  = [1,2,3,7,11,12]
    band_wv    = [469, 555, 645, 859, 1640, 2130]
    toa_bands   = (np.array(toa_refs)[band_index,]).tolist()
    view_angles = (np.array(view_ang_names)[band_index,]).tolist()
    sun_angles  = sun_ang_name
    logger.info('First pass AOT and TCWV: %.02f, %.02f'%(aot.mean(), tcwv.mean()))
    #logger.info('Running SIAC for tile: %s on %s'%(tile, obs_time.strftime('%Y-%M-%d')))
    aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,\
                         sun_angles,obs_time,cloud_mask, gamma=10., spec_m_dir= \
                         file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', wv_prior=tcwv, wv_unc=0.1,\
                         mcd43_dir=vrt_dir, aoi=aoi, log_file = log_file, global_dem  = global_dem, cams_dir = cams_dir, \
                         prior_scale = [1., 1, 46.698, 1., 1., 1.])
    aero._solving()
    toa_bands  = toa_refs
    view_angles = view_ang_names
    aot = base + 'aot.tif'
    tcwv = base + 'tcwv.tif'
    tco3 = base + 'tco3.tif'
    aot_unc = base + 'aot_unc.tif'
    tcwv_unc = base + 'tcwv_unc.tif'
    tco3_unc = base + 'tco3_unc.tif'
    rgb = [toa_bands[3], toa_bands[2], toa_bands[1]]
    band_index = [0,1,2,3,4,5,6,7,8,9,10,11,12]
    atmo = atmospheric_correction(sensor_sat,toa_bands, band_index,view_angles,\
                                  sun_angles, aot = aot, cloud_mask = cloud_mask,\
                                  tcwv = tcwv, tco3 = tco3, aot_unc = aot_unc, \
                                  tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, rgb = \
                                  rgb, emus_dir=file_path+'/emus/', log_file = log_file, global_dem  = global_dem, cams_dir = cams_dir)
    atmo._doing_correction()
    if jasmin:
        shutil.rmtree(vrt_dir)
    return aero, atmo
Esempio n. 4
0
def do_correction(sun_ang_name, view_ang_names, toa_refs, qa_name, cloud_mask, \
                  metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi = None,\
                  global_dem  = None, cams_dir    = None, jasmin = False):
    if jasmin:
        if global_dem is None:
            global_dem = '/work/scratch-pw/marcyin/DEM/global_dem.vrt'
        if cams_dir is None:
            cams_dir = '/work/scratch-pw/marcyin/CAMS/'
        os.environ['jasmin_memory_limit'] = '6.4e+10'
    else:
        if global_dem is None:
            global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt'
        if cams_dir is None:
            cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/'

    if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'):
        if not os.path.exists(home + '/MCD43/'):
            os.mkdir(home + '/MCD43/')

    if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT/'):
        if not os.path.exists(home + '/MCD43_VRT/'):
            os.mkdir(home + '/MCD43_VRT/')

    base = os.path.dirname(toa_refs[0])
    with open(metafile) as f:
        for line in f:
            if 'REFLECTANCE_MULT_BAND' in line:
                scale = float(line.split()[-1])
            elif 'REFLECTANCE_ADD_BAND' in line:
                off = float(line.split()[-1])
            elif 'DATE_ACQUIRED' in line:
                date = line.split()[-1]
            elif 'SCENE_CENTER_TIME' in line:
                time = line.split()[-1]
    log_file = os.path.dirname(metafile) + '/SIAC_L8.log'
    logger = create_logger(log_file)
    logger.info('Starting atmospheric corretion for %s' % base)
    datetime_str = date + time
    obs_time = datetime.strptime(
        datetime_str.split('.')[0], '%Y-%m-%d"%H:%M:%S')
    if not np.all(cloud_mask):
        #         handlers = logger.handlers[:]
        #         for handler in handlers:
        #             handler.close()
        #             logger.removeHandler(handler)
        #if not jasmin:
        vrt_dir = get_mcd43(toa_refs[0],
                            obs_time,
                            mcd43_dir=mcd43,
                            vrt_dir=vrt_dir,
                            logger=logger,
                            jasmin=jasmin)
        #logger = create_logger(log_file)
    else:
        logger.info('No clean pixel in this scene and no MCD43 is downloaded.')
    #get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir)
    sensor_sat = 'OLI', 'L8'
    band_index = [1, 2, 3, 4, 5, 6]
    band_wv = [469, 555, 645, 859, 1640, 2130]
    toa_bands = (np.array(toa_refs)[band_index, ]).tolist()
    view_angles = (np.array(view_ang_names)[band_index, ]).tolist()
    sun_angles = sun_ang_name

    #     sza         = gdal.Open(sun_angles).ReadAsArray()[1] * 0.01
    pixel_res = abs(gdal.Open(str(sun_angles)).GetGeoTransform()[1])
    g = gdal.Warp('', str(sun_angles), format = 'MEM', xRes = pixel_res, yRes = pixel_res, warpOptions = ['NUM_THREADS=ALL_CPUS'],\
                      srcNodata = 0, dstNodata=0, cutlineDSName= aoi, cropToCutline=True, resampleAlg = 0)
    sza = g.ReadAsArray()[1] * 0.01

    scale = scale / np.cos(np.deg2rad(sza))
    off = off / np.cos(np.deg2rad(sza))
    aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,sun_angles,\
                         obs_time,cloud_mask, gamma=10., ref_scale = scale, ref_off = off, global_dem  = global_dem, cams_dir = cams_dir,\
                         spec_m_dir=file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir, aoi=aoi, log_file = log_file)

    aero._solving()
    toa_bands = toa_refs
    view_angles = view_ang_names
    base = base + '/' + 'B'.join(toa_bands[0].split('/')[-1].split('B')[:-1])
    aot = base + 'aot.tif'
    tcwv = base + 'tcwv.tif'
    tco3 = base + 'tco3.tif'
    aot_unc = base + 'aot_unc.tif'
    tcwv_unc = base + 'tcwv_unc.tif'
    tco3_unc = base + 'tco3_unc.tif'
    rgb = [toa_bands[3], toa_bands[2], toa_bands[1]]
    band_index = [0, 1, 2, 3, 4, 5, 6]
    atmo = atmospheric_correction(sensor_sat,toa_bands, band_index,view_angles,sun_angles, \
                                  aot = aot, cloud_mask = cloud_mask,tcwv = tcwv, tco3 = tco3, \
                                  aot_unc = aot_unc, tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, global_dem  = global_dem, cams_dir = cams_dir,\
                                  rgb = rgb, ref_scale = scale, ref_off = off, emus_dir=file_path+'/emus/', aoi=aoi, log_file = log_file)
    atmo._doing_correction()
    return aero, atmo