def product_xml_imagery_files(cls, xml):
""" Return a list of which imagery files are associated with a RS-2 product.xml file"""
imagery_files = [
f.strip() for f in re.findall(".*tif", gdal.Info(xml))
]
return (imagery_files)
def __init__(self, source_img,
target_img = None,
dstSRS = None,
srcNodata = np.nan,
dstNodata = np.nan,
outputType = None,
verbose = False,
xmin = None,
xmax = None,
ymin = None,
ymax = None,
xRes = None,
yRes = None,
xSize = None,
ySize = None,
resample = 1
):
self.source_img = source_img
self.target_img = target_img
self.verbose = verbose
self.dstSRS = dstSRS
self.srcNodata = srcNodata
self.dstNodata = dstNodata
self.outputType = gdal.GDT_Unknown if outputType is None else outputType
self.xmin = xmin
self.xmax = xmax
self.ymin = ymin
self.ymax = ymax
self.xRes = xRes
self.yRes = yRes
self.xSize = xSize
self.ySize = ySize
self.resample = resample
if self.srcNodata is None:
try:
self.srcNodata = ' '.join([i.split("=")[1] for i in gdal.Info(self.source_img).split('\n') if' NoData' in i])
except:
self.srcNodata = None
if (self.target_img is None) & (self.dstSRS is None):
raise IOError('Projection should be specified ether from a file or a projection code.')
elif self.target_img is not None:
try:
g = gdal.Open(self.target_img)
except:
g = target_img
geo_t = g.GetGeoTransform()
x_size, y_size = g.RasterXSize, g.RasterYSize
if self.xRes is None:
self.xRes = abs(geo_t[1])
if self.yRes is None:
self.yRes = abs(geo_t[5])
if self.xSize is not None:
x_size = 1. * self.xSize * self.xRes / abs(geo_t[1])
if self.ySize is not None:
y_size = 1. * self.ySize * self.yRes / abs(geo_t[5])
xmin, xmax = min(geo_t[0], geo_t[0] + x_size * geo_t[1]), \
max(geo_t[0], geo_t[0] + x_size * geo_t[1])
ymin, ymax = min(geo_t[3], geo_t[3] + y_size * geo_t[5]), \
max(geo_t[3], geo_t[3] + y_size * geo_t[5])
dstSRS = osr.SpatialReference( )
raster_wkt = g.GetProjection()
dstSRS.ImportFromWkt(raster_wkt)
self.g = gdal.Warp('', self.source_img, format = 'MEM', outputBounds = [xmin, ymin, xmax, ymax], dstNodata=self.dstNodata, warpOptions = ['NUM_THREADS=ALL_CPUS'],\
xRes = self.xRes, yRes = self.yRes, dstSRS = dstSRS, outputType = self.outputType, srcNodata = self.srcNodata, resampleAlg = self.resample)
else:
self.g = gdal.Warp('', self.source_img, format = 'MEM', outputBounds = [self.xmin, self.ymin, \
self.xmax, self.ymax], xRes = self.xRes, yRes = self.yRes, dstSRS = self.dstSRS, warpOptions = ['NUM_THREADS=ALL_CPUS'],\
copyMetadata=True, outputType = self.outputType, dstNodata=self.dstNodata, srcNodata = self.srcNodata, resampleAlg = self.resample)
if self.g.RasterCount <= 3:
self.data = self.g.ReadAsArray()
#return self.data
elif self.verbose:
print('There are %d bands in this file, use g.GetRasterBand(<band>) to avoid reading the whole file.'%self.g.RasterCount)
import os
import gdal
import glob2
input_folder= r'C:/Users/arpan/Documents/MAIAC/2016'
output_folder = r'C:/Users/arpan/Documents/MAIAC/2016/Projected'
latlon_info = gdal.Info(r'C:/Users/arpan/Documents/MAIAC/MAIACLatlon.h03v04.hdf')
#print(latlon_info)
gdal.Translate(r"C:/Users/arpan/Documents/MAIAC/h03v04_lat.vrt",
r'HDF4_EOS:EOS_GRID:"C:/Users/arpan/Documents/MAIAC/MAIACLatlon.h03v04.hdf":latlon:lat', format = 'VRT' )
gdal.Translate(r"C:/Users/arpan/Documents/MAIAC/h03v04_lon.vrt",
r'HDF4_EOS:EOS_GRID:"C:/Users/arpan/Documents/MAIAC/MAIACLatlon.h03v04.hdf":latlon:lon', format = 'VRT' )
filenames = glob2.glob(r'C:/Users/arpan/Documents/MAIAC/2016/*.hdf')
#print(filenames)
geoloc_xml = """<VRTDataset rasterXSize="1200" rasterYSize="1200">
<Metadata domain="GEOLOCATION">
<mdi key="X_DATASET">h03v04_lon.vrt</mdi>
<mdi key="X_BAND">1</mdi>
<mdi key="Y_DATASET">h03v04_lat.vrt</mdi>
<mdi key="Y_BAND">1</mdi>
<mdi key="PIXEL_OFFSET">0</mdi>
<mdi key="LINE_OFFSET">0</mdi>
<mdi key="PIXEL_STEP">1</mdi>
<mdi key="LINE_STEP">1</mdi>
</Metadata>
<VRTRasterBand dataType="Float32" band="1">
for file in f:
if '.tif' in file.lower():
files.append(os.path.join(r, file))
if len(files) == 0:
print("Please specify a folder that contains TIF files.")
exit()
# Process center for each file
for file in files:
# Vars
lat = "Invalid File"
long = ""
# Analyze image file
image_transform = size = []
if file != "":
image_info = gdal.Info(file, format="json")
image_transform = image_info['geoTransform']
size = image_info["size"]
# Calculate centers
width = image_transform[1] * size[0]
height = image_transform[5] * size[1]
long = image_transform[0] + 0.5 * width
lat = image_transform[3] + 0.5 * height
# Print file info
name = ntpath.basename(file)
print("{0:<40}{1:<20}{2:<20}".format(name, lat, long))
def _get_min_max(self, raster_path):
info = gdal.Info(raster_path, format="json", computeMinMax=True)
return (info["bands"][0]["computedMax"],
info["bands"][0]["computedMax"])
def convert_wrapper(ix_im):
imd = imdates2[ix_im]
if np.mod(ix_im, 10)==0:
print(' Finished {0:4}/{1:4}th sltd...'.format(ix_im, len(imdates2)), flush=True)
ztdfile = os.path.join(gacosdir, imd+'.ztd')
sltdtiffile = os.path.join(gacosdir, imd+'.sltd.geo.tif')
if os.path.exists(sltdtiffile):
infile = os.path.basename(sltdtiffile)
try: ### Cut and resapmle. Already in rad.
sltd_geo = gdal.Warp("", sltdtiffile, format='MEM', outputBounds=outputBounds, width=width_geo, height=length_geo, resampleAlg=resampleAlg, srcNodata=0).ReadAsArray()
except: ## if broken
print (' {} cannot open. Skip'.format(infile), flush=True)
return imd
elif os.path.exists(ztdfile):
infile = os.path.basename(ztdfile)
hdrfile = os.path.join(sltddir, imd+'.hdr')
bilfile = os.path.join(sltddir, imd+'.bil')
if os.path.exists(hdrfile): os.remove(hdrfile)
if os.path.exists(bilfile): os.remove(bilfile)
make_hdr(ztdfile+'.rsc', hdrfile)
os.symlink(os.path.relpath(ztdfile, sltddir), bilfile)
## Check read error with unkown cause
if gdal.Info(bilfile) is None:
### Create new ztd by adding 0.0001m
print('{} cannot open, but trying minor update. You can ignore this error unless this script stops.'.format(ztdfile))
shutil.copy2(ztdfile, ztdfile+'.org') ## Backup
_ztd = np.fromfile(ztdfile, dtype=np.float32)
_ztd[_ztd!=0] = _ztd[_ztd!=0]+0.001
_ztd.tofile(ztdfile)
### Cut and resapmle ztd to geo
ztd_geo = gdal.Warp("", bilfile, format='MEM', outputBounds=outputBounds,\
width=width_geo, height=length_geo, \
resampleAlg=resampleAlg, srcNodata=0).ReadAsArray()
os.remove(hdrfile)
os.remove(bilfile)
### Meter to rad, slantrange
sltd_geo = ztd_geo/LOSu*m2r_coef ## LOSu=cos(inc)
else:
print(' There is no ztd|sltd.geo.tif for {}!'.format(imd), flush=True)
return imd ## Next imd
### Skip if no data in the area
if np.all((sltd_geo==0)|np.isnan(sltd_geo)):
print(' There is no valid data in {}!'.format(infile), flush=True)
return imd ## Next imd
### Fill hole is specified
if fillholeflag:
sltd_geo = fillhole(sltd_geo)
### Output as sltd.geo
sltd_geofile = os.path.join(sltddir, imd+'.sltd.geo')
sltd_geo.tofile(sltd_geofile)
return
def getInfo(self):
print(gdal.Info(self.dataset))
for j in fl:
# Reproject and resample layers.
src_filename = j
src = gdal.Open(src_filename, gdalconst.GA_ReadOnly)
src_proj = src.GetProjection()
src_geotrans = src.GetGeoTransform()
# Create output file name
dst_filename = odfor + os.sep + os.path.basename(src_filename).replace('.tif',projsuffix)
# Run if output doesn't exist
if not os.path.exists(dst_filename):
# If xmax of original raster is > 180, clip it to avoid wraparound
# Same with xmin if it is < -180.
# Need to get xmax from gdalinfo since I get wraparound when I try to calculate from geotransform
gi = gdal.Info(src_filename).split("\n")
gimax = [i for i in gi if "Upper Right" in i]
#gi = float(gi[0].split("(")[1].split(" ")[1].split(",")[0])
gimax = float(gimax[0].split("(")[1].replace(" ","").split(",")[0])
# Get xmin
gimin = [i for i in gi if "Upper Left" in i]
#gi = float(gi[0].split("(")[1].split(" ")[1].split(",")[0])
gimin = float(gimin[0].split("(")[1].replace(" ","").split(",")[0])
# Clip rasters if they're beyond domain edges
if gimax > 180:
# Specify new xmax
newcoord = "179.99"
newof = os.path.dirname(src_filename) + os.sep + os.path.basename(src_filename).split(".")[0] + "_v2.tif"
# Subset raster. Note subset window given in ulx, uly, lrx, lry order
subprocess.call(["gdal_translate", "-co", "COMPRESS=LZW", "-projwin", str(src_geotrans[0]), str(src_geotrans[3]), newcoord, str(src_geotrans[3] + src_geotrans[5]*src.RasterYSize), src_filename, newof])
# Reproject and resample layers.
]
hashDict = {}
for ii in inFnames:
thisBasename = os.path.basename(ii)
thisDS = gdal.Open(ii, gdalconst.GA_ReadOnly)
ret = gdal.Info(thisDS,
format='json',
deserialize=True,
computeMinMax=False,
reportHistograms=False,
reportProj4=False,
stats=False,
approxStats=False,
computeChecksum=True,
showGCPs=False,
showMetadata=False,
showRAT=False,
showColorTable=False,
listMDD=False,
showFileList=False)
# concatCkcksum=''
# for iband in ret['bands']:
# if 'checksum' in iband:
# concatCkcksum = '{}{}'.format(concatCkcksum, iband['checksum'])
concatCkcksum = ret['bands'][0]['checksum']
hashDict[
base=sys.argv[1], project_name=project_name, soil_name=soil_name))
else:
copytree(
soil_fn,
'{base}/{project_name}/Watershed/Rasters/Soil/{soil_name}'.format(
base=sys.argv[1], project_name=project_name, soil_name=soil_name))
# src_ds = gdal.Open(soil_fn)
# ds = gdal.Translate('{base}/{project_name}/Watershed/Rasters/Soil/{soil_name}'.format(
# base=sys.argv[1], project_name=project_name, soil_name=soil_name), src_ds, format='GTiff')
log.info("getting dem projection information", keep_log)
dataset = gdal.Open(
'{base}/{project_name}/Watershed/Rasters/DEM/{dem_name}'.format(
base=sys.argv[1], project_name=project_name, dem_name=dem_file_name_))
formated_projcs = gdal.Info(dataset).split("Data axis")[0].split(
"System is:\n")[-1]
prjcrs = ""
for line in formated_projcs.split("\n"):
while line.startswith(" "):
line = line[1:]
prjcrs += line.strip("\n")
srs = osr.SpatialReference(wkt=prjcrs)
proj4 = srs.ExportToProj4()
geogcs = srs.GetAttrValue('geogcs')
log.info("geogcs: {0}".format(geogcs), keep_log)
log.info("proj4: {0}".format(proj4), keep_log)
if prjcrs.split('"')[1] in epsg_lookup_dictionary:
def ai_makecog(repo: Repo, task: Task, roi_id, recipe_path, json_only, quiet,
no_color, less, zone, kind, source, cos_key, friendly_name,
print_res, warp_resampleAlg, overview_resampleAlg):
""" Make COG TIFF from visualized results
\b
* to make GeoTIFF from custom ENVI file use --source option with filename of ENVI file (without extension)
* to override recipe kind, use --kind option,
example: making image from 'ai preview --export path/to/envi' file use
makecog zone --kind Image --source path/to/envi
* to avoid overriding recipe main results use --cos-key and --friendly-name option
if --friendly-name starts with '+' value will be used as suffix for friendly_name in GeoJSON
if --cos-key starts with '+' value will be used as suffix for COS.ResultKey in GeoJSON
"""
driver = 'MAKECOG'
if source:
try:
# Only valid ENVI or GeoTiff files are alloed
str = gdal.Info(source, format='json')
driver = str['driverShortName']
if driver not in ['ENVI', 'GTiff']:
raise OCLIException(
f"Unsupported source file type: {str['driverShortName']} ({str['driverLongName']})"
)
except Exception as e:
raise OCLIException(f"Option --source: {e}")
_recipe = recipe_path if recipe_path else resolve_recipe(
repo, task, roi_id)
recipe = Recipe(_recipe)
kind = kind if kind != 'auto' else recipe.get('type')
if kind in ['Rvi', 'Image']:
recipe['type'] = 'Image'
# log.error(recipe['COS']['ResultKey'])
if cos_key:
if cos_key.startswith('+'):
recipe['COS']['ResultKey'] += cos_key[1:]
else:
recipe['COS']['ResultKey'] = cos_key
if friendly_name:
if friendly_name.startswith('+'):
recipe['friendly_name'] += friendly_name[1:]
else:
recipe['friendly_name'] = friendly_name
# log.error(recipe['COS']['ResultKey'])
# log.error(recipe['friendly_name'])
filenames = Filenames(zone, recipe)
if source:
input_file = source
else:
input_file = filenames.pred8c_img
out_file = filenames.out_tiff
cog_file = filenames.out_cog_tiff
check_file = cog_file if json_only else input_file
if not Path(check_file).is_file():
raise OCLIException(f'file not found: {check_file}')
os.makedirs(Path(cog_file).parent, exist_ok=True)
w = GDALWrap3(recipe, input_file, out_file, cog_file)
try:
if not json_only:
if driver == 'GTiff':
try:
shutil.copy(input_file, cog_file)
output.success(f"file {input_file} copied to {cog_file}")
except Exception as e:
raise OCLIException(
f'Could not copy "{input_file}" to "{cog_file}" : {e}'
)
else:
if not quiet:
with pfac(log, total=100,
desc='Assembling') as (_, callback):
def cb(pct, msg, user_data):
_ud = user_data[0]
# self.log.debug(f"translaing: {round(pct * 100, 2)}% -{msg}- {user_data} {pct}")
# reset counter, this t\callback called from multiple prcesses
if pct < 0.01:
user_data[0] = 0
if user_data[0] == 0 or pct - _ud > 0.10:
log.debug(
f"Local translating : {round(pct * 100, 2)}%"
)
user_data[0] = pct
callback(100, pct, 'translating')
w.make_cog(cb, warp_resampleAlg, overview_resampleAlg)
else:
w.make_cog(None, warp_resampleAlg, overview_resampleAlg)
_json = w.make_geo_json()
_json = _json if no_color else colorful_json(_json)
if print_res:
click.echo("\n\n\n")
if less:
click.echo_via_pager(_json)
else:
click.echo(_json)
except (RuntimeError, OCLIException) as e:
raise click.UsageError(
output.error_style(f"COG-tiff generation failed,reason: {e}"))
pass
import gdal
import osr
#LSDTopoTools specific imports
#Loading the LSDTT setup configuration
setup_file = open('chi_automation.config','r')
LSDMT_PT = setup_file.readline().rstrip()
LSDMT_MF = setup_file.readline().rstrip()
Iguanodon = setup_file.readline().rstrip()
setup_file.close()
sys.path.append(LSDMT_PT)
sys.path.append(LSDMT_MF)
sys.path.append(Iguanodon)
import LSDPlottingTools as LSDPT
directory = "/exports/csce/datastore/geos/groups/LSDTopoData/Himalayan_Ksn_Concavity/cosmo_data/basin_shapefiles/"
rasterDS = gdal.Open(directory+'output5.tif')
print gdal.Info(rasterDS)
srs = osr.SpatialReference()
###problematic - get from input file directly
srs.ImportFromEPSG(4326)
#outputDS = gdal.Open(directory+'output6.tif')
gdal.Warp(directory+'outputDS.tif',rasterDS,dstSRS=srs)
#print gdal.Info(outputDS)
def modis_dl_reproject(product, collection, date, download_root, needy=True):
# Select the correct nasa server based on the product requested
if product == 'MOD09GA':
nasa_server = 'MOLT'
elif product == 'MYD09GA':
nasa_server = 'MOLA'
# The url of the file we wish to retrieve
#url = "https://e4ftl01.cr.usgs.gov/MOLT/MOD09GA.006/2014.06.25/"
#myd = 'https://e4ftl01.cr.usgs.gov/MOLA/MYD09GA.006/'
url = "https://e4ftl01.cr.usgs.gov/{}/{}.{}/{}/".format(nasa_server,product,collection,date)
# ADD: Test to make sure they entered a valid product / collection / date
# Initiate the username / password manager, and the cookie jar for this
# session.
cookie_jar = initiate_urllib()
## ------------------------------------------------------------------------
## Find list of files (https) to download
## ------------------------------------------------------------------------
# Read the html of the http server for the desired modis product
try:
html_body = read_html(url)
except urllib.error.HTTPError:
print(("Could not reach url: {}".format(url)))
return
# instantiate the parser and fed it some HTML text
parser = MyHTMLParser()
parser.feed(html_body)
# Get a list of the available data on the http server
data_list = parser.data_list
# Calculate the granules needed to cover the arctic ocean
granule_list = calc_arctic_granules()
# Compare the available files with the ones required. Goes through the
# whole list of needed granules, and finds the full image name that matches
# each granule ID. Once the right image is found, stopiteration.
download_list = []
for granule in granule_list:
download_list.append(next((file_ for file_ in data_list if granule in file_),None))
## ------------------------------------------------------------------------
### Download requested granules from NASA server
## ------------------------------------------------------------------------
question = "There are [{}] files to download. Continue? [y/n] ".format(len(download_list))
answer = query_yes_no(question,needy)
if answer is True:
# Create the download path based on the images to be downloaded
download_path = os.path.join(download_root, 'MODIS/{}/{}/originals'.format(product,date))
if not os.path.isdir(download_path):
os.makedirs(download_path)
# Submit the download list to be downloaded
batch_job(url,download_list,download_path)
else:
print("Goodbye.")
quit()
# print "Normally I'd quit here."
# Check that download succeeded
pass_ = False
while pass_ is False:
pass_, failed_dl = verify_downloads(download_list,download_path)
if pass_:
infostring = gdal.Info(os.path.join(download_path,download_list[0]))
question = "File verification successful. Continue to mosaic and reproject? [y/n] "
if query_yes_no(question,needy) is False: quit()
else:
for dl in failed_dl:
print(("Failed download: " + dl))
question = "Would you like to redownload [{}] failed images? [y/n]".format(len(failed_dl))
answer = query_yes_no(question,needy)
if answer is True:
for dl in failed_dl:
os.remove(os.path.join(download_path,dl))
batch_job(url,failed_dl,download_path)
else:
print("Good day.")
quit()
## ------------------------------------------------------------------------
## Process downloaded files
## ------------------------------------------------------------------------
# Reproject to Arctic Polar Stereographic, mosaic all tiles together,
# and convert to a geotiff. This is all done using the gdal_warp utility
# Find the names of all of the subdatasets contained within the downloaded
# hdf files. In the future, maybe add a way for the user to select the
# datasets they're actually interested in.
sample_im = download_list[0] # name of one of the downloaded images
# Read the metadata in the sample image
# list of subdatasets, read from the metadata
sds_list = parse_gdalinfo(infostring)
# We want to put the mosaic image in the parent directory of the downloads
dst_path = os.path.split(download_path)[0]
# output image name is "product_date_sds"
dst_image_base = sample_im.split('.')[0]+'.'+sample_im.split('.')[1]
# list of bands we are interested in reprojecting and mosaicing for now
# Experiementing with queues (maybe useful for multithreading later?)
reproj_queue = queue.Queue()
reproj_list = [ 'sur_refl_b01', 'sur_refl_b02', 'sur_refl_b03', 'sur_refl_b04',
'sur_refl_b05', 'sur_refl_b06', 'sur_refl_b07', 'state_1km' ]
# reproj_list = [ 'state_1km']
for band in reproj_list:
reproj_queue.put(band)
# Reproject and mosaic each band individually
# We 'probably' dont want to mosaic _all_ of the bands
while not reproj_queue.empty():
band = reproj_queue.get()
sds = (next(s for s in sds_list if band in s))
# Parse the band name out of the full sds name
band_name = sds.split('"')[-1]
# Replace ':' with '_' in the output name
dst_band_name = band_name.replace(':','_')
# Create the full output path
dst_image = os.path.join(dst_path,dst_image_base+dst_band_name+'.tif')
# Reproject, mosaic, and convert to geotiff
print(("Reprojecting and mosaicing " + band_name))
reproject_mosaic(download_list,band_name,dst_image)
print("Removing raw files in {}".format(download_path))
shutil.rmtree(download_path)
ref_pth = r'Y:\germany-drought\vrt\masks\2015_MASK_FOREST-BROADLEAF_BUFF-01.vrt'
#ref_pth = r'O:\Student_Data\CJaenicke\00_MA\data\climate\dwd_precipitation\SPI\SPI_12\SPI_012018_12_3035.tif'
netcdf_pth = r"O:\Student_Data\CJaenicke\00_MA\data\climate\SMI\SMI_Lall_Gesamtboden_monatlich_1951_2018_inv.nc"
#netcdf_pth = r"O:\Student_Data\CJaenicke\00_MA\data\climate\SMI\SMI_L02_Oberboden_monatlich_1951_2018_inv.nc"
out_pth = r'O:\Student_Data\CJaenicke\00_MA\data\climate\SMI\SMI_2018_Gesamtboden.tif'
print('opening input')
## open a reference raster
ref_ras = gdal.Open(ref_pth)
gt = ref_ras.GetGeoTransform()
pr = ref_ras.GetProjection()
ref_arr = ref_ras.ReadAsArray()
## get Info about netcdf File
print(gdal.Info(netcdf_pth))
## open netcdf file, load subds and read them as arrays
ds = gdal.Open(netcdf_pth)
subds_lst = ds.GetSubDatasets()
## SMI
ds_smi = gdal.Open(ds.GetSubDatasets()[0][0])
smi_arr = ds_smi.ReadAsArray()
## kerndel width
ds_kw = gdal.Open(ds.GetSubDatasets()[1][0])
kw_arr = ds_kw.ReadAsArray()
## latitude
ds_lat = gdal.Open(ds.GetSubDatasets()[2][0])
lat_arr = ds_lat.ReadAsArray()
## longtitude
ds_lon = gdal.Open(ds.GetSubDatasets()[3][0])
final_del = np.delete(row_del,0,1)
print(final_del)
cmap ='viridis'
plt.figure(figsize=(10,10))
plt.imshow(final_del,cmap=cmap)
plt.show()
complete = Image.fromarray(final_del)
complete.save("test_1.tif")
sr = dataset.GetProjection()
gt = dataset.GetGeoTransform()
del dataset
dataset = gdal.Open("test_1.tif", gdal.GA_Update)
dataset.SetProjection(sr)
dataset.SetGeoTransform(gt)
print(gdal.Info("test_1.tif"))
del dataset
#3
dataset= gdal.Open("IMG_0111_3.tif", gdal.GA_Update)
band = dataset.GetRasterBand(1)
array = band.ReadAsArray()
print(array)
#rotated = np.rot90(array, k=1)
#rotated = ndimage.rotate(array,359,reshape=False)
padded = np.pad(array, [(0,3000),(0,3000)], 'constant', constant_values=(0))
input_file_1 = "test_1.tif"
input_file_2 = "../test_data/IMG_0111_2.tif"
input_file_3 = "../test_data/IMG_0111_3.tif"
ds = gdal.Open(input_file_2, gdal.GA_ReadOnly)
print("driver: {}/{}".format(ds.GetDriver().ShortName,
ds.GetDriver().LongName))
print("Size is {} x {} x {}".format(ds.RasterXSize, ds.RasterYSize,
ds.RasterCount))
print("Projection is {}".format(ds.GetProjection()))
geotransform = ds.GetGeoTransform()
if geotransform:
print("Origin = ({}, {})".format(geotransform[0], geotransform[3]))
print("Pixel Size = ({}, {})".format(geotransform[1], geotransform[5]))
ulx, xres, xskew, uly, yskew, yres = ds.GetGeoTransform()
lrx = ulx + (ds.RasterXSize * xres)
lry = uly + (ds.RasterYSize * yres)
print(gdal.Info(input_file_1))
vrt_options = gdal.BuildVRTOptions(separate=True)
my_vrt = gdal.BuildVRT('my.vrt', [input_file_1, input_file_2, input_file_3],
options=vrt_options)
my_vrt = None
vrt = gdal.Open('my.vrt')
print(vrt)
gdal.Translate('stack.tif', vrt)
def main(argv=None):
#%% Check argv
if argv == None:
argv = sys.argv
start = time.time()
ver = 1.4
date = 20200703
author = "Y. Morishita"
print("\n{} ver{} {} {}".format(os.path.basename(argv[0]), ver, date,
author),
flush=True)
print("{} {}".format(os.path.basename(argv[0]), ' '.join(argv[1:])),
flush=True)
#%% Set default
in_dir = []
out_dir = []
gacosdir = 'GACOS'
resampleAlg = 'cubicspline' # None # 'cubic'
fillholeflag = False
#%% Read options
try:
try:
opts, args = getopt.getopt(argv[1:], "hi:o:g:z:",
["fillhole", "help"])
except getopt.error as msg:
raise Usage(msg)
for o, a in opts:
if o == '-h' or o == '--help':
print(__doc__)
return 0
elif o == '-i':
in_dir = a
elif o == '-o':
out_dir = a
elif o == '-z': ## for backward-compatible
gacosdir = a
elif o == '-g':
gacosdir = a
elif o == "--fillhole":
fillholeflag = True
if not in_dir:
raise Usage('No input directory given, -i is not optional!')
elif not os.path.isdir(in_dir):
raise Usage('No {} dir exists!'.format(in_dir))
elif not os.path.exists(os.path.join(in_dir, 'slc.mli.par')):
raise Usage('No slc.mli.par file exists in {}!'.format(in_dir))
if not out_dir:
raise Usage('No output directory given, -o is not optional!')
if not os.path.isdir(gacosdir):
raise Usage('No {} dir exists!'.format(gacosdir))
except Usage as err:
print("\nERROR:", file=sys.stderr, end='')
print(" " + str(err.msg), file=sys.stderr)
print("\nFor help, use -h or --help.\n", file=sys.stderr)
return 2
#%% Read data information
### Directory
in_dir = os.path.abspath(in_dir)
gacosdir = os.path.abspath(gacosdir)
out_dir = os.path.abspath(out_dir)
if not os.path.exists(out_dir): os.mkdir(out_dir)
sltddir = os.path.join(os.path.join(out_dir), 'sltd')
if not os.path.exists(sltddir): os.mkdir(sltddir)
### Get general info
mlipar = os.path.join(in_dir, 'slc.mli.par')
width_unw = int(io_lib.get_param_par(mlipar, 'range_samples'))
length_unw = int(io_lib.get_param_par(mlipar, 'azimuth_lines'))
speed_of_light = 299792458 #m/s
radar_frequency = float(io_lib.get_param_par(mlipar,
'radar_frequency')) #Hz
wavelength = speed_of_light / radar_frequency #meter
m2r_coef = 4 * np.pi / wavelength
if wavelength > 0.2: ## L-band
cycle = 1.5 # 2pi/cycle for png
else: ## C-band
cycle = 3 # 2pi*3/cycle for png
### Get geo info. Grid registration
dempar = os.path.join(in_dir, 'EQA.dem_par')
width_geo = int(io_lib.get_param_par(dempar, 'width'))
length_geo = int(io_lib.get_param_par(dempar, 'nlines'))
dlat_geo = float(io_lib.get_param_par(dempar, 'post_lat')) #minus
dlon_geo = float(io_lib.get_param_par(dempar, 'post_lon'))
latn_geo = float(io_lib.get_param_par(dempar, 'corner_lat'))
lonw_geo = float(io_lib.get_param_par(dempar, 'corner_lon'))
lats_geo = latn_geo + dlat_geo * (length_geo - 1)
lone_geo = lonw_geo + dlon_geo * (width_geo - 1)
### Check coordinate
if width_unw != width_geo or length_unw != length_geo:
print('\n{} seems to contain files in radar coordinate!!\n'.format(
in_dir),
file=sys.stderr)
print('Not supported.\n'.format(in_dir), file=sys.stderr)
return 1
### Calc incidence angle from U.geo
ufile = os.path.join(in_dir, 'U.geo')
LOSu = io_lib.read_img(ufile, length_geo, width_geo)
LOSu[LOSu == 0] = np.nan
### Get ifgdates and imdates
ifgdates = tools_lib.get_ifgdates(in_dir)
imdates = tools_lib.ifgdates2imdates(ifgdates)
n_ifg = len(ifgdates)
n_im = len(imdates)
#%% Process ztd files
print('\nConvert ztd/sltd.geo.tif files to sltd.geo files...', flush=True)
### First check if sltd already exist
imdates2 = []
for imd in imdates:
sltd_geofile = os.path.join(sltddir, imd + '.sltd.geo')
if not os.path.exists(sltd_geofile):
imdates2.append(imd)
n_im2 = len(imdates2)
if n_im - n_im2 > 0:
print(" {0:3}/{1:3} sltd already exist. Skip".format(
n_im - n_im2, n_im),
flush=True)
no_gacos_imfile = os.path.join(out_dir, 'no_gacos_im.txt')
if os.path.exists(no_gacos_imfile): os.remove(no_gacos_imfile)
for ix_im, imd in enumerate(imdates2):
if np.mod(ix_im, 10) == 0:
print(' Finished {0:4}/{1:4}th sltd...'.format(ix_im, n_im2),
flush=True)
ztdfile = os.path.join(gacosdir, imd + '.ztd')
sltdtiffile = os.path.join(gacosdir, imd + '.sltd.geo.tif')
if os.path.exists(sltdtiffile):
infile = os.path.basename(sltdtiffile)
try: ### Cut and resapmle. Already in rad.
sltd_geo = gdal.Warp("",
sltdtiffile,
format='MEM',
outputBounds=(lonw_geo, lats_geo,
lone_geo, latn_geo),
width=width_geo,
height=length_geo,
resampleAlg=resampleAlg,
srcNodata=0).ReadAsArray()
except: ## if broken
print(' {} cannot open. Skip'.format(infile), flush=True)
with open(no_gacos_imfile, mode='a') as fnogacos:
print('{}'.format(imd), file=fnogacos)
continue
elif os.path.exists(ztdfile):
infile = os.path.basename(ztdfile)
hdrfile = os.path.join(sltddir, imd + '.hdr')
bilfile = os.path.join(sltddir, imd + '.bil')
if os.path.exists(hdrfile): os.remove(hdrfile)
if os.path.exists(bilfile): os.remove(bilfile)
make_hdr(ztdfile + '.rsc', hdrfile)
os.symlink(os.path.relpath(ztdfile, sltddir), bilfile)
## Check read error with unkown cause
if gdal.Info(bilfile) is None:
### Create new ztd by adding 0.0001m
print(
'{} cannot open, but trying minor update. You can ignore this error unless this script stops.'
.format(ztdfile))
shutil.copy2(ztdfile, ztdfile + '.org') ## Backup
_ztd = np.fromfile(ztdfile, dtype=np.float32)
_ztd[_ztd != 0] = _ztd[_ztd != 0] + 0.001
_ztd.tofile(ztdfile)
### Cut and resapmle ztd to geo
ztd_geo = gdal.Warp("", bilfile, format='MEM', \
outputBounds=(lonw_geo, lats_geo, lone_geo, latn_geo), \
width=width_geo, height=length_geo, \
resampleAlg=resampleAlg, srcNodata=0).ReadAsArray()
os.remove(hdrfile)
os.remove(bilfile)
### Meter to rad, slantrange
sltd_geo = ztd_geo / LOSu * m2r_coef ## LOSu=cos(inc)
else:
print(' There is no ztd|sltd.geo.tif for {}!'.format(imd),
flush=True)
with open(no_gacos_imfile, mode='a') as fnogacos:
print('{}'.format(imd), file=fnogacos)
continue ## Next imd
### Skip if no data in the area
if np.all((sltd_geo == 0) | np.isnan(sltd_geo)):
print(' There is no valid data in {}!'.format(infile), flush=True)
with open(no_gacos_imfile, mode='a') as fnogacos:
print('{}'.format(imd), file=fnogacos)
continue ## Next imd
### Fill hole is specified
if fillholeflag:
sltd_geo = fillhole(sltd_geo)
### Output as sltd.geo
sltd_geofile = os.path.join(sltddir, imd + '.sltd.geo')
sltd_geo.tofile(sltd_geofile)
#%% Correct unw files
print('\nCorrect unw data...', flush=True)
### Information files
gacinfofile = os.path.join(out_dir, 'GACOS_info.txt')
if not os.path.exists(gacinfofile):
### Add header
with open(gacinfofile, "w") as f:
print(' Phase STD (rad) Before After ReductionRate', file=f)
no_gacos_ifgfile = os.path.join(out_dir, 'no_gacos_ifg.txt')
if os.path.exists(no_gacos_ifgfile): os.remove(no_gacos_ifgfile)
### First check if already corrected unw exist
ifgdates2 = []
for i, ifgd in enumerate(ifgdates):
out_dir1 = os.path.join(out_dir, ifgd)
unw_corfile = os.path.join(out_dir1, ifgd + '.unw')
if not os.path.exists(unw_corfile):
ifgdates2.append(ifgd)
n_ifg2 = len(ifgdates2)
if n_ifg - n_ifg2 > 0:
print(" {0:3}/{1:3} corrected unw already exist. Skip".format(
n_ifg - n_ifg2, n_ifg),
flush=True)
### Correct
for i, ifgd in enumerate(ifgdates2):
if np.mod(i, 10) == 0:
print(' Finished {0:4}/{1:4}th unw...'.format(i, n_ifg2),
flush=True)
md = ifgd[:8]
sd = ifgd[-8:]
msltdfile = os.path.join(sltddir, md + '.sltd.geo')
ssltdfile = os.path.join(sltddir, sd + '.sltd.geo')
in_dir1 = os.path.join(in_dir, ifgd)
out_dir1 = os.path.join(out_dir, ifgd)
### Check if sltd available for both primary and secondary. If not continue
## Not use in tsa because loop cannot be closed
if not (os.path.exists(msltdfile) and os.path.exists(ssltdfile)):
print(' ztd file not available for {}'.format(ifgd), flush=True)
with open(no_gacos_ifgfile, mode='a') as fnogacos:
print('{}'.format(ifgd), file=fnogacos)
continue
### Prepare directory and file
if not os.path.exists(out_dir1): os.mkdir(out_dir1)
unwfile = os.path.join(in_dir1, ifgd + '.unw')
unw_corfile = os.path.join(out_dir1, ifgd + '.unw')
### Calculate dsltd
msltd = io_lib.read_img(msltdfile, length_unw, width_unw)
ssltd = io_lib.read_img(ssltdfile, length_unw, width_unw)
msltd[msltd == 0] = np.nan
ssltd[ssltd == 0] = np.nan
dsltd = ssltd - msltd
### Correct unw
unw = io_lib.read_img(unwfile, length_unw, width_unw)
unw[unw == 0] = np.nan
unw_cor = unw - dsltd
unw_cor.tofile(unw_corfile)
### Output std
std_unw = np.nanstd(unw)
std_unwcor = np.nanstd(unw_cor)
rate = (std_unw - std_unwcor) / std_unw * 100
with open(gacinfofile, "a") as f:
print('{0} {1:4.1f} {2:4.1f} {3:5.1f}%'.format(
ifgd, std_unw, std_unwcor, rate),
file=f)
### Link cc
if not os.path.exists(os.path.join(out_dir1, ifgd + '.cc')):
os.symlink(
os.path.relpath(os.path.join(in_dir1, ifgd + '.cc'), out_dir1),
os.path.join(out_dir1, ifgd + '.cc'))
### Output png for comparison
data3 = [
np.angle(np.exp(1j * (data / cycle)) * cycle)
for data in [unw, unw_cor, dsltd]
]
title3 = [
'unw_org (STD: {:.1f} rad)'.format(std_unw),
'unw_cor (STD: {:.1f} rad)'.format(std_unwcor),
'dsltd ({:.1f}% reduced)'.format(rate)
]
pngfile = os.path.join(out_dir1, ifgd + '.gacos.png')
plot_lib.make_3im_png(data3,
pngfile,
'insar',
title3,
vmin=-np.pi,
vmax=np.pi,
cbar=False)
## Output png for corrected unw
pngfile = os.path.join(out_dir1, ifgd + '.unw.png')
title = '{} ({}pi/cycle)'.format(ifgd, cycle * 2)
plot_lib.make_im_png(np.angle(np.exp(1j * unw_cor / cycle) * cycle),
pngfile,
'insar',
title,
-np.pi,
np.pi,
cbar=False)
print("", flush=True)
#%% Create correlation png
pngfile = os.path.join(out_dir, 'GACOS_info.png')
plot_lib.plot_gacos_info(gacinfofile, pngfile)
#%% Copy other files
files = glob.glob(os.path.join(in_dir, '*'))
for file in files:
if not os.path.isdir(file): #not copy directory, only file
print('Copy {}'.format(os.path.basename(file)), flush=True)
shutil.copy(file, out_dir)
#%% Finish
elapsed_time = time.time() - start
hour = int(elapsed_time / 3600)
minite = int(np.mod((elapsed_time / 60), 60))
sec = int(np.mod(elapsed_time, 60))
print("\nElapsed time: {0:02}h {1:02}m {2:02}s".format(hour, minite, sec))
print('\n{} Successfully finished!!\n'.format(os.path.basename(argv[0])))
print('Output directory: {}\n'.format(os.path.relpath(out_dir)))
if os.path.exists(no_gacos_ifgfile):
print('Caution: Some ifgs below are excluded due to GACOS unavailable')
with open(no_gacos_ifgfile) as f:
for line in f:
print(line, end='')
print('')
if os.path.exists(no_gacos_imfile):
print('GACOS data for the following dates are missing:')
with open(no_gacos_imfile) as f:
for line in f:
print(line, end='')
print('')
def read_image_header(fpath):
print('\n\n\n')
print(gdal.Info(fpath))
def MatchProjResNDV(source_file, target_fhs, output_dir,
resample='near', dtype='float32', scale=None, ndv_to_zero=False):
"""
Matches the projection, resolution and no-data-value of a list of target-files
with a source-file and saves the new maps in output_dir.
Parameters
----------
source_file : str
The file to match the projection, resolution and ndv with.
target_fhs : list
The files to be reprojected.
output_dir : str
Folder to store the output.
resample : str, optional
Resampling method to use, default is 'near' (nearest neighbour).
dtype : str, optional
Datatype of output, default is 'float32'.
scale : int, optional
Multiple all maps with this value, default is None.
Returns
-------
output_files: :obj:`numpy.ndarray`
Filehandles of the created files.
"""
print('WaPOR GIS: Matching projection...')
output_files = np.array([])
dst_info = gdal.Info(gdal.Open(source_file), format='json')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
for target_file in target_fhs:
folder, fn = os.path.split(target_file)
output_file = os.path.join(output_dir, fn)
src_info = gdal.Info(gdal.Open(target_file), format='json')
gdal.Warp(output_file, target_file, format='GTiff',
srcSRS=src_info['coordinateSystem']['wkt'],
dstSRS=dst_info['coordinateSystem']['wkt'],
srcNodata=src_info['bands'][0]['noDataValue'],
dstNodata=dst_info['bands'][0]['noDataValue'],
width=dst_info['size'][0],
height=dst_info['size'][1],
outputBounds=(dst_info['cornerCoordinates']['lowerLeft'][0],
dst_info['cornerCoordinates']['lowerLeft'][1],
dst_info['cornerCoordinates']['upperRight'][0],
dst_info['cornerCoordinates']['upperRight'][1]),
outputBoundsSRS=dst_info['coordinateSystem']['wkt'],
resampleAlg=resample)
output_files = np.append(output_files, output_file)
if not np.any([scale == 1.0, scale is None, scale == 1]):
driver, NDV, xsize, ysize, GeoT, Projection = GetGeoInfo(
output_file)
DATA = OpenAsArray(output_file, nan_values=True) * scale
CreateGeoTiff(
output_file,
DATA,
driver,
NDV,
xsize,
ysize,
GeoT,
Projection)
if ndv_to_zero:
driver, NDV, xsize, ysize, GeoT, Projection = GetGeoInfo(
output_file)
DATA = OpenAsArray(output_file, nan_values=False)
DATA[DATA == NDV] = 0.0
CreateGeoTiff(
output_file,
DATA,
driver,
NDV,
xsize,
ysize,
GeoT,
Projection)
return output_files
def pixel_size(self):
info = gdal.Info(self._path, format="json")
return abs(info["geoTransform"][1])
def _get_native_pixel_size(self, raster_path):
info = gdal.Info(raster_path, format="json")
return abs(info["geoTransform"][1])
def data_type(self):
info = gdal.Info(self._path, format="json")
return info["bands"][0]["type"]
def extract_sensing_month(filepath):
metadata = gdal.Info(filepath)
st_index = metadata.find("SENSING_TIME")
if st_index != -1:
month = metadata[st_index + 18:st_index + 20]
return int(month)
@author: bodo
"""
import pandas as pd
from io import StringIO
import ogr, osr, gdal
import numpy as np
import matplotlib.pyplot as plt
input_profile = 'elevation_velocity_profile.txt'
old = pd.read_csv("profile_no_nlm.csv")
utm_fname = 'offset_20150819_20170909_ws32_sws48_zws8_skip2_os4_snr09_Y_m_yr.tif'
inputEPSG = 4326
outputEPSG = int(
gdal.Info(utm_fname, format='json')['coordinateSystem']['wkt'].rsplit(
'"EPSG","', 1)[-1].split('"')[0])
# create coordinate transformation fo rinput
inSpatialRef = osr.SpatialReference()
inSpatialRef.ImportFromEPSG(inputEPSG)
# create coordinate transformation for output
outSpatialRef = osr.SpatialReference()
outSpatialRef.ImportFromEPSG(outputEPSG)
coordTransform = osr.CoordinateTransformation(inSpatialRef, outSpatialRef)
f = open(input_profile, 'r')
profile_data = f.read()
profile_chunks = profile_data.split('>')
profile_pd = pd.DataFrame(columns=[
'Longitude', 'Latitude', 'UTM_X', 'UTM_Y', 'Distance_Profile_m',
'Elevation_mean', 'Elevation_stddev', 'Vel1_mean', 'Vel1_stddev',
def rast(shape_path, tiff_path, output_path):
print("clip shape")
shape_crop = shape_path
inputfile = tiff_path
classes_file = shape_crop
m_info = gdal.Info(inputfile).split("\n")
for line in m_info:
if line.startswith("Pixel Size"):
p_size_x = float(line.split("(")[1].split(",")[0])
p_size_y = float(
line.split("(")[1].split(",")[1]
[0:len(line.split("(")[1].split(",")[1]) - 1])
cropped_size = [
y.strip(",") for y in ([
x for x in gdal.Info(inputfile).split("\n")
if x.startswith("Size is")
][0]).split() if y.strip(",").isdigit()
]
outshape = [int(x) for x in cropped_size]
print("rasterize")
# assign values to classes
file = ogr.Open(classes_file)
layer = file.GetLayer(0)
m_drv = ogr.GetDriverByName('Memory')
m_ds = m_drv.CreateDataSource('wrk')
m_ds.CopyLayer(layer, 'ColouredClasses')
layerCopy = m_ds.GetLayer(0)
layerCopy.CreateField(ogr.FieldDefn("FEATVAR", ogr.OFTInteger))
colormap = {
"Traffic area": 255,
"Buildings": 230,
"Semi-built terrain": 205,
"Recreatieterrein": 180,
"Agrarisch terrein": 155,
"Overig agrarisch terrein": 130,
"Binnenwater": 105,
"Buitenwater": 80,
"Buitenland": 55,
'high': 0,
"low": 240
}
for cnt in range(layerCopy.GetFeatureCount()):
feat = layerCopy.GetFeature(cnt)
clss = feat.GetFieldAsString('wordt2012')
feat.SetField("FEATVAR", colormap[classmap[clss]])
layerCopy.SetFeature(feat)
layerCopy.ResetReading()
# Rasterize with GDAL
# Define pixel_size and NoData value of new raster
pixel_size_x = p_size_x
pixel_size_y = p_size_y
NoData_value = -9999
# Filename of input OGR file
vector_fn = classes_file
# Filename of the raster Tiff that will be created
raster_fn = output_path
# Open the data source and read in the extent
source_ds = ogr.Open(vector_fn)
source_layer = layerCopy
x_min, x_max, y_min, y_max = source_layer.GetExtent()
# Create the destination data source
x_res = outshape[0]
y_res = outshape[1]
target_ds = gdal.GetDriverByName('GTiff').Create(raster_fn, x_res, y_res,
1, gdal.GDT_Byte)
target_ds.SetGeoTransform((x_min, pixel_size_x, 0, y_max, 0, pixel_size_y))
target_ds.SetProjection(layer.GetSpatialRef().ExportToWkt())
band = target_ds.GetRasterBand(1)
band.SetNoDataValue(NoData_value)
# Rasterize
print("Rasterize")
gdal.RasterizeLayer(target_ds, [1],
source_layer,
burn_values=[255],
options=["ATTRIBUTE=FEATVAR"])
del file
del source_ds
prune_factor = args.prune_factor
scale_factor = args.scale_factor
threshold = args.threshold
URasterInfo = GeoImgInfo(args.Udata)
VRasterInfo = GeoImgInfo(args.Vdata)
URasterCount = URasterInfo.RasterCount
VRasterCount = VRasterInfo.RasterCount
assert URasterCount == VRasterCount
RasterCount = URasterCount
RasterInfo = URasterInfo
Ufill_value = URasterInfo.nodatavalue
Vfill_value = VRasterInfo.nodatavalue
gdi = gdal.Info(args.Udata)
driver = re.search("Driver:[ \t]*([^\n\r]*)", gdi).group(1)
# It would be nice to only use gdal and not netcdf4python
tdim = None
if driver == "netCDF/Network Common Data Format":
nc_file_u = args.Udata.split(":")[1]
nc = NC(nc_file_u, "r")
xdim, ydim, zdim, tdim = get_dims(nc)
if tdim:
time = nc.variables[tdim]
time_units = time.units
time_calendar = time.calendar
cdftime = utime(time_units, time_calendar)
projection =
#REPROJECT ALL RASTERS IF THIER PROJECTIONS DIFFERFROM THE SPECIFIED PROJECTION
#get EPSG
ds=gdal.Open(path)
prj=ds.GetProjection()
print prj
srs=osr.SpatialReference(wkt=prj)
if srs.IsProjected:
print srs.GetAttrValue('projcs')
print srs.GetAttrValue('geogcs')
#reads the WKT representation of the file's spatial reference, then parses the string to extract the EPSG
epsg = int(gdal.Info(input, format='json')['coordinateSystem']['wkt'].rsplit('"EPSG","', 1)[-1].split('"')[0])
#or
#path = r"H:\workspace\TEUI\BridgerTeton"
d = gdal.Open(path)
proj = osr.SpatialReference(wkt=d.getProjection())
proj.AutoIdentifyEPSG()
print(proj.GetAttrValue('AUTHORITY', 1))
#Reproject rasters
def reproject_raster (raster, \pixel spacing= , epsg_from=, epsg_to= ):
#IF RASTERS ARE DIFFERENT EXTENTS, CLIP TO SMALLEST EXTENT
#determine raster extents
exec("cmap = {}.reversed()".format(cmap_name[:-2]))
else:
exec("cmap = {}".format(cmap_name))
cmap_name = cmap_name.replace('SCM.', '')
plt.register_cmap(name=cmap_name, cmap=cmap, lut=n_color)
elif cmap_name == 'insar':
cdict = tools_lib.cmap_insar()
plt.register_cmap(name='insar', data=cdict)
cmap = plt.get_cmap(cmap_name, n_color)
#%% Set color range
### Auto
if cmin is None:
cmin = gdal.Info(infile, computeMinMax=True, format="json")["bands"][0]['computedMin']
if cmax is None:
cmax = gdal.Info(infile, computeMinMax=True, format="json")["bands"][0]['computedMax']
### float -> int if int
if np.mod(cmin, 1) == 0: cmin = int(cmin)
if np.mod(cmax, 1) == 0: cmax = int(cmax)
#%% Set file name
if not outfile:
outfile = infile.replace('.tif', '.{}_{}_{}.tif'.format(cmap_name, cmin, cmax))
#%% Create color table
### Format: value R G B alpha
#==============================================================================
# CONTOUR HERE IS USED TO DEFINE THE BOUNDARIES OF THE ROS DATA
#==============================================================================
contourband = opentif.GetRasterBand(1)
contourarray = contourband.ReadAsArray()
contour = contourarray >= 0
#==============================================================================
# SET PLOT EXTENT VARIABLES
# FROM 'info' LOWER LEFT COORDINATES ARE EXTRACTED AND CONVERTED TO DECIMAL
# DEGREES. THESE COORDINATES ARE USED TO CONVERT THE PLOT FROM ROWS/COLUMNS
# TO LAT/LONG THERE WILL LIKELY BE DISTORTION IN THE LAT/LONG LABELS.
#==============================================================================
info = gdal.Info(opentif)
longll = -168.20750000
latll = 53.34969444
cs = 0.048
extent = [longll, longll + shape[1] * cs, latll, latll + shape[0] * cs]
#==============================================================================
# PLOT THE ROS ARRAY
#==============================================================================
fig, ax = plt.subplots()
cmap = plt.cm.YlOrRd #designate the color ramp to display ROS images
outline = ax.contour(contour,extent, colors = 'k', origin='upper',\
linewidths = 0.075)
sums = ax.imshow(array, extent=extent, cmap=cmap)
ax.grid(linestyle='-.')
def lambda_handler(event, context):
print(event)
# establish some variables
# prepare input event
if 'sourceBucket' not in event.keys():
source_bucket = event['Records'][0]['s3']['bucket']['name']
source_key = event['Records'][0]['s3']['object']['key']
print('fired by event!')
print(source_bucket, source_key)
else:
source_bucket = event['sourceBucket']
source_key = event['sourceKey']
# update ACL for uploaded /scanned tif
# /scanned tif only exists for frames and indexes - NOT mosaics
if '/frames/scanned/' in source_key or '/index/scanned/' in source_key:
print('updating scanned/ tif ACL to public-read')
print(source_key)
client = boto3.client('s3')
response = client.put_object_acl(ACL='public_read',Bucket=source_bucket,Key=source_key)
print(response)
gdal.VSICurlClearCache()
return
# verify input is a georef upload
if 'georef/' not in source_key:
print("error: key doesn't include '/georef/'. exiting...")
print(source_key)
return
# verify not a temp tif in the georefSubDir
elif georef_sub_dir in source_key:
print("error: key includes the 'georefSubDir' env variable. exiting...")
print(source_key)
return
# verify not a temp tif in epsg3857 directory
elif epsg_sub_dir in source_key:
print("error: key includes the 'epsgSubDir' env variable. exiting...")
print(source_key)
return
else:
dst_crs = 'EPSG:3857'
prefix = '/vsis3/' + source_bucket + '/'
s3_path = prefix + source_key
epsg_folder = 'georef/' + epsg_sub_dir
upload_key = source_key.replace('georef/', epsg_folder).replace('TIF', 'tif')
gdal.VSICurlClearCache()
try:
epsg = int(gdal.Info(s3_path, format='json')['coordinateSystem']['wkt'].rsplit('"EPSG","', 1)[-1].split('"')[0])
print(epsg)
except Exception as e:
print(e)
print("no epsg. uh oh... this raster is probably georeferenced but wasn't exported to GeoTiff")
epsg = ''
# connect to s3
client = boto3.client('s3')
if epsg != 3857:
print("let's reproject...")
reprojected_tif = '/tmp/reprojected.tif'
try:
gdal.Warp(reprojected_tif,s3_path,dstSRS=dst_crs)
except Exception as e:
print(e)
print('uploading reprojected tif')
client.upload_file(reprojected_tif, source_bucket, upload_key)
gdal.VSICurlClearCache()
# cleanup /tmp
print('cleaning up /tmp')
os.remove(reprojected_tif)
else:
print("already EPSG 3857; copying to epsg folder.")
client.copy({'Bucket': source_bucket, 'Key': source_key}, source_bucket, upload_key)
print('invoking ls4-01-compress')
client = boto3.client('lambda')
payload = {'sourceBucket': source_bucket, 'sourceKey': upload_key}
response = client.invoke(
FunctionName='ls4-01-compress',
InvocationType='Event',
Payload=json.dumps(payload)
)
print(response)
# check if original exists in /scanned
client = boto3.client('s3')
# /scanned tif only exists for frames and indexes - NOT mosaics
if '/frames/georef/' in source_key or '/index/georef/' in source_key:
orig = source_key.replace('georef/', 'scanned/')
# original index uploads don't include tiled number
if '/index/' in orig:
tile_num = orig.split('/')[-1].split('_')[-1]
sfx = tile_num.split('.')[-1]
orig = orig.replace('_' + tile_num, '.' + sfx)
try:
r = client.head_object(Bucket=source_bucket, Key=orig)
print('original exists at %s' % orig)
except Exception as e:
print(e)
# try alternative extension
try:
if '.tif' in orig:
alt = orig.replace('.tif', '.TIF')
elif '.TIF' in orig:
alt = orig.replace('.TIF', '.tif')
else:
raise Exception('no tif or TIF extension!')
r = client.head_object(Bucket=source_bucket, Key=alt)
print('original exists at %s' % alt)
except Exception as e:
print(e)
# publish message to the project SNS topic
sns = boto3.resource('sns')
arn = 'arn:aws:sns:us-east-1:%s:%s' % (aws_account, sns_error_topic)
topic = sns.Topic(arn)
m = ("GeoTiff uploaded at '%s' in the LS4 bucket is missing "
"it's original in the relative /scanned folder. Expected "
"it to exist at '%s'. Please upload it now while you're "
"doing LS4y stuffs. Might as well... ya know? Thanks "
"friend!" % (source_key, orig)
)
response = topic.publish(
Message=m,
Subject='LS4 Notification'
)
print('original /scanned not in s3. sns error message dispatched.')
print("that's all folks!!")
