def product_xml_imagery_files(cls, xml): """ Return a list of which imagery files are associated with a RS-2 product.xml file""" imagery_files = [ f.strip() for f in re.findall(".*tif", gdal.Info(xml)) ] return (imagery_files)
def __init__(self, source_img, target_img = None, dstSRS = None, srcNodata = np.nan, dstNodata = np.nan, outputType = None, verbose = False, xmin = None, xmax = None, ymin = None, ymax = None, xRes = None, yRes = None, xSize = None, ySize = None, resample = 1 ): self.source_img = source_img self.target_img = target_img self.verbose = verbose self.dstSRS = dstSRS self.srcNodata = srcNodata self.dstNodata = dstNodata self.outputType = gdal.GDT_Unknown if outputType is None else outputType self.xmin = xmin self.xmax = xmax self.ymin = ymin self.ymax = ymax self.xRes = xRes self.yRes = yRes self.xSize = xSize self.ySize = ySize self.resample = resample if self.srcNodata is None: try: self.srcNodata = ' '.join([i.split("=")[1] for i in gdal.Info(self.source_img).split('\n') if' NoData' in i]) except: self.srcNodata = None if (self.target_img is None) & (self.dstSRS is None): raise IOError('Projection should be specified ether from a file or a projection code.') elif self.target_img is not None: try: g = gdal.Open(self.target_img) except: g = target_img geo_t = g.GetGeoTransform() x_size, y_size = g.RasterXSize, g.RasterYSize if self.xRes is None: self.xRes = abs(geo_t[1]) if self.yRes is None: self.yRes = abs(geo_t[5]) if self.xSize is not None: x_size = 1. * self.xSize * self.xRes / abs(geo_t[1]) if self.ySize is not None: y_size = 1. * self.ySize * self.yRes / abs(geo_t[5]) xmin, xmax = min(geo_t[0], geo_t[0] + x_size * geo_t[1]), \ max(geo_t[0], geo_t[0] + x_size * geo_t[1]) ymin, ymax = min(geo_t[3], geo_t[3] + y_size * geo_t[5]), \ max(geo_t[3], geo_t[3] + y_size * geo_t[5]) dstSRS = osr.SpatialReference( ) raster_wkt = g.GetProjection() dstSRS.ImportFromWkt(raster_wkt) self.g = gdal.Warp('', self.source_img, format = 'MEM', outputBounds = [xmin, ymin, xmax, ymax], dstNodata=self.dstNodata, warpOptions = ['NUM_THREADS=ALL_CPUS'],\ xRes = self.xRes, yRes = self.yRes, dstSRS = dstSRS, outputType = self.outputType, srcNodata = self.srcNodata, resampleAlg = self.resample) else: self.g = gdal.Warp('', self.source_img, format = 'MEM', outputBounds = [self.xmin, self.ymin, \ self.xmax, self.ymax], xRes = self.xRes, yRes = self.yRes, dstSRS = self.dstSRS, warpOptions = ['NUM_THREADS=ALL_CPUS'],\ copyMetadata=True, outputType = self.outputType, dstNodata=self.dstNodata, srcNodata = self.srcNodata, resampleAlg = self.resample) if self.g.RasterCount <= 3: self.data = self.g.ReadAsArray() #return self.data elif self.verbose: print('There are %d bands in this file, use g.GetRasterBand(<band>) to avoid reading the whole file.'%self.g.RasterCount)
import os import gdal import glob2 input_folder= r'C:/Users/arpan/Documents/MAIAC/2016' output_folder = r'C:/Users/arpan/Documents/MAIAC/2016/Projected' latlon_info = gdal.Info(r'C:/Users/arpan/Documents/MAIAC/MAIACLatlon.h03v04.hdf') #print(latlon_info) gdal.Translate(r"C:/Users/arpan/Documents/MAIAC/h03v04_lat.vrt", r'HDF4_EOS:EOS_GRID:"C:/Users/arpan/Documents/MAIAC/MAIACLatlon.h03v04.hdf":latlon:lat', format = 'VRT' ) gdal.Translate(r"C:/Users/arpan/Documents/MAIAC/h03v04_lon.vrt", r'HDF4_EOS:EOS_GRID:"C:/Users/arpan/Documents/MAIAC/MAIACLatlon.h03v04.hdf":latlon:lon', format = 'VRT' ) filenames = glob2.glob(r'C:/Users/arpan/Documents/MAIAC/2016/*.hdf') #print(filenames) geoloc_xml = """<VRTDataset rasterXSize="1200" rasterYSize="1200"> <Metadata domain="GEOLOCATION"> <mdi key="X_DATASET">h03v04_lon.vrt</mdi> <mdi key="X_BAND">1</mdi> <mdi key="Y_DATASET">h03v04_lat.vrt</mdi> <mdi key="Y_BAND">1</mdi> <mdi key="PIXEL_OFFSET">0</mdi> <mdi key="LINE_OFFSET">0</mdi> <mdi key="PIXEL_STEP">1</mdi> <mdi key="LINE_STEP">1</mdi> </Metadata> <VRTRasterBand dataType="Float32" band="1">
for file in f: if '.tif' in file.lower(): files.append(os.path.join(r, file)) if len(files) == 0: print("Please specify a folder that contains TIF files.") exit() # Process center for each file for file in files: # Vars lat = "Invalid File" long = "" # Analyze image file image_transform = size = [] if file != "": image_info = gdal.Info(file, format="json") image_transform = image_info['geoTransform'] size = image_info["size"] # Calculate centers width = image_transform[1] * size[0] height = image_transform[5] * size[1] long = image_transform[0] + 0.5 * width lat = image_transform[3] + 0.5 * height # Print file info name = ntpath.basename(file) print("{0:<40}{1:<20}{2:<20}".format(name, lat, long))
def _get_min_max(self, raster_path): info = gdal.Info(raster_path, format="json", computeMinMax=True) return (info["bands"][0]["computedMax"], info["bands"][0]["computedMax"])
def convert_wrapper(ix_im): imd = imdates2[ix_im] if np.mod(ix_im, 10)==0: print(' Finished {0:4}/{1:4}th sltd...'.format(ix_im, len(imdates2)), flush=True) ztdfile = os.path.join(gacosdir, imd+'.ztd') sltdtiffile = os.path.join(gacosdir, imd+'.sltd.geo.tif') if os.path.exists(sltdtiffile): infile = os.path.basename(sltdtiffile) try: ### Cut and resapmle. Already in rad. sltd_geo = gdal.Warp("", sltdtiffile, format='MEM', outputBounds=outputBounds, width=width_geo, height=length_geo, resampleAlg=resampleAlg, srcNodata=0).ReadAsArray() except: ## if broken print (' {} cannot open. Skip'.format(infile), flush=True) return imd elif os.path.exists(ztdfile): infile = os.path.basename(ztdfile) hdrfile = os.path.join(sltddir, imd+'.hdr') bilfile = os.path.join(sltddir, imd+'.bil') if os.path.exists(hdrfile): os.remove(hdrfile) if os.path.exists(bilfile): os.remove(bilfile) make_hdr(ztdfile+'.rsc', hdrfile) os.symlink(os.path.relpath(ztdfile, sltddir), bilfile) ## Check read error with unkown cause if gdal.Info(bilfile) is None: ### Create new ztd by adding 0.0001m print('{} cannot open, but trying minor update. You can ignore this error unless this script stops.'.format(ztdfile)) shutil.copy2(ztdfile, ztdfile+'.org') ## Backup _ztd = np.fromfile(ztdfile, dtype=np.float32) _ztd[_ztd!=0] = _ztd[_ztd!=0]+0.001 _ztd.tofile(ztdfile) ### Cut and resapmle ztd to geo ztd_geo = gdal.Warp("", bilfile, format='MEM', outputBounds=outputBounds,\ width=width_geo, height=length_geo, \ resampleAlg=resampleAlg, srcNodata=0).ReadAsArray() os.remove(hdrfile) os.remove(bilfile) ### Meter to rad, slantrange sltd_geo = ztd_geo/LOSu*m2r_coef ## LOSu=cos(inc) else: print(' There is no ztd|sltd.geo.tif for {}!'.format(imd), flush=True) return imd ## Next imd ### Skip if no data in the area if np.all((sltd_geo==0)|np.isnan(sltd_geo)): print(' There is no valid data in {}!'.format(infile), flush=True) return imd ## Next imd ### Fill hole is specified if fillholeflag: sltd_geo = fillhole(sltd_geo) ### Output as sltd.geo sltd_geofile = os.path.join(sltddir, imd+'.sltd.geo') sltd_geo.tofile(sltd_geofile) return
def getInfo(self): print(gdal.Info(self.dataset))
for j in fl: # Reproject and resample layers. src_filename = j src = gdal.Open(src_filename, gdalconst.GA_ReadOnly) src_proj = src.GetProjection() src_geotrans = src.GetGeoTransform() # Create output file name dst_filename = odfor + os.sep + os.path.basename(src_filename).replace('.tif',projsuffix) # Run if output doesn't exist if not os.path.exists(dst_filename): # If xmax of original raster is > 180, clip it to avoid wraparound # Same with xmin if it is < -180. # Need to get xmax from gdalinfo since I get wraparound when I try to calculate from geotransform gi = gdal.Info(src_filename).split("\n") gimax = [i for i in gi if "Upper Right" in i] #gi = float(gi[0].split("(")[1].split(" ")[1].split(",")[0]) gimax = float(gimax[0].split("(")[1].replace(" ","").split(",")[0]) # Get xmin gimin = [i for i in gi if "Upper Left" in i] #gi = float(gi[0].split("(")[1].split(" ")[1].split(",")[0]) gimin = float(gimin[0].split("(")[1].replace(" ","").split(",")[0]) # Clip rasters if they're beyond domain edges if gimax > 180: # Specify new xmax newcoord = "179.99" newof = os.path.dirname(src_filename) + os.sep + os.path.basename(src_filename).split(".")[0] + "_v2.tif" # Subset raster. Note subset window given in ulx, uly, lrx, lry order subprocess.call(["gdal_translate", "-co", "COMPRESS=LZW", "-projwin", str(src_geotrans[0]), str(src_geotrans[3]), newcoord, str(src_geotrans[3] + src_geotrans[5]*src.RasterYSize), src_filename, newof]) # Reproject and resample layers.
] hashDict = {} for ii in inFnames: thisBasename = os.path.basename(ii) thisDS = gdal.Open(ii, gdalconst.GA_ReadOnly) ret = gdal.Info(thisDS, format='json', deserialize=True, computeMinMax=False, reportHistograms=False, reportProj4=False, stats=False, approxStats=False, computeChecksum=True, showGCPs=False, showMetadata=False, showRAT=False, showColorTable=False, listMDD=False, showFileList=False) # concatCkcksum='' # for iband in ret['bands']: # if 'checksum' in iband: # concatCkcksum = '{}{}'.format(concatCkcksum, iband['checksum']) concatCkcksum = ret['bands'][0]['checksum'] hashDict[
base=sys.argv[1], project_name=project_name, soil_name=soil_name)) else: copytree( soil_fn, '{base}/{project_name}/Watershed/Rasters/Soil/{soil_name}'.format( base=sys.argv[1], project_name=project_name, soil_name=soil_name)) # src_ds = gdal.Open(soil_fn) # ds = gdal.Translate('{base}/{project_name}/Watershed/Rasters/Soil/{soil_name}'.format( # base=sys.argv[1], project_name=project_name, soil_name=soil_name), src_ds, format='GTiff') log.info("getting dem projection information", keep_log) dataset = gdal.Open( '{base}/{project_name}/Watershed/Rasters/DEM/{dem_name}'.format( base=sys.argv[1], project_name=project_name, dem_name=dem_file_name_)) formated_projcs = gdal.Info(dataset).split("Data axis")[0].split( "System is:\n")[-1] prjcrs = "" for line in formated_projcs.split("\n"): while line.startswith(" "): line = line[1:] prjcrs += line.strip("\n") srs = osr.SpatialReference(wkt=prjcrs) proj4 = srs.ExportToProj4() geogcs = srs.GetAttrValue('geogcs') log.info("geogcs: {0}".format(geogcs), keep_log) log.info("proj4: {0}".format(proj4), keep_log) if prjcrs.split('"')[1] in epsg_lookup_dictionary:
def ai_makecog(repo: Repo, task: Task, roi_id, recipe_path, json_only, quiet, no_color, less, zone, kind, source, cos_key, friendly_name, print_res, warp_resampleAlg, overview_resampleAlg): """ Make COG TIFF from visualized results \b * to make GeoTIFF from custom ENVI file use --source option with filename of ENVI file (without extension) * to override recipe kind, use --kind option, example: making image from 'ai preview --export path/to/envi' file use makecog zone --kind Image --source path/to/envi * to avoid overriding recipe main results use --cos-key and --friendly-name option if --friendly-name starts with '+' value will be used as suffix for friendly_name in GeoJSON if --cos-key starts with '+' value will be used as suffix for COS.ResultKey in GeoJSON """ driver = 'MAKECOG' if source: try: # Only valid ENVI or GeoTiff files are alloed str = gdal.Info(source, format='json') driver = str['driverShortName'] if driver not in ['ENVI', 'GTiff']: raise OCLIException( f"Unsupported source file type: {str['driverShortName']} ({str['driverLongName']})" ) except Exception as e: raise OCLIException(f"Option --source: {e}") _recipe = recipe_path if recipe_path else resolve_recipe( repo, task, roi_id) recipe = Recipe(_recipe) kind = kind if kind != 'auto' else recipe.get('type') if kind in ['Rvi', 'Image']: recipe['type'] = 'Image' # log.error(recipe['COS']['ResultKey']) if cos_key: if cos_key.startswith('+'): recipe['COS']['ResultKey'] += cos_key[1:] else: recipe['COS']['ResultKey'] = cos_key if friendly_name: if friendly_name.startswith('+'): recipe['friendly_name'] += friendly_name[1:] else: recipe['friendly_name'] = friendly_name # log.error(recipe['COS']['ResultKey']) # log.error(recipe['friendly_name']) filenames = Filenames(zone, recipe) if source: input_file = source else: input_file = filenames.pred8c_img out_file = filenames.out_tiff cog_file = filenames.out_cog_tiff check_file = cog_file if json_only else input_file if not Path(check_file).is_file(): raise OCLIException(f'file not found: {check_file}') os.makedirs(Path(cog_file).parent, exist_ok=True) w = GDALWrap3(recipe, input_file, out_file, cog_file) try: if not json_only: if driver == 'GTiff': try: shutil.copy(input_file, cog_file) output.success(f"file {input_file} copied to {cog_file}") except Exception as e: raise OCLIException( f'Could not copy "{input_file}" to "{cog_file}" : {e}' ) else: if not quiet: with pfac(log, total=100, desc='Assembling') as (_, callback): def cb(pct, msg, user_data): _ud = user_data[0] # self.log.debug(f"translaing: {round(pct * 100, 2)}% -{msg}- {user_data} {pct}") # reset counter, this t\callback called from multiple prcesses if pct < 0.01: user_data[0] = 0 if user_data[0] == 0 or pct - _ud > 0.10: log.debug( f"Local translating : {round(pct * 100, 2)}%" ) user_data[0] = pct callback(100, pct, 'translating') w.make_cog(cb, warp_resampleAlg, overview_resampleAlg) else: w.make_cog(None, warp_resampleAlg, overview_resampleAlg) _json = w.make_geo_json() _json = _json if no_color else colorful_json(_json) if print_res: click.echo("\n\n\n") if less: click.echo_via_pager(_json) else: click.echo(_json) except (RuntimeError, OCLIException) as e: raise click.UsageError( output.error_style(f"COG-tiff generation failed,reason: {e}")) pass
import gdal import osr #LSDTopoTools specific imports #Loading the LSDTT setup configuration setup_file = open('chi_automation.config','r') LSDMT_PT = setup_file.readline().rstrip() LSDMT_MF = setup_file.readline().rstrip() Iguanodon = setup_file.readline().rstrip() setup_file.close() sys.path.append(LSDMT_PT) sys.path.append(LSDMT_MF) sys.path.append(Iguanodon) import LSDPlottingTools as LSDPT directory = "/exports/csce/datastore/geos/groups/LSDTopoData/Himalayan_Ksn_Concavity/cosmo_data/basin_shapefiles/" rasterDS = gdal.Open(directory+'output5.tif') print gdal.Info(rasterDS) srs = osr.SpatialReference() ###problematic - get from input file directly srs.ImportFromEPSG(4326) #outputDS = gdal.Open(directory+'output6.tif') gdal.Warp(directory+'outputDS.tif',rasterDS,dstSRS=srs) #print gdal.Info(outputDS)
def modis_dl_reproject(product, collection, date, download_root, needy=True): # Select the correct nasa server based on the product requested if product == 'MOD09GA': nasa_server = 'MOLT' elif product == 'MYD09GA': nasa_server = 'MOLA' # The url of the file we wish to retrieve #url = "https://e4ftl01.cr.usgs.gov/MOLT/MOD09GA.006/2014.06.25/" #myd = 'https://e4ftl01.cr.usgs.gov/MOLA/MYD09GA.006/' url = "https://e4ftl01.cr.usgs.gov/{}/{}.{}/{}/".format(nasa_server,product,collection,date) # ADD: Test to make sure they entered a valid product / collection / date # Initiate the username / password manager, and the cookie jar for this # session. cookie_jar = initiate_urllib() ## ------------------------------------------------------------------------ ## Find list of files (https) to download ## ------------------------------------------------------------------------ # Read the html of the http server for the desired modis product try: html_body = read_html(url) except urllib.error.HTTPError: print(("Could not reach url: {}".format(url))) return # instantiate the parser and fed it some HTML text parser = MyHTMLParser() parser.feed(html_body) # Get a list of the available data on the http server data_list = parser.data_list # Calculate the granules needed to cover the arctic ocean granule_list = calc_arctic_granules() # Compare the available files with the ones required. Goes through the # whole list of needed granules, and finds the full image name that matches # each granule ID. Once the right image is found, stopiteration. download_list = [] for granule in granule_list: download_list.append(next((file_ for file_ in data_list if granule in file_),None)) ## ------------------------------------------------------------------------ ### Download requested granules from NASA server ## ------------------------------------------------------------------------ question = "There are [{}] files to download. Continue? [y/n] ".format(len(download_list)) answer = query_yes_no(question,needy) if answer is True: # Create the download path based on the images to be downloaded download_path = os.path.join(download_root, 'MODIS/{}/{}/originals'.format(product,date)) if not os.path.isdir(download_path): os.makedirs(download_path) # Submit the download list to be downloaded batch_job(url,download_list,download_path) else: print("Goodbye.") quit() # print "Normally I'd quit here." # Check that download succeeded pass_ = False while pass_ is False: pass_, failed_dl = verify_downloads(download_list,download_path) if pass_: infostring = gdal.Info(os.path.join(download_path,download_list[0])) question = "File verification successful. Continue to mosaic and reproject? [y/n] " if query_yes_no(question,needy) is False: quit() else: for dl in failed_dl: print(("Failed download: " + dl)) question = "Would you like to redownload [{}] failed images? [y/n]".format(len(failed_dl)) answer = query_yes_no(question,needy) if answer is True: for dl in failed_dl: os.remove(os.path.join(download_path,dl)) batch_job(url,failed_dl,download_path) else: print("Good day.") quit() ## ------------------------------------------------------------------------ ## Process downloaded files ## ------------------------------------------------------------------------ # Reproject to Arctic Polar Stereographic, mosaic all tiles together, # and convert to a geotiff. This is all done using the gdal_warp utility # Find the names of all of the subdatasets contained within the downloaded # hdf files. In the future, maybe add a way for the user to select the # datasets they're actually interested in. sample_im = download_list[0] # name of one of the downloaded images # Read the metadata in the sample image # list of subdatasets, read from the metadata sds_list = parse_gdalinfo(infostring) # We want to put the mosaic image in the parent directory of the downloads dst_path = os.path.split(download_path)[0] # output image name is "product_date_sds" dst_image_base = sample_im.split('.')[0]+'.'+sample_im.split('.')[1] # list of bands we are interested in reprojecting and mosaicing for now # Experiementing with queues (maybe useful for multithreading later?) reproj_queue = queue.Queue() reproj_list = [ 'sur_refl_b01', 'sur_refl_b02', 'sur_refl_b03', 'sur_refl_b04', 'sur_refl_b05', 'sur_refl_b06', 'sur_refl_b07', 'state_1km' ] # reproj_list = [ 'state_1km'] for band in reproj_list: reproj_queue.put(band) # Reproject and mosaic each band individually # We 'probably' dont want to mosaic _all_ of the bands while not reproj_queue.empty(): band = reproj_queue.get() sds = (next(s for s in sds_list if band in s)) # Parse the band name out of the full sds name band_name = sds.split('"')[-1] # Replace ':' with '_' in the output name dst_band_name = band_name.replace(':','_') # Create the full output path dst_image = os.path.join(dst_path,dst_image_base+dst_band_name+'.tif') # Reproject, mosaic, and convert to geotiff print(("Reprojecting and mosaicing " + band_name)) reproject_mosaic(download_list,band_name,dst_image) print("Removing raw files in {}".format(download_path)) shutil.rmtree(download_path)
ref_pth = r'Y:\germany-drought\vrt\masks\2015_MASK_FOREST-BROADLEAF_BUFF-01.vrt' #ref_pth = r'O:\Student_Data\CJaenicke\00_MA\data\climate\dwd_precipitation\SPI\SPI_12\SPI_012018_12_3035.tif' netcdf_pth = r"O:\Student_Data\CJaenicke\00_MA\data\climate\SMI\SMI_Lall_Gesamtboden_monatlich_1951_2018_inv.nc" #netcdf_pth = r"O:\Student_Data\CJaenicke\00_MA\data\climate\SMI\SMI_L02_Oberboden_monatlich_1951_2018_inv.nc" out_pth = r'O:\Student_Data\CJaenicke\00_MA\data\climate\SMI\SMI_2018_Gesamtboden.tif' print('opening input') ## open a reference raster ref_ras = gdal.Open(ref_pth) gt = ref_ras.GetGeoTransform() pr = ref_ras.GetProjection() ref_arr = ref_ras.ReadAsArray() ## get Info about netcdf File print(gdal.Info(netcdf_pth)) ## open netcdf file, load subds and read them as arrays ds = gdal.Open(netcdf_pth) subds_lst = ds.GetSubDatasets() ## SMI ds_smi = gdal.Open(ds.GetSubDatasets()[0][0]) smi_arr = ds_smi.ReadAsArray() ## kerndel width ds_kw = gdal.Open(ds.GetSubDatasets()[1][0]) kw_arr = ds_kw.ReadAsArray() ## latitude ds_lat = gdal.Open(ds.GetSubDatasets()[2][0]) lat_arr = ds_lat.ReadAsArray() ## longtitude ds_lon = gdal.Open(ds.GetSubDatasets()[3][0])
final_del = np.delete(row_del,0,1) print(final_del) cmap ='viridis' plt.figure(figsize=(10,10)) plt.imshow(final_del,cmap=cmap) plt.show() complete = Image.fromarray(final_del) complete.save("test_1.tif") sr = dataset.GetProjection() gt = dataset.GetGeoTransform() del dataset dataset = gdal.Open("test_1.tif", gdal.GA_Update) dataset.SetProjection(sr) dataset.SetGeoTransform(gt) print(gdal.Info("test_1.tif")) del dataset #3 dataset= gdal.Open("IMG_0111_3.tif", gdal.GA_Update) band = dataset.GetRasterBand(1) array = band.ReadAsArray() print(array) #rotated = np.rot90(array, k=1) #rotated = ndimage.rotate(array,359,reshape=False) padded = np.pad(array, [(0,3000),(0,3000)], 'constant', constant_values=(0))
input_file_1 = "test_1.tif" input_file_2 = "../test_data/IMG_0111_2.tif" input_file_3 = "../test_data/IMG_0111_3.tif" ds = gdal.Open(input_file_2, gdal.GA_ReadOnly) print("driver: {}/{}".format(ds.GetDriver().ShortName, ds.GetDriver().LongName)) print("Size is {} x {} x {}".format(ds.RasterXSize, ds.RasterYSize, ds.RasterCount)) print("Projection is {}".format(ds.GetProjection())) geotransform = ds.GetGeoTransform() if geotransform: print("Origin = ({}, {})".format(geotransform[0], geotransform[3])) print("Pixel Size = ({}, {})".format(geotransform[1], geotransform[5])) ulx, xres, xskew, uly, yskew, yres = ds.GetGeoTransform() lrx = ulx + (ds.RasterXSize * xres) lry = uly + (ds.RasterYSize * yres) print(gdal.Info(input_file_1)) vrt_options = gdal.BuildVRTOptions(separate=True) my_vrt = gdal.BuildVRT('my.vrt', [input_file_1, input_file_2, input_file_3], options=vrt_options) my_vrt = None vrt = gdal.Open('my.vrt') print(vrt) gdal.Translate('stack.tif', vrt)
def main(argv=None): #%% Check argv if argv == None: argv = sys.argv start = time.time() ver = 1.4 date = 20200703 author = "Y. Morishita" print("\n{} ver{} {} {}".format(os.path.basename(argv[0]), ver, date, author), flush=True) print("{} {}".format(os.path.basename(argv[0]), ' '.join(argv[1:])), flush=True) #%% Set default in_dir = [] out_dir = [] gacosdir = 'GACOS' resampleAlg = 'cubicspline' # None # 'cubic' fillholeflag = False #%% Read options try: try: opts, args = getopt.getopt(argv[1:], "hi:o:g:z:", ["fillhole", "help"]) except getopt.error as msg: raise Usage(msg) for o, a in opts: if o == '-h' or o == '--help': print(__doc__) return 0 elif o == '-i': in_dir = a elif o == '-o': out_dir = a elif o == '-z': ## for backward-compatible gacosdir = a elif o == '-g': gacosdir = a elif o == "--fillhole": fillholeflag = True if not in_dir: raise Usage('No input directory given, -i is not optional!') elif not os.path.isdir(in_dir): raise Usage('No {} dir exists!'.format(in_dir)) elif not os.path.exists(os.path.join(in_dir, 'slc.mli.par')): raise Usage('No slc.mli.par file exists in {}!'.format(in_dir)) if not out_dir: raise Usage('No output directory given, -o is not optional!') if not os.path.isdir(gacosdir): raise Usage('No {} dir exists!'.format(gacosdir)) except Usage as err: print("\nERROR:", file=sys.stderr, end='') print(" " + str(err.msg), file=sys.stderr) print("\nFor help, use -h or --help.\n", file=sys.stderr) return 2 #%% Read data information ### Directory in_dir = os.path.abspath(in_dir) gacosdir = os.path.abspath(gacosdir) out_dir = os.path.abspath(out_dir) if not os.path.exists(out_dir): os.mkdir(out_dir) sltddir = os.path.join(os.path.join(out_dir), 'sltd') if not os.path.exists(sltddir): os.mkdir(sltddir) ### Get general info mlipar = os.path.join(in_dir, 'slc.mli.par') width_unw = int(io_lib.get_param_par(mlipar, 'range_samples')) length_unw = int(io_lib.get_param_par(mlipar, 'azimuth_lines')) speed_of_light = 299792458 #m/s radar_frequency = float(io_lib.get_param_par(mlipar, 'radar_frequency')) #Hz wavelength = speed_of_light / radar_frequency #meter m2r_coef = 4 * np.pi / wavelength if wavelength > 0.2: ## L-band cycle = 1.5 # 2pi/cycle for png else: ## C-band cycle = 3 # 2pi*3/cycle for png ### Get geo info. Grid registration dempar = os.path.join(in_dir, 'EQA.dem_par') width_geo = int(io_lib.get_param_par(dempar, 'width')) length_geo = int(io_lib.get_param_par(dempar, 'nlines')) dlat_geo = float(io_lib.get_param_par(dempar, 'post_lat')) #minus dlon_geo = float(io_lib.get_param_par(dempar, 'post_lon')) latn_geo = float(io_lib.get_param_par(dempar, 'corner_lat')) lonw_geo = float(io_lib.get_param_par(dempar, 'corner_lon')) lats_geo = latn_geo + dlat_geo * (length_geo - 1) lone_geo = lonw_geo + dlon_geo * (width_geo - 1) ### Check coordinate if width_unw != width_geo or length_unw != length_geo: print('\n{} seems to contain files in radar coordinate!!\n'.format( in_dir), file=sys.stderr) print('Not supported.\n'.format(in_dir), file=sys.stderr) return 1 ### Calc incidence angle from U.geo ufile = os.path.join(in_dir, 'U.geo') LOSu = io_lib.read_img(ufile, length_geo, width_geo) LOSu[LOSu == 0] = np.nan ### Get ifgdates and imdates ifgdates = tools_lib.get_ifgdates(in_dir) imdates = tools_lib.ifgdates2imdates(ifgdates) n_ifg = len(ifgdates) n_im = len(imdates) #%% Process ztd files print('\nConvert ztd/sltd.geo.tif files to sltd.geo files...', flush=True) ### First check if sltd already exist imdates2 = [] for imd in imdates: sltd_geofile = os.path.join(sltddir, imd + '.sltd.geo') if not os.path.exists(sltd_geofile): imdates2.append(imd) n_im2 = len(imdates2) if n_im - n_im2 > 0: print(" {0:3}/{1:3} sltd already exist. Skip".format( n_im - n_im2, n_im), flush=True) no_gacos_imfile = os.path.join(out_dir, 'no_gacos_im.txt') if os.path.exists(no_gacos_imfile): os.remove(no_gacos_imfile) for ix_im, imd in enumerate(imdates2): if np.mod(ix_im, 10) == 0: print(' Finished {0:4}/{1:4}th sltd...'.format(ix_im, n_im2), flush=True) ztdfile = os.path.join(gacosdir, imd + '.ztd') sltdtiffile = os.path.join(gacosdir, imd + '.sltd.geo.tif') if os.path.exists(sltdtiffile): infile = os.path.basename(sltdtiffile) try: ### Cut and resapmle. Already in rad. sltd_geo = gdal.Warp("", sltdtiffile, format='MEM', outputBounds=(lonw_geo, lats_geo, lone_geo, latn_geo), width=width_geo, height=length_geo, resampleAlg=resampleAlg, srcNodata=0).ReadAsArray() except: ## if broken print(' {} cannot open. Skip'.format(infile), flush=True) with open(no_gacos_imfile, mode='a') as fnogacos: print('{}'.format(imd), file=fnogacos) continue elif os.path.exists(ztdfile): infile = os.path.basename(ztdfile) hdrfile = os.path.join(sltddir, imd + '.hdr') bilfile = os.path.join(sltddir, imd + '.bil') if os.path.exists(hdrfile): os.remove(hdrfile) if os.path.exists(bilfile): os.remove(bilfile) make_hdr(ztdfile + '.rsc', hdrfile) os.symlink(os.path.relpath(ztdfile, sltddir), bilfile) ## Check read error with unkown cause if gdal.Info(bilfile) is None: ### Create new ztd by adding 0.0001m print( '{} cannot open, but trying minor update. You can ignore this error unless this script stops.' .format(ztdfile)) shutil.copy2(ztdfile, ztdfile + '.org') ## Backup _ztd = np.fromfile(ztdfile, dtype=np.float32) _ztd[_ztd != 0] = _ztd[_ztd != 0] + 0.001 _ztd.tofile(ztdfile) ### Cut and resapmle ztd to geo ztd_geo = gdal.Warp("", bilfile, format='MEM', \ outputBounds=(lonw_geo, lats_geo, lone_geo, latn_geo), \ width=width_geo, height=length_geo, \ resampleAlg=resampleAlg, srcNodata=0).ReadAsArray() os.remove(hdrfile) os.remove(bilfile) ### Meter to rad, slantrange sltd_geo = ztd_geo / LOSu * m2r_coef ## LOSu=cos(inc) else: print(' There is no ztd|sltd.geo.tif for {}!'.format(imd), flush=True) with open(no_gacos_imfile, mode='a') as fnogacos: print('{}'.format(imd), file=fnogacos) continue ## Next imd ### Skip if no data in the area if np.all((sltd_geo == 0) | np.isnan(sltd_geo)): print(' There is no valid data in {}!'.format(infile), flush=True) with open(no_gacos_imfile, mode='a') as fnogacos: print('{}'.format(imd), file=fnogacos) continue ## Next imd ### Fill hole is specified if fillholeflag: sltd_geo = fillhole(sltd_geo) ### Output as sltd.geo sltd_geofile = os.path.join(sltddir, imd + '.sltd.geo') sltd_geo.tofile(sltd_geofile) #%% Correct unw files print('\nCorrect unw data...', flush=True) ### Information files gacinfofile = os.path.join(out_dir, 'GACOS_info.txt') if not os.path.exists(gacinfofile): ### Add header with open(gacinfofile, "w") as f: print(' Phase STD (rad) Before After ReductionRate', file=f) no_gacos_ifgfile = os.path.join(out_dir, 'no_gacos_ifg.txt') if os.path.exists(no_gacos_ifgfile): os.remove(no_gacos_ifgfile) ### First check if already corrected unw exist ifgdates2 = [] for i, ifgd in enumerate(ifgdates): out_dir1 = os.path.join(out_dir, ifgd) unw_corfile = os.path.join(out_dir1, ifgd + '.unw') if not os.path.exists(unw_corfile): ifgdates2.append(ifgd) n_ifg2 = len(ifgdates2) if n_ifg - n_ifg2 > 0: print(" {0:3}/{1:3} corrected unw already exist. Skip".format( n_ifg - n_ifg2, n_ifg), flush=True) ### Correct for i, ifgd in enumerate(ifgdates2): if np.mod(i, 10) == 0: print(' Finished {0:4}/{1:4}th unw...'.format(i, n_ifg2), flush=True) md = ifgd[:8] sd = ifgd[-8:] msltdfile = os.path.join(sltddir, md + '.sltd.geo') ssltdfile = os.path.join(sltddir, sd + '.sltd.geo') in_dir1 = os.path.join(in_dir, ifgd) out_dir1 = os.path.join(out_dir, ifgd) ### Check if sltd available for both primary and secondary. If not continue ## Not use in tsa because loop cannot be closed if not (os.path.exists(msltdfile) and os.path.exists(ssltdfile)): print(' ztd file not available for {}'.format(ifgd), flush=True) with open(no_gacos_ifgfile, mode='a') as fnogacos: print('{}'.format(ifgd), file=fnogacos) continue ### Prepare directory and file if not os.path.exists(out_dir1): os.mkdir(out_dir1) unwfile = os.path.join(in_dir1, ifgd + '.unw') unw_corfile = os.path.join(out_dir1, ifgd + '.unw') ### Calculate dsltd msltd = io_lib.read_img(msltdfile, length_unw, width_unw) ssltd = io_lib.read_img(ssltdfile, length_unw, width_unw) msltd[msltd == 0] = np.nan ssltd[ssltd == 0] = np.nan dsltd = ssltd - msltd ### Correct unw unw = io_lib.read_img(unwfile, length_unw, width_unw) unw[unw == 0] = np.nan unw_cor = unw - dsltd unw_cor.tofile(unw_corfile) ### Output std std_unw = np.nanstd(unw) std_unwcor = np.nanstd(unw_cor) rate = (std_unw - std_unwcor) / std_unw * 100 with open(gacinfofile, "a") as f: print('{0} {1:4.1f} {2:4.1f} {3:5.1f}%'.format( ifgd, std_unw, std_unwcor, rate), file=f) ### Link cc if not os.path.exists(os.path.join(out_dir1, ifgd + '.cc')): os.symlink( os.path.relpath(os.path.join(in_dir1, ifgd + '.cc'), out_dir1), os.path.join(out_dir1, ifgd + '.cc')) ### Output png for comparison data3 = [ np.angle(np.exp(1j * (data / cycle)) * cycle) for data in [unw, unw_cor, dsltd] ] title3 = [ 'unw_org (STD: {:.1f} rad)'.format(std_unw), 'unw_cor (STD: {:.1f} rad)'.format(std_unwcor), 'dsltd ({:.1f}% reduced)'.format(rate) ] pngfile = os.path.join(out_dir1, ifgd + '.gacos.png') plot_lib.make_3im_png(data3, pngfile, 'insar', title3, vmin=-np.pi, vmax=np.pi, cbar=False) ## Output png for corrected unw pngfile = os.path.join(out_dir1, ifgd + '.unw.png') title = '{} ({}pi/cycle)'.format(ifgd, cycle * 2) plot_lib.make_im_png(np.angle(np.exp(1j * unw_cor / cycle) * cycle), pngfile, 'insar', title, -np.pi, np.pi, cbar=False) print("", flush=True) #%% Create correlation png pngfile = os.path.join(out_dir, 'GACOS_info.png') plot_lib.plot_gacos_info(gacinfofile, pngfile) #%% Copy other files files = glob.glob(os.path.join(in_dir, '*')) for file in files: if not os.path.isdir(file): #not copy directory, only file print('Copy {}'.format(os.path.basename(file)), flush=True) shutil.copy(file, out_dir) #%% Finish elapsed_time = time.time() - start hour = int(elapsed_time / 3600) minite = int(np.mod((elapsed_time / 60), 60)) sec = int(np.mod(elapsed_time, 60)) print("\nElapsed time: {0:02}h {1:02}m {2:02}s".format(hour, minite, sec)) print('\n{} Successfully finished!!\n'.format(os.path.basename(argv[0]))) print('Output directory: {}\n'.format(os.path.relpath(out_dir))) if os.path.exists(no_gacos_ifgfile): print('Caution: Some ifgs below are excluded due to GACOS unavailable') with open(no_gacos_ifgfile) as f: for line in f: print(line, end='') print('') if os.path.exists(no_gacos_imfile): print('GACOS data for the following dates are missing:') with open(no_gacos_imfile) as f: for line in f: print(line, end='') print('')
def read_image_header(fpath): print('\n\n\n') print(gdal.Info(fpath))
def MatchProjResNDV(source_file, target_fhs, output_dir, resample='near', dtype='float32', scale=None, ndv_to_zero=False): """ Matches the projection, resolution and no-data-value of a list of target-files with a source-file and saves the new maps in output_dir. Parameters ---------- source_file : str The file to match the projection, resolution and ndv with. target_fhs : list The files to be reprojected. output_dir : str Folder to store the output. resample : str, optional Resampling method to use, default is 'near' (nearest neighbour). dtype : str, optional Datatype of output, default is 'float32'. scale : int, optional Multiple all maps with this value, default is None. Returns ------- output_files: :obj:`numpy.ndarray` Filehandles of the created files. """ print('WaPOR GIS: Matching projection...') output_files = np.array([]) dst_info = gdal.Info(gdal.Open(source_file), format='json') if not os.path.exists(output_dir): os.makedirs(output_dir) for target_file in target_fhs: folder, fn = os.path.split(target_file) output_file = os.path.join(output_dir, fn) src_info = gdal.Info(gdal.Open(target_file), format='json') gdal.Warp(output_file, target_file, format='GTiff', srcSRS=src_info['coordinateSystem']['wkt'], dstSRS=dst_info['coordinateSystem']['wkt'], srcNodata=src_info['bands'][0]['noDataValue'], dstNodata=dst_info['bands'][0]['noDataValue'], width=dst_info['size'][0], height=dst_info['size'][1], outputBounds=(dst_info['cornerCoordinates']['lowerLeft'][0], dst_info['cornerCoordinates']['lowerLeft'][1], dst_info['cornerCoordinates']['upperRight'][0], dst_info['cornerCoordinates']['upperRight'][1]), outputBoundsSRS=dst_info['coordinateSystem']['wkt'], resampleAlg=resample) output_files = np.append(output_files, output_file) if not np.any([scale == 1.0, scale is None, scale == 1]): driver, NDV, xsize, ysize, GeoT, Projection = GetGeoInfo( output_file) DATA = OpenAsArray(output_file, nan_values=True) * scale CreateGeoTiff( output_file, DATA, driver, NDV, xsize, ysize, GeoT, Projection) if ndv_to_zero: driver, NDV, xsize, ysize, GeoT, Projection = GetGeoInfo( output_file) DATA = OpenAsArray(output_file, nan_values=False) DATA[DATA == NDV] = 0.0 CreateGeoTiff( output_file, DATA, driver, NDV, xsize, ysize, GeoT, Projection) return output_files
def pixel_size(self): info = gdal.Info(self._path, format="json") return abs(info["geoTransform"][1])
def _get_native_pixel_size(self, raster_path): info = gdal.Info(raster_path, format="json") return abs(info["geoTransform"][1])
def data_type(self): info = gdal.Info(self._path, format="json") return info["bands"][0]["type"]
def extract_sensing_month(filepath): metadata = gdal.Info(filepath) st_index = metadata.find("SENSING_TIME") if st_index != -1: month = metadata[st_index + 18:st_index + 20] return int(month)
@author: bodo """ import pandas as pd from io import StringIO import ogr, osr, gdal import numpy as np import matplotlib.pyplot as plt input_profile = 'elevation_velocity_profile.txt' old = pd.read_csv("profile_no_nlm.csv") utm_fname = 'offset_20150819_20170909_ws32_sws48_zws8_skip2_os4_snr09_Y_m_yr.tif' inputEPSG = 4326 outputEPSG = int( gdal.Info(utm_fname, format='json')['coordinateSystem']['wkt'].rsplit( '"EPSG","', 1)[-1].split('"')[0]) # create coordinate transformation fo rinput inSpatialRef = osr.SpatialReference() inSpatialRef.ImportFromEPSG(inputEPSG) # create coordinate transformation for output outSpatialRef = osr.SpatialReference() outSpatialRef.ImportFromEPSG(outputEPSG) coordTransform = osr.CoordinateTransformation(inSpatialRef, outSpatialRef) f = open(input_profile, 'r') profile_data = f.read() profile_chunks = profile_data.split('>') profile_pd = pd.DataFrame(columns=[ 'Longitude', 'Latitude', 'UTM_X', 'UTM_Y', 'Distance_Profile_m', 'Elevation_mean', 'Elevation_stddev', 'Vel1_mean', 'Vel1_stddev',
def rast(shape_path, tiff_path, output_path): print("clip shape") shape_crop = shape_path inputfile = tiff_path classes_file = shape_crop m_info = gdal.Info(inputfile).split("\n") for line in m_info: if line.startswith("Pixel Size"): p_size_x = float(line.split("(")[1].split(",")[0]) p_size_y = float( line.split("(")[1].split(",")[1] [0:len(line.split("(")[1].split(",")[1]) - 1]) cropped_size = [ y.strip(",") for y in ([ x for x in gdal.Info(inputfile).split("\n") if x.startswith("Size is") ][0]).split() if y.strip(",").isdigit() ] outshape = [int(x) for x in cropped_size] print("rasterize") # assign values to classes file = ogr.Open(classes_file) layer = file.GetLayer(0) m_drv = ogr.GetDriverByName('Memory') m_ds = m_drv.CreateDataSource('wrk') m_ds.CopyLayer(layer, 'ColouredClasses') layerCopy = m_ds.GetLayer(0) layerCopy.CreateField(ogr.FieldDefn("FEATVAR", ogr.OFTInteger)) colormap = { "Traffic area": 255, "Buildings": 230, "Semi-built terrain": 205, "Recreatieterrein": 180, "Agrarisch terrein": 155, "Overig agrarisch terrein": 130, "Binnenwater": 105, "Buitenwater": 80, "Buitenland": 55, 'high': 0, "low": 240 } for cnt in range(layerCopy.GetFeatureCount()): feat = layerCopy.GetFeature(cnt) clss = feat.GetFieldAsString('wordt2012') feat.SetField("FEATVAR", colormap[classmap[clss]]) layerCopy.SetFeature(feat) layerCopy.ResetReading() # Rasterize with GDAL # Define pixel_size and NoData value of new raster pixel_size_x = p_size_x pixel_size_y = p_size_y NoData_value = -9999 # Filename of input OGR file vector_fn = classes_file # Filename of the raster Tiff that will be created raster_fn = output_path # Open the data source and read in the extent source_ds = ogr.Open(vector_fn) source_layer = layerCopy x_min, x_max, y_min, y_max = source_layer.GetExtent() # Create the destination data source x_res = outshape[0] y_res = outshape[1] target_ds = gdal.GetDriverByName('GTiff').Create(raster_fn, x_res, y_res, 1, gdal.GDT_Byte) target_ds.SetGeoTransform((x_min, pixel_size_x, 0, y_max, 0, pixel_size_y)) target_ds.SetProjection(layer.GetSpatialRef().ExportToWkt()) band = target_ds.GetRasterBand(1) band.SetNoDataValue(NoData_value) # Rasterize print("Rasterize") gdal.RasterizeLayer(target_ds, [1], source_layer, burn_values=[255], options=["ATTRIBUTE=FEATVAR"]) del file del source_ds
prune_factor = args.prune_factor scale_factor = args.scale_factor threshold = args.threshold URasterInfo = GeoImgInfo(args.Udata) VRasterInfo = GeoImgInfo(args.Vdata) URasterCount = URasterInfo.RasterCount VRasterCount = VRasterInfo.RasterCount assert URasterCount == VRasterCount RasterCount = URasterCount RasterInfo = URasterInfo Ufill_value = URasterInfo.nodatavalue Vfill_value = VRasterInfo.nodatavalue gdi = gdal.Info(args.Udata) driver = re.search("Driver:[ \t]*([^\n\r]*)", gdi).group(1) # It would be nice to only use gdal and not netcdf4python tdim = None if driver == "netCDF/Network Common Data Format": nc_file_u = args.Udata.split(":")[1] nc = NC(nc_file_u, "r") xdim, ydim, zdim, tdim = get_dims(nc) if tdim: time = nc.variables[tdim] time_units = time.units time_calendar = time.calendar cdftime = utime(time_units, time_calendar)
projection = #REPROJECT ALL RASTERS IF THIER PROJECTIONS DIFFERFROM THE SPECIFIED PROJECTION #get EPSG ds=gdal.Open(path) prj=ds.GetProjection() print prj srs=osr.SpatialReference(wkt=prj) if srs.IsProjected: print srs.GetAttrValue('projcs') print srs.GetAttrValue('geogcs') #reads the WKT representation of the file's spatial reference, then parses the string to extract the EPSG epsg = int(gdal.Info(input, format='json')['coordinateSystem']['wkt'].rsplit('"EPSG","', 1)[-1].split('"')[0]) #or #path = r"H:\workspace\TEUI\BridgerTeton" d = gdal.Open(path) proj = osr.SpatialReference(wkt=d.getProjection()) proj.AutoIdentifyEPSG() print(proj.GetAttrValue('AUTHORITY', 1)) #Reproject rasters def reproject_raster (raster, \pixel spacing= , epsg_from=, epsg_to= ): #IF RASTERS ARE DIFFERENT EXTENTS, CLIP TO SMALLEST EXTENT #determine raster extents
exec("cmap = {}.reversed()".format(cmap_name[:-2])) else: exec("cmap = {}".format(cmap_name)) cmap_name = cmap_name.replace('SCM.', '') plt.register_cmap(name=cmap_name, cmap=cmap, lut=n_color) elif cmap_name == 'insar': cdict = tools_lib.cmap_insar() plt.register_cmap(name='insar', data=cdict) cmap = plt.get_cmap(cmap_name, n_color) #%% Set color range ### Auto if cmin is None: cmin = gdal.Info(infile, computeMinMax=True, format="json")["bands"][0]['computedMin'] if cmax is None: cmax = gdal.Info(infile, computeMinMax=True, format="json")["bands"][0]['computedMax'] ### float -> int if int if np.mod(cmin, 1) == 0: cmin = int(cmin) if np.mod(cmax, 1) == 0: cmax = int(cmax) #%% Set file name if not outfile: outfile = infile.replace('.tif', '.{}_{}_{}.tif'.format(cmap_name, cmin, cmax)) #%% Create color table ### Format: value R G B alpha
#============================================================================== # CONTOUR HERE IS USED TO DEFINE THE BOUNDARIES OF THE ROS DATA #============================================================================== contourband = opentif.GetRasterBand(1) contourarray = contourband.ReadAsArray() contour = contourarray >= 0 #============================================================================== # SET PLOT EXTENT VARIABLES # FROM 'info' LOWER LEFT COORDINATES ARE EXTRACTED AND CONVERTED TO DECIMAL # DEGREES. THESE COORDINATES ARE USED TO CONVERT THE PLOT FROM ROWS/COLUMNS # TO LAT/LONG THERE WILL LIKELY BE DISTORTION IN THE LAT/LONG LABELS. #============================================================================== info = gdal.Info(opentif) longll = -168.20750000 latll = 53.34969444 cs = 0.048 extent = [longll, longll + shape[1] * cs, latll, latll + shape[0] * cs] #============================================================================== # PLOT THE ROS ARRAY #============================================================================== fig, ax = plt.subplots() cmap = plt.cm.YlOrRd #designate the color ramp to display ROS images outline = ax.contour(contour,extent, colors = 'k', origin='upper',\ linewidths = 0.075) sums = ax.imshow(array, extent=extent, cmap=cmap) ax.grid(linestyle='-.')
def lambda_handler(event, context): print(event) # establish some variables # prepare input event if 'sourceBucket' not in event.keys(): source_bucket = event['Records'][0]['s3']['bucket']['name'] source_key = event['Records'][0]['s3']['object']['key'] print('fired by event!') print(source_bucket, source_key) else: source_bucket = event['sourceBucket'] source_key = event['sourceKey'] # update ACL for uploaded /scanned tif # /scanned tif only exists for frames and indexes - NOT mosaics if '/frames/scanned/' in source_key or '/index/scanned/' in source_key: print('updating scanned/ tif ACL to public-read') print(source_key) client = boto3.client('s3') response = client.put_object_acl(ACL='public_read',Bucket=source_bucket,Key=source_key) print(response) gdal.VSICurlClearCache() return # verify input is a georef upload if 'georef/' not in source_key: print("error: key doesn't include '/georef/'. exiting...") print(source_key) return # verify not a temp tif in the georefSubDir elif georef_sub_dir in source_key: print("error: key includes the 'georefSubDir' env variable. exiting...") print(source_key) return # verify not a temp tif in epsg3857 directory elif epsg_sub_dir in source_key: print("error: key includes the 'epsgSubDir' env variable. exiting...") print(source_key) return else: dst_crs = 'EPSG:3857' prefix = '/vsis3/' + source_bucket + '/' s3_path = prefix + source_key epsg_folder = 'georef/' + epsg_sub_dir upload_key = source_key.replace('georef/', epsg_folder).replace('TIF', 'tif') gdal.VSICurlClearCache() try: epsg = int(gdal.Info(s3_path, format='json')['coordinateSystem']['wkt'].rsplit('"EPSG","', 1)[-1].split('"')[0]) print(epsg) except Exception as e: print(e) print("no epsg. uh oh... this raster is probably georeferenced but wasn't exported to GeoTiff") epsg = '' # connect to s3 client = boto3.client('s3') if epsg != 3857: print("let's reproject...") reprojected_tif = '/tmp/reprojected.tif' try: gdal.Warp(reprojected_tif,s3_path,dstSRS=dst_crs) except Exception as e: print(e) print('uploading reprojected tif') client.upload_file(reprojected_tif, source_bucket, upload_key) gdal.VSICurlClearCache() # cleanup /tmp print('cleaning up /tmp') os.remove(reprojected_tif) else: print("already EPSG 3857; copying to epsg folder.") client.copy({'Bucket': source_bucket, 'Key': source_key}, source_bucket, upload_key) print('invoking ls4-01-compress') client = boto3.client('lambda') payload = {'sourceBucket': source_bucket, 'sourceKey': upload_key} response = client.invoke( FunctionName='ls4-01-compress', InvocationType='Event', Payload=json.dumps(payload) ) print(response) # check if original exists in /scanned client = boto3.client('s3') # /scanned tif only exists for frames and indexes - NOT mosaics if '/frames/georef/' in source_key or '/index/georef/' in source_key: orig = source_key.replace('georef/', 'scanned/') # original index uploads don't include tiled number if '/index/' in orig: tile_num = orig.split('/')[-1].split('_')[-1] sfx = tile_num.split('.')[-1] orig = orig.replace('_' + tile_num, '.' + sfx) try: r = client.head_object(Bucket=source_bucket, Key=orig) print('original exists at %s' % orig) except Exception as e: print(e) # try alternative extension try: if '.tif' in orig: alt = orig.replace('.tif', '.TIF') elif '.TIF' in orig: alt = orig.replace('.TIF', '.tif') else: raise Exception('no tif or TIF extension!') r = client.head_object(Bucket=source_bucket, Key=alt) print('original exists at %s' % alt) except Exception as e: print(e) # publish message to the project SNS topic sns = boto3.resource('sns') arn = 'arn:aws:sns:us-east-1:%s:%s' % (aws_account, sns_error_topic) topic = sns.Topic(arn) m = ("GeoTiff uploaded at '%s' in the LS4 bucket is missing " "it's original in the relative /scanned folder. Expected " "it to exist at '%s'. Please upload it now while you're " "doing LS4y stuffs. Might as well... ya know? Thanks " "friend!" % (source_key, orig) ) response = topic.publish( Message=m, Subject='LS4 Notification' ) print('original /scanned not in s3. sns error message dispatched.') print("that's all folks!!")