def test_dissolve_with_lists(): assert anc.dissolve([[1, 2], [3, 4]]) == [1, 2, 3, 4] assert anc.dissolve([[[1]]]) == [1] assert anc.dissolve((( 1, 2, ), (3, 4))) == [1, 2, 3, 4] assert anc.dissolve(((1, 2), (1, 2))) == [1, 2, 1, 2]
def getAuxdata(datasets, scenes): auxDataPath = os.path.join(expanduser("~"), '.snap/auxdata') scenes = [identify(scene) if isinstance(scene, str) else scene for scene in scenes] sensors = list(set([scene.sensor for scene in scenes])) for dataset in datasets: if dataset == 'SRTM 1Sec HGT': files = [x.replace('hgt', 'SRTMGL1.hgt.zip') for x in list(set(dissolve([scene.getHGT() for scene in scenes])))] for file in files: infile = os.path.join('http://step.esa.int/auxdata/dem/SRTMGL1', file) outfile = os.path.join(auxDataPath, 'dem/SRTM 1Sec HGT', file) if not os.path.isfile(outfile): print(infile) try: input = urlopen(infile) except HTTPError: print('-> not available') continue with open(outfile, 'wb') as output: output.write(input.read()) input.close() elif dataset == 'POEORB': for sensor in sensors: if re.search('S1[AB]', sensor): dates = [(scene.start[:4], scene.start[4:6]) for scene in scenes] years = list(set([x[0] for x in dates])) remote_contentVersion = urlopen( 'http://step.esa.int/auxdata/orbits/Sentinel-1/POEORB/remote_contentVersion.txt') versions_remote = getOrbitContentVersions(remote_contentVersion) for year in years: dir_orb = os.path.join(auxDataPath, 'Orbits/Sentinel-1/POEORB', year) if not os.path.isdir(dir_orb): os.makedirs(dir_orb) contentVersionFile = os.path.join(dir_orb, 'contentVersion.txt') if os.path.isfile(contentVersionFile): contentVersion = open(contentVersionFile, 'r+') versions_local = getOrbitContentVersions(contentVersion) else: contentVersion = open(contentVersionFile, 'w') versions_local = {} combine = dict(set(versions_local.items()) & set(versions_remote.items())) dates_select = [x for x in dates if x[0] == year] months = list(set([x[1] for x in dates_select])) orb_ids = sorted( [x for x in ['{}-{}.zip'.format(year, month) for month in months] if not x in combine]) if len(orb_ids) > 0: contentVersion.write('#\n#{}\n'.format(strftime('%a %b %d %H:%M:%S %Z %Y', gmtime()))) for orb_id in orb_ids: orb_remote = urlopen( 'http://step.esa.int/auxdata/orbits/Sentinel-1/POEORB/{}'.format(orb_id)) orb_remote_stream = zf.ZipFile(StringIO(orb_remote.read()), 'r') orb_remote.close() targets = [x for x in orb_remote_stream.namelist() if not os.path.isfile(os.path.join(dir_orb, x))] orb_remote_stream.extractall(dir_orb, targets) orb_remote_stream.close() versions_local[orb_id] = versions_remote[orb_id] for key, val in versions_local.iteritems(): contentVersion.write('{}={}\n'.format(key, val)) contentVersion.close() remote_contentVersion.close() else: print('not implemented yet') elif dataset == 'Delft Precise Orbits': path_server = 'dutlru2.lr.tudelft.nl' subdirs = {'ASAR:': 'ODR.ENVISAT1/eigen-cg03c', 'ERS1': 'ODR.ERS-1/dgm-e04', 'ERS2': 'ODR.ERS-2/dgm-e04'} ftp = FTP(path_server) ftp.login() for sensor in sensors: if sensor in subdirs.keys(): path_target = os.path.join('pub/orbits', subdirs[sensor]) path_local = os.path.join(auxDataPath, 'Orbits/Delft Precise Orbits', subdirs[sensor]) ftp.cwd(path_target) for item in ftp.nlst(): ftp.retrbinary('RETR ' + item, open(os.path.join(path_local, item), 'wb').write) ftp.quit() else: print('not implemented yet')
def test_dissolve_3_list_recursion(self): self.assertEqual(anc.dissolve([[[1]]]), [1])
def test_dissolve_with_lists(self): self.assertEqual(anc.dissolve([[1, 2], [3, 4]]), [1, 2, 3, 4])
def test_dissolve_tuple_recursion(self): self.assertEqual(anc.dissolve(((1, 2), (1, 2))), [1, 2, 1, 2])
def test_dissolve_with_tuples(self): self.assertEqual(anc.dissolve((( 1, 2, ), (3, 4))), [1, 2, 3, 4])
def stack(srcfiles, dstfile, resampling, targetres, srcnodata, dstnodata, shapefile=None, layernames=None, sortfun=None, separate=False, overwrite=False, compress=True, cores=4): """ function for mosaicking, resampling and stacking of multiple raster files into a 3D data cube Parameters ---------- srcfiles: list a list of file names or a list of lists; each sub-list is treated as an order to mosaic its containing files dstfile: str the destination file (if sesparate) or a directory resampling: {near, bilinear, cubic, cubicspline, lanczos, average, mode, max, min, med, Q1, Q3} the resampling method; see documentation of gdalwarp targetres: tuple a list with two entries for x and y spatial resolution srcnodata: int or float the nodata value of the source files dstnodata: int or float the nodata value of the destination file(s) shapefile: str or spatial.vector.Vector a shapefile for defining the area of the destination files layernames: list the names of the output layers; if None, the basenames of the input files are used sortfun: function a function for sorting the input files; this is needed for defining the mosaicking order separate: bool should the files be written to a single raster block or separate files? If separate, each tile is written to geotiff. overwrite: bool overwrite the file if it already exists? compress: bool compress the geotiff files? cores: int the number of CPU threads to use; this is only relevant if separate = True Returns ------- """ if len(dissolve(srcfiles)) == 0: raise IOError('no input files provided to function raster.stack') if layernames is not None: if len(layernames) != len(srcfiles): raise IOError( 'mismatch between number of source file groups and layernames') if not isinstance(targetres, (list, tuple)) or len(targetres) != 2: raise RuntimeError( 'targetres must be a list or tuple with two entries for x and y resolution' ) if len(srcfiles) == 1 and not isinstance(srcfiles[0], list): raise IOError('only one file specified; nothing to be done') if resampling not in [ 'near', 'bilinear', 'cubic', 'cubicspline', 'lanczos', 'average', 'mode', 'max', 'min', 'med', 'Q1', 'Q3' ]: raise IOError('resampling method not supported') projections = list() for x in dissolve(srcfiles): try: projection = Raster(x).projection except OSError as e: print('cannot read file: {}'.format(x)) raise e projections.append(projection) projections = list(set(projections)) if len(projections) > 1: raise IOError('raster projection mismatch') elif len(projections) == 0: raise RuntimeError( 'could not retrieve the projection from any of the {} input images' .format(len(srcfiles))) else: srs = projections[0] # read shapefile bounding coordinates and reduce list of rasters to those overlapping with the shapefile if shapefile is not None: shp = shapefile if isinstance(shapefile, Vector) else Vector(shapefile) shp.reproject(srs) ext = shp.extent arg_ext = (ext['xmin'], ext['ymin'], ext['xmax'], ext['ymax']) for i in range(len(srcfiles)): group = sorted(srcfiles[i], key=sortfun) if isinstance( srcfiles[i], list) else [srcfiles[i]] group = [x for x in group if intersect(shp, Raster(x).bbox())] if len(group) > 1: srcfiles[i] = group elif len(group) == 1: srcfiles[i] = group[0] else: srcfiles[i] = None srcfiles = filter(None, srcfiles) else: arg_ext = None # create temporary directory for writing intermediate files dst_base = os.path.splitext(dstfile)[0] tmpdir = dst_base + '__tmp' if not os.path.isdir(tmpdir): os.makedirs(tmpdir) options_warp = { 'options': ['-q'], 'format': 'GTiff' if separate else 'ENVI', 'outputBounds': arg_ext, 'multithread': True, 'srcNodata': srcnodata, 'dstNodata': dstnodata, 'xRes': targetres[0], 'yRes': targetres[1], 'resampleAlg': resampling } if overwrite: options_warp['options'] += ['-overwrite'] if separate and compress: options_warp['options'] += [ '-co', 'COMPRESS=DEFLATE', '-co', 'PREDICTOR=2' ] options_buildvrt = {'outputBounds': arg_ext, 'srcNodata': srcnodata} # create VRT files for mosaicing for i in range(len(srcfiles)): base = srcfiles[i][0] if isinstance(srcfiles[i], list) else srcfiles[i] vrt = os.path.join( tmpdir, os.path.splitext(os.path.basename(base))[0] + '.vrt') gdalbuildvrt(srcfiles[i], vrt, options_buildvrt) srcfiles[i] = vrt # if no specific layernames are defined and sortfun is not set to None, # sort files by custom function or, by default, the basename of the raster/VRT file if layernames is None and sortfun is not None: srcfiles = sorted(srcfiles, key=sortfun if sortfun else os.path.basename) bandnames = [os.path.splitext(os.path.basename(x))[0] for x in srcfiles] if layernames is None else layernames if separate or len(srcfiles) == 1: if not os.path.isdir(dstfile): os.makedirs(dstfile) dstfiles = [os.path.join(dstfile, x) + '.tif' for x in bandnames] if overwrite: files = [x for x in zip(srcfiles, dstfiles)] else: files = [ x for x in zip(srcfiles, dstfiles) if not os.path.isfile(x[1]) ] if len(files) == 0: print( 'all target tiff files already exist, nothing to be done') shutil.rmtree(tmpdir) return srcfiles, dstfiles = map(list, zip(*files)) multicore(gdalwarp, cores=cores, multiargs={ 'src': srcfiles, 'dst': dstfiles }, options=options_warp) else: # create VRT for stacking vrt = os.path.join(tmpdir, os.path.basename(dst_base) + '.vrt') options_buildvrt['options'] = ['-separate'] gdalbuildvrt(srcfiles, vrt, options_buildvrt) # warp files gdalwarp(vrt, dstfile, options_warp) # edit ENVI HDR files to contain specific layer names par = envi.HDRobject(dstfile + '.hdr') par.band_names = bandnames envi.hdr(par) # remove temporary directory and files shutil.rmtree(tmpdir)