def vsifile_4(): fp = gdal.VSIFOpenL('vsifile.py', 'rb') data = gdal.VSIFReadL(1000000, 1, fp) #print(len(data)) gdal.VSIFSeekL(fp, 0, 0) data = gdal.VSIFReadL(1, 1000000, fp) if len(data) == 0: return 'fail' return 'success'
def misc_10(): f = gdal.VSIFOpenL('/vsigzip/./data/byte.tif.gz', 'rb') gdal.VSIFReadL(1, 1, f) gdal.VSIFSeekL(f, 0, 2) gdal.VSIFSeekL(f, 0, 0) data = gdal.VSIFReadL(1, 4, f) gdal.VSIFCloseL(f) import struct ar = struct.unpack('B' * 4, data) if ar != (73, 73, 42, 0): return 'fail' return 'success'
def rfc30_2(): if version_info >= (3, 0, 0): filename = 'tmp/yy\u4E2D\u6587.\u4E2D\u6587' else: exec("filename = u'tmp/yy\u4E2D\u6587.\u4E2D\u6587'") # The typemaps should accept Unicode strings directly #filename = filename.encode( 'utf-8' ) fd = gdal.VSIFOpenL(filename, 'w') if fd is None: gdaltest.post_reason('failed to create utf-8 named file.') return 'failure' gdal.VSIFWriteL('abc', 3, 1, fd) gdal.VSIFCloseL(fd) # rename if version_info >= (3, 0, 0): new_filename = 'tmp/yy\u4E2D\u6587.\u4E2D\u6587' filename_for_rename = filename else: exec("new_filename = u'tmp/yy\u4E2D\u6587.\u4E2D\u6587'") filename_for_rename = filename.encode( 'utf-8') # FIXME ? rename should perhaps accept unicode strings new_filename = new_filename.encode( 'utf-8') # FIXME ? rename should perhaps accept unicode strings if gdal.Rename(filename_for_rename, new_filename) != 0: gdaltest.post_reason('utf-8 rename failed.') return 'failure' fd = gdal.VSIFOpenL(new_filename, 'r') if fd is None: gdaltest.post_reason('reopen failed with utf8') return 'failure' data = gdal.VSIFReadL(3, 1, fd) gdal.VSIFCloseL(fd) if version_info >= (3, 0, 0): ok = eval("data == b'abc'") else: ok = data == 'abc' if not ok: gdaltest.post_reason('did not get expected data.') return 'failure' gdal.Unlink(new_filename) fd = gdal.VSIFOpenL(new_filename, 'r') if fd is not None: gdaltest.post_reason('did unlink fail on utf8 filename?') return 'failure' return 'success'
def ogr_gml_32(): if not gdaltest.have_gml_reader: return 'skip' # Test without .xsd or .gfs f = gdal.VSIFOpenL("data/testfmegml.gml", "rb") data = gdal.VSIFReadL(1, 10000, f) gdal.VSIFCloseL(f) f = gdal.VSIFOpenL("/vsimem/ogr_gml_31.gml", "wb") gdal.VSIFWriteL(data, 1, len(data), f) gdal.VSIFCloseL(f) ds = ogr.Open('/vsimem/ogr_gml_31.gml') lyr = ds.GetLayer(1) feat = lyr.GetNextFeature() feat = lyr.GetNextFeature() if feat.GetFID() != 1: gdaltest.post_reason( 'did not get feature when reading directly second layer') return 'fail' ds = None f = gdal.VSIFOpenL("/vsimem/ogr_gml_31.gfs", "rb") data = gdal.VSIFReadL(1, 10000, f) gdal.VSIFCloseL(f) data = str(data) if data.find("<SequentialLayers>true</SequentialLayers>") == -1: gdaltest.post_reason( 'did not find <SequentialLayers>true</SequentialLayers> in .gfs') return 'fail' gdal.Unlink("/vsimem/ogr_gml_31.gml") gdal.Unlink("/vsimem/ogr_gml_31.gfs") return 'success'
def read_file(self, filename): """ Read an in-memory file https://gis.stackexchange.com/questions/255153/gdal-vectortranslate-returns-an-empty-file also see: https://gis.stackexchange.com/questions/318916/getting-png-binary-data-from-gdaldataset """ vsifile = gdal.VSIFOpenL(filename, 'r') gdal.VSIFSeekL(vsifile, 0, 2) vsileng = gdal.VSIFTellL(vsifile) gdal.VSIFSeekL(vsifile, 0, 0) return gdal.VSIFReadL(1, vsileng, vsifile)
def read_vsimem(self, fn): """Read GDAL vsimem files""" vsifile = None try: vsifile = gdal.VSIFOpenL(fn, 'r') gdal.VSIFSeekL(vsifile, 0, 2) vsileng = gdal.VSIFTellL(vsifile) gdal.VSIFSeekL(vsifile, 0, 0) return gdal.VSIFReadL(1, vsileng, vsifile) finally: if vsifile: gdal.VSIFCloseL(vsifile)
def ogr_kml_write_schema(): ds = ogr.GetDriverByName('KML').CreateDataSource( '/vsimem/ogr_kml_write_schema.kml') lyr = ds.CreateLayer("lyr") lyr.CreateField(ogr.FieldDefn('strfield', ogr.OFTString)) lyr.CreateField(ogr.FieldDefn('intfield', ogr.OFTInteger)) lyr.CreateField(ogr.FieldDefn('realfield', ogr.OFTReal)) feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField('strfield', 'strfield_val') feat.SetField('intfield', '1') feat.SetField('realfield', '2.34') lyr.CreateFeature(feat) ds = None f = gdal.VSIFOpenL('/vsimem/ogr_kml_write_schema.kml', 'rb') content = gdal.VSIFReadL(1, 1000, f) gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/ogr_kml_write_schema.kml') expected_content = """<?xml version="1.0" encoding="utf-8" ?> <kml xmlns="http://www.opengis.net/kml/2.2"> <Document><Folder><name>lyr</name> <Placemark> <ExtendedData><SchemaData schemaUrl="#lyr"> <SimpleData name="strfield">strfield_val</SimpleData> <SimpleData name="intfield">1</SimpleData> <SimpleData name="realfield">2.34</SimpleData> </SchemaData></ExtendedData> </Placemark> </Folder> <Schema name="lyr" id="lyr"> <SimpleField name="strfield" type="string"></SimpleField> <SimpleField name="intfield" type="int"></SimpleField> <SimpleField name="realfield" type="float"></SimpleField> </Schema> </Document></kml>""" content_lines = content.strip().split('\n') expected_lines = expected_content.strip().split('\n') if len(content_lines) != len(expected_lines): gdaltest.post_reason('fail') print(content) return 'fail' for i in range(len(content_lines)): if content_lines[i].strip() != expected_lines[i].strip(): gdaltest.post_reason('fail') print(content) return 'fail' return 'success'
def vsifile_generic(filename): fp = gdal.VSIFOpenL(filename, 'wb+') gdal.VSIFWriteL('0123456789', 1, 10, fp) gdal.VSIFTruncateL(fp, 5) if gdal.VSIFTellL(fp) != 10: gdaltest.post_reason('failure') return 'fail' gdal.VSIFSeekL(fp, 0, 2) if gdal.VSIFTellL(fp) != 5: gdaltest.post_reason('failure') return 'fail' gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 7: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' fp = gdal.VSIFOpenL(filename, 'rb') buf = gdal.VSIFReadL(1, 7, fp) gdal.VSIFCloseL(fp) if buf.decode('ascii') != '01234XX': gdaltest.post_reason('failure') return 'fail' gdal.Unlink(filename) statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG) if statBuf is not None: gdaltest.post_reason('failure') return 'fail' return 'success'
def ogr_ods_6(): drv = ogr.GetDriverByName('ODS') if drv is None: return 'skip' src_ds = ogr.Open('ODS:data/content_formulas.xml') out_ds = ogr.GetDriverByName('CSV').CopyDataSource( src_ds, '/vsimem/content_formulas.csv') out_ds = None src_ds = None fp = gdal.VSIFOpenL('/vsimem/content_formulas.csv', 'rb') res = gdal.VSIFReadL(1, 10000, fp) gdal.VSIFCloseL(fp) gdal.Unlink('/vsimem/content_formulas.csv') res = res.decode('ascii').split() expected_res = """Field1,Field2,Field3,Field4,Field5,Field6,Field7,Field8,Field9,Field10,Field11,Field12,Field13,Field14,Field15,Field16,Field17,Field18,Field19,Field20,Field21,Field22,Field23,Field24,Field25,Field26,Field27,Field28,Field29,Field30,Field31,Field32 of:=[.B1],of:=[.C1],of:=[.A1],,,,,,,,,,,,,,,,,,,,,,,,,,,,, 1,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,, ab,ab,ab,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 1,a,,3.5,MIN,1,MIN,3.5,SUM,4.5,AVERAGE,2.25,COUNT,2,COUNTA,3,,,,,,,,,,,,,,,, abcdef,6,,a,abcdef,,f,abcdef,"of:=MID([.A5];0;1)",,a,abcdef,,a,ef,ef,,,,,,,,,,,,,,,, 1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, AB,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 2,2,0,3,1,0,0,1,1,1,0,0,0,1,1,0,,,,,,,,,,,,,,,, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 """.split() if res != expected_res: gdaltest.post_reason('did not get expected result') print(res) return 'fail' return 'success'
def vsizip_1(): # We can keep the handle open during all the ZIP writing hZIP = gdal.VSIFOpenL("/vsizip/vsimem/test.zip", "wb") if hZIP is None: gdaltest.post_reason('fail 1') return 'fail' # One way to create a directory f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir2/", "wb") if f is None: gdaltest.post_reason('fail 2') return 'fail' gdal.VSIFCloseL(f) # A more natural one gdal.Mkdir("/vsizip/vsimem/test.zip/subdir1", 0) # Create 1st file f2 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "wb") if f2 is None: gdaltest.post_reason('fail 3') return 'fail' gdal.VSIFWriteL("abcd", 1, 4, f2) gdal.VSIFCloseL(f2) # Test that we cannot read a zip file being written gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb") gdal.PopErrorHandler() if gdal.GetLastErrorMsg() != 'Cannot read a zip file being written': gdaltest.post_reason('expected error') print(gdal.GetLastErrorMsg()) return 'fail' if f is not None: gdaltest.post_reason('should not have been successful 1') return 'fail' # Create 2nd file f3 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/efghi", "wb") if f3 is None: gdaltest.post_reason('fail 4') return 'fail' gdal.VSIFWriteL("efghi", 1, 5, f3) # Try creating a 3d file gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') f4 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/that_wont_work", "wb") gdal.PopErrorHandler() if gdal.GetLastErrorMsg() != 'Cannot create that_wont_work while another file is being written in the .zip': gdaltest.post_reason('expected error') print(gdal.GetLastErrorMsg()) return 'fail' if f4 is not None: gdaltest.post_reason('should not have been successful 2') return 'fail' gdal.VSIFCloseL(f3) # Now we can close the main handle gdal.VSIFCloseL(hZIP) f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb") if f is None: gdaltest.post_reason('fail 5') return 'fail' data = gdal.VSIFReadL(1, 4, f) gdal.VSIFCloseL(f) gdal.Unlink("/vsimem/test.zip") if data.decode('ASCII') != 'abcd': print(data) return 'fail' return 'success'
def vsizip_2(): fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/foo.bar", "wb") if fmain is None: gdaltest.post_reason('fail 1') return 'fail' gdal.VSIFWriteL("12345", 1, 5, fmain) gdal.VSIFCloseL(fmain) content = gdal.ReadDir("/vsizip/vsimem/test2.zip") if content != ['foo.bar']: gdaltest.post_reason('bad content 1') print(content) return 'fail' # Now append a second file fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/bar.baz", "wb") if fmain is None: gdaltest.post_reason('fail 2') return 'fail' gdal.VSIFWriteL("67890", 1, 5, fmain) gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') content = gdal.ReadDir("/vsizip/vsimem/test2.zip") gdal.PopErrorHandler() if gdal.GetLastErrorMsg() != 'Cannot read a zip file being written': gdaltest.post_reason('expected error') print(gdal.GetLastErrorMsg()) return 'fail' if content != None: gdaltest.post_reason('bad content 2') print(content) return 'fail' gdal.VSIFCloseL(fmain) content = gdal.ReadDir("/vsizip/vsimem/test2.zip") if content != ['foo.bar', 'bar.baz']: gdaltest.post_reason('bad content 3') print(content) return 'fail' fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/foo.bar", "rb") if fmain is None: gdaltest.post_reason('fail 3') return 'fail' data = gdal.VSIFReadL(1, 5, fmain) gdal.VSIFCloseL(fmain) if data.decode('ASCII') != '12345': print(data) return 'fail' fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/bar.baz", "rb") if fmain is None: gdaltest.post_reason('fail 4') return 'fail' data = gdal.VSIFReadL(1, 5, fmain) gdal.VSIFCloseL(fmain) if data.decode('ASCII') != '67890': print(data) return 'fail' gdal.Unlink("/vsimem/test2.zip") return 'success'
def vsifile_5(): fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'wb') ref_data = ''.join(['%08X' % i for i in range(5 * 32768)]) gdal.VSIFWriteL(ref_data, 1, len(ref_data), fp) gdal.VSIFCloseL(fp) gdal.SetConfigOption('VSI_CACHE', 'YES') for i in range(3): if i == 0: gdal.SetConfigOption('VSI_CACHE_SIZE', '0') elif i == 1: gdal.SetConfigOption('VSI_CACHE_SIZE', '65536') else: gdal.SetConfigOption('VSI_CACHE_SIZE', None) fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'rb') gdal.VSIFSeekL(fp, 50000, 0) if gdal.VSIFTellL(fp) != 50000: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFSeekL(fp, 50000, 1) if gdal.VSIFTellL(fp) != 100000: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFSeekL(fp, 0, 2) if gdal.VSIFTellL(fp) != 5 * 32768 * 8: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFReadL(1, 1, fp) gdal.VSIFSeekL(fp, 0, 0) data = gdal.VSIFReadL(1, 3 * 32768, fp) if data.decode('ascii') != ref_data[0:3 * 32768]: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFSeekL(fp, 16384, 0) data = gdal.VSIFReadL(1, 5 * 32768, fp) if data.decode('ascii') != ref_data[16384:16384 + 5 * 32768]: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' data = gdal.VSIFReadL(1, 50 * 32768, fp) if data[0:1130496].decode('ascii') != ref_data[16384 + 5 * 32768:]: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFCloseL(fp) gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) gdal.Unlink('tmp/vsifile_5.bin') return 'success'
def full_check_band( # noqa pylint: disable=too-many-branches f, band_name, band, errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, mask_interleaved_with_imagery, ): block_size = band.GetBlockSize() mask_band = None if mask_interleaved_with_imagery: mask_band = band.GetMaskBand() mask_block_size = mask_band.GetBlockSize() if block_size != mask_block_size: errors += [ band_name + ": mask block size is different from its imagery band" ] mask_band = None yblocks = (band.YSize + block_size[1] - 1) // block_size[1] xblocks = (band.XSize + block_size[0] - 1) // block_size[0] last_offset = 0 for y in range(yblocks): # noqa pylint: disable=too-many-nested-blocks for x in range(xblocks): offset = band.GetMetadataItem("BLOCK_OFFSET_%d_%d" % (x, y), "TIFF") offset = int(offset) if offset is not None else 0 bytecount = band.GetMetadataItem("BLOCK_SIZE_%d_%d" % (x, y), "TIFF") bytecount = int(bytecount) if bytecount is not None else 0 if offset > 0: if block_order_row_major and offset < last_offset: errors += [ band_name + ": offset of block (%d, %d) is smaller than previous block" % (x, y) ] if block_leader_size_as_uint4: gdal.VSIFSeekL(f, offset - 4, 0) leader_size = struct.unpack("<I", gdal.VSIFReadL(4, 1, f))[0] if leader_size != bytecount: errors += [ band_name + ": for block (%d, %d), size in leader bytes is %d instead of %d" % (x, y, leader_size, bytecount) ] if block_trailer_last_4_bytes_repeated: if bytecount >= 4: gdal.VSIFSeekL(f, offset + bytecount - 4, 0) last_bytes = gdal.VSIFReadL(8, 1, f) if last_bytes[0:4] != last_bytes[4:8]: errors += [ band_name + ": for block (%d, %d), trailer bytes are invalid" % (x, y) ] if mask_band: offset_mask = mask_band.GetMetadataItem( "BLOCK_OFFSET_%d_%d" % (x, y), "TIFF") offset_mask = int( offset_mask) if offset_mask is not None else 0 if offset > 0 and offset_mask > 0: # bytecount_mask = int(mask_band.GetMetadataItem('BLOCK_SIZE_%d_%d' % (x,y), 'TIFF')) expected_offset_mask = ( offset + bytecount + (4 if block_leader_size_as_uint4 else 0) + (4 if block_trailer_last_4_bytes_repeated else 0)) if offset_mask != expected_offset_mask: errors += [ "Mask of " + band_name + ": for block (%d, %d), offset is %d, whereas %d was expected" % (x, y, offset_mask, expected_offset_mask) ] elif offset == 0 and offset_mask > 0: if block_order_row_major and offset_mask < last_offset: errors += [ "Mask of " + band_name + ": offset of block (%d, %d) is smaller than previous block" % (x, y) ] offset = offset_mask last_offset = offset
def validate( # noqa pylint: disable=too-many-branches ds, check_tiled=True, full_check=False): """Check if a file is a (Geo)TIFF with cloud optimized compatible structure. Args: ds: GDAL Dataset for the file to inspect. check_tiled: Set to False to ignore missing tiling. full_check: Set to TRUe to check tile/strip leader/trailer bytes. Might be slow on remote files Returns: A tuple, whose first element is an array of error messages (empty if there is no error), and the second element, a dictionary with the structure of the GeoTIFF file. Raises: ValidateCloudOptimizedGeoTIFFException: Unable to open the file or the file is not a Tiff. """ if int(gdal.VersionInfo("VERSION_NUM")) < 2020000: raise ValidateCloudOptimizedGeoTIFFException( "GDAL 2.2 or above required") unicode_type = type("".encode("utf-8").decode("utf-8")) if isinstance(ds, (str, unicode_type)): gdal.PushErrorHandler() ds = gdal.Open(ds) gdal.PopErrorHandler() if ds is None: raise ValidateCloudOptimizedGeoTIFFException( "Invalid file : %s" % gdal.GetLastErrorMsg()) if ds.GetDriver().ShortName != "GTiff": raise ValidateCloudOptimizedGeoTIFFException( "The file is not a GeoTIFF") details = {} errors = [] warnings = [] filename = ds.GetDescription() main_band = ds.GetRasterBand(1) ovr_count = main_band.GetOverviewCount() filelist = ds.GetFileList() if filelist is not None and filename + ".ovr" in filelist: errors += [ "Overviews found in external .ovr file. They should be internal" ] if main_band.XSize > 512 or main_band.YSize > 512: if check_tiled: block_size = main_band.GetBlockSize() if block_size[0] == main_band.XSize and block_size[0] > 1024: errors += [ "The file is greater than 512xH or Wx512, but is not tiled" ] if ovr_count == 0: warnings += [ "The file is greater than 512xH or Wx512, it is recommended " "to include internal overviews" ] ifd_offset = int(main_band.GetMetadataItem("IFD_OFFSET", "TIFF")) ifd_offsets = [ifd_offset] block_order_row_major = False block_leader_size_as_uint4 = False block_trailer_last_4_bytes_repeated = False mask_interleaved_with_imagery = False if ifd_offset not in (8, 16): # Check if there is GDAL hidden structural metadata f = gdal.VSIFOpenL(filename, "rb") if not f: raise ValidateCloudOptimizedGeoTIFFException("Cannot open file") signature = struct.unpack("B" * 4, gdal.VSIFReadL(4, 1, f)) bigtiff = signature in ((0x49, 0x49, 0x2B, 0x00), (0x4D, 0x4D, 0x00, 0x2B)) if bigtiff: expected_ifd_pos = 16 else: expected_ifd_pos = 8 gdal.VSIFSeekL(f, expected_ifd_pos, 0) pattern = "GDAL_STRUCTURAL_METADATA_SIZE=%06d bytes\n" % 0 got = gdal.VSIFReadL(len(pattern), 1, f).decode("LATIN1") if len(got) == len(pattern) and got.startswith( "GDAL_STRUCTURAL_METADATA_SIZE="): size = int(got[len("GDAL_STRUCTURAL_METADATA_SIZE="):][0:6]) extra_md = gdal.VSIFReadL(size, 1, f).decode("LATIN1") block_order_row_major = "BLOCK_ORDER=ROW_MAJOR" in extra_md block_leader_size_as_uint4 = "BLOCK_LEADER=SIZE_AS_UINT4" in extra_md block_trailer_last_4_bytes_repeated = ( "BLOCK_TRAILER=LAST_4_BYTES_REPEATED" in extra_md) mask_interleaved_with_imagery = ( "MASK_INTERLEAVED_WITH_IMAGERY=YES" in extra_md) if "KNOWN_INCOMPATIBLE_EDITION=YES" in extra_md: errors += [ "KNOWN_INCOMPATIBLE_EDITION=YES is declared in the file" ] expected_ifd_pos += len(pattern) + size expected_ifd_pos += (expected_ifd_pos % 2 ) # IFD offset starts on a 2-byte boundary gdal.VSIFCloseL(f) if expected_ifd_pos != ifd_offsets[0]: errors += [ "The offset of the main IFD should be %d. It is %d instead" % (expected_ifd_pos, ifd_offsets[0]) ] details["ifd_offsets"] = {} details["ifd_offsets"]["main"] = ifd_offset for i in range(ovr_count): # Check that overviews are by descending sizes ovr_band = ds.GetRasterBand(1).GetOverview(i) if i == 0: if ovr_band.XSize > main_band.XSize or ovr_band.YSize > main_band.YSize: errors += [ "First overview has larger dimension than main band" ] else: prev_ovr_band = ds.GetRasterBand(1).GetOverview(i - 1) if (ovr_band.XSize > prev_ovr_band.XSize or ovr_band.YSize > prev_ovr_band.YSize): errors += [ "Overview of index %d has larger dimension than " "overview of index %d" % (i, i - 1) ] if check_tiled: block_size = ovr_band.GetBlockSize() if block_size[0] == ovr_band.XSize and block_size[0] > 1024: errors += ["Overview of index %d is not tiled" % i] # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int(ovr_band.GetMetadataItem("IFD_OFFSET", "TIFF")) ifd_offsets.append(ifd_offset) details["ifd_offsets"]["overview_%d" % i] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if i == 0: errors += [ "The offset of the IFD for overview of index %d is %d, " "whereas it should be greater than the one of the main " "image, which is at byte %d" % (i, ifd_offsets[-1], ifd_offsets[-2]) ] else: errors += [ "The offset of the IFD for overview of index %d is %d, " "whereas it should be greater than the one of index %d, " "which is at byte %d" % (i, ifd_offsets[-1], i - 1, ifd_offsets[-2]) ] # Check that the imagery starts by the smallest overview and ends with # the main resolution dataset def get_block_offset(band): blockxsize, blockysize = band.GetBlockSize() for y in range(int((band.YSize + blockysize - 1) / blockysize)): for x in range(int((band.XSize + blockxsize - 1) / blockxsize)): block_offset = band.GetMetadataItem( "BLOCK_OFFSET_%d_%d" % (x, y), "TIFF") if block_offset: return int(block_offset) return 0 block_offset = get_block_offset(main_band) data_offsets = [block_offset] details["data_offsets"] = {} details["data_offsets"]["main"] = block_offset for i in range(ovr_count): ovr_band = ds.GetRasterBand(1).GetOverview(i) block_offset = get_block_offset(ovr_band) data_offsets.append(block_offset) details["data_offsets"]["overview_%d" % i] = block_offset if data_offsets[-1] != 0 and data_offsets[-1] < ifd_offsets[-1]: if ovr_count > 0: errors += [ "The offset of the first block of the smallest overview " "should be after its IFD" ] else: errors += [ "The offset of the first block of the image should be after its IFD" ] for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] != 0 and data_offsets[i] < data_offsets[i + 1]: errors += [ "The offset of the first block of overview of index %d should " "be after the one of the overview of index %d" % (i - 1, i) ] if (len(data_offsets) >= 2 and data_offsets[0] != 0 and data_offsets[0] < data_offsets[1]): errors += [ "The offset of the first block of the main resolution image " "should be after the one of the overview of index %d" % (ovr_count - 1) ] if full_check and (block_order_row_major or block_leader_size_as_uint4 or block_trailer_last_4_bytes_repeated or mask_interleaved_with_imagery): f = gdal.VSIFOpenL(filename, "rb") if not f: raise ValidateCloudOptimizedGeoTIFFException("Cannot open file") full_check_band( f, "Main resolution image", main_band, errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, mask_interleaved_with_imagery, ) if (main_band.GetMaskFlags() == gdal.GMF_PER_DATASET and (filename + ".msk") not in ds.GetFileList()): full_check_band( f, "Mask band of main resolution image", main_band.GetMaskBand(), errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, False, ) for i in range(ovr_count): ovr_band = ds.GetRasterBand(1).GetOverview(i) full_check_band( f, "Overview %d" % i, ovr_band, errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, mask_interleaved_with_imagery, ) if (ovr_band.GetMaskFlags() == gdal.GMF_PER_DATASET and (filename + ".msk") not in ds.GetFileList()): full_check_band( f, "Mask band of overview %d" % i, ovr_band.GetMaskBand(), errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, False, ) gdal.VSIFCloseL(f) return warnings, errors, details
def jpeg_19(): import struct for (width, height, iX) in [(32, 32, 12), (25, 25, 8), (24, 25, 8)]: src_ds = gdal.GetDriverByName('GTiff').Create('/vsimem/jpeg_19.tif', width, height, 1) src_ds.CreateMaskBand(gdal.GMF_PER_DATASET) src_ds.GetRasterBand(1).GetMaskBand().WriteRaster( 0, 0, iX, height, struct.pack('B' * 1, 255), 1, 1) src_ds.GetRasterBand(1).GetMaskBand().WriteRaster( iX, 0, width - iX, height, struct.pack('B' * 1, 0), 1, 1) tiff_mask_data = src_ds.GetRasterBand(1).GetMaskBand().ReadRaster( 0, 0, width, height) # Generate a JPEG file with a (default) LSB bit mask order out_ds = gdal.GetDriverByName('JPEG').CreateCopy( '/vsimem/jpeg_19.jpg', src_ds) out_ds = None # Generate a JPEG file with a MSB bit mask order gdal.SetConfigOption('JPEG_WRITE_MASK_BIT_ORDER', 'MSB') out_ds = gdal.GetDriverByName('JPEG').CreateCopy( '/vsimem/jpeg_19_msb.jpg', src_ds) out_ds = None gdal.SetConfigOption('JPEG_WRITE_MASK_BIT_ORDER', None) src_ds = None # Check that the file are indeed different statBuf = gdal.VSIStatL('/vsimem/jpeg_19.jpg') f = gdal.VSIFOpenL('/vsimem/jpeg_19.jpg', 'rb') data1 = gdal.VSIFReadL(1, statBuf.size, f) gdal.VSIFCloseL(f) statBuf = gdal.VSIStatL('/vsimem/jpeg_19_msb.jpg') f = gdal.VSIFOpenL('/vsimem/jpeg_19_msb.jpg', 'rb') data2 = gdal.VSIFReadL(1, statBuf.size, f) gdal.VSIFCloseL(f) if (width, height, iX) == (24, 25, 8): if data1 != data2: gdaltest.post_reason('fail') return 'fail' else: if data1 == data2: gdaltest.post_reason('fail') return 'fail' # Check the file with the LSB bit mask order ds = gdal.Open('/vsimem/jpeg_19.jpg') jpg_mask_data = ds.GetRasterBand(1).GetMaskBand().ReadRaster( 0, 0, width, height) ds = None if tiff_mask_data != jpg_mask_data: gdaltest.post_reason('fail') return 'fail' # Check the file with the MSB bit mask order ds = gdal.Open('/vsimem/jpeg_19_msb.jpg') jpg_mask_data = ds.GetRasterBand(1).GetMaskBand().ReadRaster( 0, 0, width, height) ds = None if tiff_mask_data != jpg_mask_data: gdaltest.post_reason('fail') return 'fail' gdal.GetDriverByName('GTiff').Delete('/vsimem/jpeg_19.tif') gdal.GetDriverByName('JPEG').Delete('/vsimem/jpeg_19.jpg') gdal.GetDriverByName('JPEG').Delete('/vsimem/jpeg_19_msb.jpg') return 'success'