def read_file(self, filename):
"""
Read an in-memory file
https://gis.stackexchange.com/questions/255153/gdal-vectortranslate-returns-an-empty-file
also see:
https://gis.stackexchange.com/questions/318916/getting-png-binary-data-from-gdaldataset
"""
vsifile = gdal.VSIFOpenL(filename, 'r')
gdal.VSIFSeekL(vsifile, 0, 2)
vsileng = gdal.VSIFTellL(vsifile)
gdal.VSIFSeekL(vsifile, 0, 0)
return gdal.VSIFReadL(1, vsileng, vsifile)
def read_vsimem(self, fn):
"""Read GDAL vsimem files"""
vsifile = None
try:
vsifile = gdal.VSIFOpenL(fn, 'r')
gdal.VSIFSeekL(vsifile, 0, 2)
vsileng = gdal.VSIFTellL(vsifile)
gdal.VSIFSeekL(vsifile, 0, 0)
return gdal.VSIFReadL(1, vsileng, vsifile)
finally:
if vsifile:
gdal.VSIFCloseL(vsifile)
def misc_10():
f = gdal.VSIFOpenL('/vsigzip/./data/byte.tif.gz', 'rb')
gdal.VSIFReadL(1, 1, f)
gdal.VSIFSeekL(f, 0, 2)
gdal.VSIFSeekL(f, 0, 0)
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
import struct
ar = struct.unpack('B' * 4, data)
if ar != (73, 73, 42, 0):
return 'fail'
return 'success'
def ace2_1():
f = gdal.VSIFOpenL('/vsimem/45N015E_5M.ACE2', 'wb')
gdal.VSIFSeekL(f, 180 * 180 * 4 - 1, 0)
gdal.VSIFWriteL('\0', 1, 1, f)
gdal.VSIFCloseL(f)
tst = gdaltest.GDALTest('ACE2',
'/vsimem/45N015E_5M.ACE2',
1,
0,
filename_absolute=1)
expected_gt = [
15.0, 0.08333333333333333, 0.0, 60.0, 0.0, -0.08333333333333333
]
expected_srs = """GEOGCS["WGS 84",
DATUM["WGS_1984",
SPHEROID["WGS 84",6378137,298.257223563,
AUTHORITY["EPSG","7030"]],
TOWGS84[0,0,0,0,0,0,0],
AUTHORITY["EPSG","6326"]],
PRIMEM["Greenwich",0,
AUTHORITY["EPSG","8901"]],
UNIT["degree",0.0174532925199433,
AUTHORITY["EPSG","9108"]],
AUTHORITY["EPSG","4326"]]"""
ret = tst.testOpen(check_gt=expected_gt, check_prj=expected_srs)
gdal.Unlink('/vsimem/45N015E_5M.ACE2')
return ret
def vsifile_4():
fp = gdal.VSIFOpenL('vsifile.py', 'rb')
data = gdal.VSIFReadL(1000000, 1, fp)
#print(len(data))
gdal.VSIFSeekL(fp, 0, 0)
data = gdal.VSIFReadL(1, 1000000, fp)
if len(data) == 0:
return 'fail'
return 'success'
def vsifile_3():
if (gdaltest.filesystem_supports_sparse_files('tmp') == False):
return 'skip'
filename = 'tmp/vsifile_3'
fp = gdal.VSIFOpenL(filename, 'wb+')
gdal.VSIFTruncateL(fp, 10 * 1024 * 1024 * 1024)
gdal.VSIFSeekL(fp, 0, 2)
pos = gdal.VSIFTellL(fp)
if pos != 10 * 1024 * 1024 * 1024:
gdaltest.post_reason('failure')
gdal.VSIFCloseL(fp)
gdal.Unlink(filename)
print(pos)
return 'fail'
gdal.VSIFSeekL(fp, 0, 0)
gdal.VSIFSeekL(fp, pos, 0)
pos = gdal.VSIFTellL(fp)
if pos != 10 * 1024 * 1024 * 1024:
gdaltest.post_reason('failure')
gdal.VSIFCloseL(fp)
gdal.Unlink(filename)
print(pos)
return 'fail'
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
gdal.Unlink(filename)
if statBuf.size != 10 * 1024 * 1024 * 1024:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
return 'success'
def vsifile_generic(filename):
fp = gdal.VSIFOpenL(filename, 'wb+')
gdal.VSIFWriteL('0123456789', 1, 10, fp)
gdal.VSIFTruncateL(fp, 5)
if gdal.VSIFTellL(fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
gdal.VSIFSeekL(fp, 0, 2)
if gdal.VSIFTellL(fp) != 5:
gdaltest.post_reason('failure')
return 'fail'
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 7:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
fp = gdal.VSIFOpenL(filename, 'rb')
buf = gdal.VSIFReadL(1, 7, fp)
gdal.VSIFCloseL(fp)
if buf.decode('ascii') != '01234XX':
gdaltest.post_reason('failure')
return 'fail'
gdal.Unlink(filename)
statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG)
if statBuf is not None:
gdaltest.post_reason('failure')
return 'fail'
return 'success'
def vsifile_5():
fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'wb')
ref_data = ''.join(['%08X' % i for i in range(5 * 32768)])
gdal.VSIFWriteL(ref_data, 1, len(ref_data), fp)
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSI_CACHE', 'YES')
for i in range(3):
if i == 0:
gdal.SetConfigOption('VSI_CACHE_SIZE', '0')
elif i == 1:
gdal.SetConfigOption('VSI_CACHE_SIZE', '65536')
else:
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'rb')
gdal.VSIFSeekL(fp, 50000, 0)
if gdal.VSIFTellL(fp) != 50000:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 50000, 1)
if gdal.VSIFTellL(fp) != 100000:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 0, 2)
if gdal.VSIFTellL(fp) != 5 * 32768 * 8:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFReadL(1, 1, fp)
gdal.VSIFSeekL(fp, 0, 0)
data = gdal.VSIFReadL(1, 3 * 32768, fp)
if data.decode('ascii') != ref_data[0:3 * 32768]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 16384, 0)
data = gdal.VSIFReadL(1, 5 * 32768, fp)
if data.decode('ascii') != ref_data[16384:16384 + 5 * 32768]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
data = gdal.VSIFReadL(1, 50 * 32768, fp)
if data[0:1130496].decode('ascii') != ref_data[16384 + 5 * 32768:]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
gdal.Unlink('tmp/vsifile_5.bin')
return 'success'
def full_check_band( # noqa pylint: disable=too-many-branches
f,
band_name,
band,
errors,
block_order_row_major,
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery,
):
block_size = band.GetBlockSize()
mask_band = None
if mask_interleaved_with_imagery:
mask_band = band.GetMaskBand()
mask_block_size = mask_band.GetBlockSize()
if block_size != mask_block_size:
errors += [
band_name +
": mask block size is different from its imagery band"
]
mask_band = None
yblocks = (band.YSize + block_size[1] - 1) // block_size[1]
xblocks = (band.XSize + block_size[0] - 1) // block_size[0]
last_offset = 0
for y in range(yblocks): # noqa pylint: disable=too-many-nested-blocks
for x in range(xblocks):
offset = band.GetMetadataItem("BLOCK_OFFSET_%d_%d" % (x, y),
"TIFF")
offset = int(offset) if offset is not None else 0
bytecount = band.GetMetadataItem("BLOCK_SIZE_%d_%d" % (x, y),
"TIFF")
bytecount = int(bytecount) if bytecount is not None else 0
if offset > 0:
if block_order_row_major and offset < last_offset:
errors += [
band_name +
": offset of block (%d, %d) is smaller than previous block"
% (x, y)
]
if block_leader_size_as_uint4:
gdal.VSIFSeekL(f, offset - 4, 0)
leader_size = struct.unpack("<I", gdal.VSIFReadL(4, 1,
f))[0]
if leader_size != bytecount:
errors += [
band_name +
": for block (%d, %d), size in leader bytes is %d instead of %d"
% (x, y, leader_size, bytecount)
]
if block_trailer_last_4_bytes_repeated:
if bytecount >= 4:
gdal.VSIFSeekL(f, offset + bytecount - 4, 0)
last_bytes = gdal.VSIFReadL(8, 1, f)
if last_bytes[0:4] != last_bytes[4:8]:
errors += [
band_name +
": for block (%d, %d), trailer bytes are invalid"
% (x, y)
]
if mask_band:
offset_mask = mask_band.GetMetadataItem(
"BLOCK_OFFSET_%d_%d" % (x, y), "TIFF")
offset_mask = int(
offset_mask) if offset_mask is not None else 0
if offset > 0 and offset_mask > 0:
# bytecount_mask = int(mask_band.GetMetadataItem('BLOCK_SIZE_%d_%d' % (x,y), 'TIFF'))
expected_offset_mask = (
offset + bytecount +
(4 if block_leader_size_as_uint4 else 0) +
(4 if block_trailer_last_4_bytes_repeated else 0))
if offset_mask != expected_offset_mask:
errors += [
"Mask of " + band_name +
": for block (%d, %d), offset is %d, whereas %d was expected"
% (x, y, offset_mask, expected_offset_mask)
]
elif offset == 0 and offset_mask > 0:
if block_order_row_major and offset_mask < last_offset:
errors += [
"Mask of " + band_name +
": offset of block (%d, %d) is smaller than previous block"
% (x, y)
]
offset = offset_mask
last_offset = offset
def validate( # noqa pylint: disable=too-many-branches
ds,
check_tiled=True,
full_check=False):
"""Check if a file is a (Geo)TIFF with cloud optimized compatible structure.
Args:
ds: GDAL Dataset for the file to inspect.
check_tiled: Set to False to ignore missing tiling.
full_check: Set to TRUe to check tile/strip leader/trailer bytes. Might be slow on remote files
Returns:
A tuple, whose first element is an array of error messages
(empty if there is no error), and the second element, a dictionary
with the structure of the GeoTIFF file.
Raises:
ValidateCloudOptimizedGeoTIFFException: Unable to open the file or the
file is not a Tiff.
"""
if int(gdal.VersionInfo("VERSION_NUM")) < 2020000:
raise ValidateCloudOptimizedGeoTIFFException(
"GDAL 2.2 or above required")
unicode_type = type("".encode("utf-8").decode("utf-8"))
if isinstance(ds, (str, unicode_type)):
gdal.PushErrorHandler()
ds = gdal.Open(ds)
gdal.PopErrorHandler()
if ds is None:
raise ValidateCloudOptimizedGeoTIFFException(
"Invalid file : %s" % gdal.GetLastErrorMsg())
if ds.GetDriver().ShortName != "GTiff":
raise ValidateCloudOptimizedGeoTIFFException(
"The file is not a GeoTIFF")
details = {}
errors = []
warnings = []
filename = ds.GetDescription()
main_band = ds.GetRasterBand(1)
ovr_count = main_band.GetOverviewCount()
filelist = ds.GetFileList()
if filelist is not None and filename + ".ovr" in filelist:
errors += [
"Overviews found in external .ovr file. They should be internal"
]
if main_band.XSize > 512 or main_band.YSize > 512:
if check_tiled:
block_size = main_band.GetBlockSize()
if block_size[0] == main_band.XSize and block_size[0] > 1024:
errors += [
"The file is greater than 512xH or Wx512, but is not tiled"
]
if ovr_count == 0:
warnings += [
"The file is greater than 512xH or Wx512, it is recommended "
"to include internal overviews"
]
ifd_offset = int(main_band.GetMetadataItem("IFD_OFFSET", "TIFF"))
ifd_offsets = [ifd_offset]
block_order_row_major = False
block_leader_size_as_uint4 = False
block_trailer_last_4_bytes_repeated = False
mask_interleaved_with_imagery = False
if ifd_offset not in (8, 16):
# Check if there is GDAL hidden structural metadata
f = gdal.VSIFOpenL(filename, "rb")
if not f:
raise ValidateCloudOptimizedGeoTIFFException("Cannot open file")
signature = struct.unpack("B" * 4, gdal.VSIFReadL(4, 1, f))
bigtiff = signature in ((0x49, 0x49, 0x2B, 0x00), (0x4D, 0x4D, 0x00,
0x2B))
if bigtiff:
expected_ifd_pos = 16
else:
expected_ifd_pos = 8
gdal.VSIFSeekL(f, expected_ifd_pos, 0)
pattern = "GDAL_STRUCTURAL_METADATA_SIZE=%06d bytes\n" % 0
got = gdal.VSIFReadL(len(pattern), 1, f).decode("LATIN1")
if len(got) == len(pattern) and got.startswith(
"GDAL_STRUCTURAL_METADATA_SIZE="):
size = int(got[len("GDAL_STRUCTURAL_METADATA_SIZE="):][0:6])
extra_md = gdal.VSIFReadL(size, 1, f).decode("LATIN1")
block_order_row_major = "BLOCK_ORDER=ROW_MAJOR" in extra_md
block_leader_size_as_uint4 = "BLOCK_LEADER=SIZE_AS_UINT4" in extra_md
block_trailer_last_4_bytes_repeated = (
"BLOCK_TRAILER=LAST_4_BYTES_REPEATED" in extra_md)
mask_interleaved_with_imagery = (
"MASK_INTERLEAVED_WITH_IMAGERY=YES" in extra_md)
if "KNOWN_INCOMPATIBLE_EDITION=YES" in extra_md:
errors += [
"KNOWN_INCOMPATIBLE_EDITION=YES is declared in the file"
]
expected_ifd_pos += len(pattern) + size
expected_ifd_pos += (expected_ifd_pos % 2
) # IFD offset starts on a 2-byte boundary
gdal.VSIFCloseL(f)
if expected_ifd_pos != ifd_offsets[0]:
errors += [
"The offset of the main IFD should be %d. It is %d instead" %
(expected_ifd_pos, ifd_offsets[0])
]
details["ifd_offsets"] = {}
details["ifd_offsets"]["main"] = ifd_offset
for i in range(ovr_count):
# Check that overviews are by descending sizes
ovr_band = ds.GetRasterBand(1).GetOverview(i)
if i == 0:
if ovr_band.XSize > main_band.XSize or ovr_band.YSize > main_band.YSize:
errors += [
"First overview has larger dimension than main band"
]
else:
prev_ovr_band = ds.GetRasterBand(1).GetOverview(i - 1)
if (ovr_band.XSize > prev_ovr_band.XSize
or ovr_band.YSize > prev_ovr_band.YSize):
errors += [
"Overview of index %d has larger dimension than "
"overview of index %d" % (i, i - 1)
]
if check_tiled:
block_size = ovr_band.GetBlockSize()
if block_size[0] == ovr_band.XSize and block_size[0] > 1024:
errors += ["Overview of index %d is not tiled" % i]
# Check that the IFD of descending overviews are sorted by increasing
# offsets
ifd_offset = int(ovr_band.GetMetadataItem("IFD_OFFSET", "TIFF"))
ifd_offsets.append(ifd_offset)
details["ifd_offsets"]["overview_%d" % i] = ifd_offset
if ifd_offsets[-1] < ifd_offsets[-2]:
if i == 0:
errors += [
"The offset of the IFD for overview of index %d is %d, "
"whereas it should be greater than the one of the main "
"image, which is at byte %d" %
(i, ifd_offsets[-1], ifd_offsets[-2])
]
else:
errors += [
"The offset of the IFD for overview of index %d is %d, "
"whereas it should be greater than the one of index %d, "
"which is at byte %d" %
(i, ifd_offsets[-1], i - 1, ifd_offsets[-2])
]
# Check that the imagery starts by the smallest overview and ends with
# the main resolution dataset
def get_block_offset(band):
blockxsize, blockysize = band.GetBlockSize()
for y in range(int((band.YSize + blockysize - 1) / blockysize)):
for x in range(int((band.XSize + blockxsize - 1) / blockxsize)):
block_offset = band.GetMetadataItem(
"BLOCK_OFFSET_%d_%d" % (x, y), "TIFF")
if block_offset:
return int(block_offset)
return 0
block_offset = get_block_offset(main_band)
data_offsets = [block_offset]
details["data_offsets"] = {}
details["data_offsets"]["main"] = block_offset
for i in range(ovr_count):
ovr_band = ds.GetRasterBand(1).GetOverview(i)
block_offset = get_block_offset(ovr_band)
data_offsets.append(block_offset)
details["data_offsets"]["overview_%d" % i] = block_offset
if data_offsets[-1] != 0 and data_offsets[-1] < ifd_offsets[-1]:
if ovr_count > 0:
errors += [
"The offset of the first block of the smallest overview "
"should be after its IFD"
]
else:
errors += [
"The offset of the first block of the image should be after its IFD"
]
for i in range(len(data_offsets) - 2, 0, -1):
if data_offsets[i] != 0 and data_offsets[i] < data_offsets[i + 1]:
errors += [
"The offset of the first block of overview of index %d should "
"be after the one of the overview of index %d" % (i - 1, i)
]
if (len(data_offsets) >= 2 and data_offsets[0] != 0
and data_offsets[0] < data_offsets[1]):
errors += [
"The offset of the first block of the main resolution image "
"should be after the one of the overview of index %d" %
(ovr_count - 1)
]
if full_check and (block_order_row_major or block_leader_size_as_uint4
or block_trailer_last_4_bytes_repeated
or mask_interleaved_with_imagery):
f = gdal.VSIFOpenL(filename, "rb")
if not f:
raise ValidateCloudOptimizedGeoTIFFException("Cannot open file")
full_check_band(
f,
"Main resolution image",
main_band,
errors,
block_order_row_major,
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery,
)
if (main_band.GetMaskFlags() == gdal.GMF_PER_DATASET
and (filename + ".msk") not in ds.GetFileList()):
full_check_band(
f,
"Mask band of main resolution image",
main_band.GetMaskBand(),
errors,
block_order_row_major,
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
False,
)
for i in range(ovr_count):
ovr_band = ds.GetRasterBand(1).GetOverview(i)
full_check_band(
f,
"Overview %d" % i,
ovr_band,
errors,
block_order_row_major,
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery,
)
if (ovr_band.GetMaskFlags() == gdal.GMF_PER_DATASET
and (filename + ".msk") not in ds.GetFileList()):
full_check_band(
f,
"Mask band of overview %d" % i,
ovr_band.GetMaskBand(),
errors,
block_order_row_major,
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
False,
)
gdal.VSIFCloseL(f)
return warnings, errors, details
