def test_vsiaz_fake_write():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
# Test creation of BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:AigkrY7q66WCrx3JRKBte56k7kxV2cxB/ZyGNubxk5I=' or \
'Expect' not in h or h['Expect'] != '100-continue' or \
'Content-Length' not in h or h['Content-Length'] != '40000' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'BlockBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(40000).decode('ascii')
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Simulate illegal read
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
assert not ret
gdal.VSIFCloseL(f)
# Simulate illegal seek
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
assert ret != 0
gdal.VSIFCloseL(f)
# Simulate failure when putting BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
if gdal.VSIFSeekL(f, 0, 0) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if gdal.VSIFTellL(f) != 35000:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFSeekL(f, 35000, 0) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFSeekL(f, 0, 1) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFSeekL(f, 0, 2) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFEofL(f) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFCloseL(f)
if ret == 0:
gdal.VSIFCloseL(f)
pytest.fail(ret)
# Simulate creation of BlockBob over an existing blob of incompatible type
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 409)
handler.add('DELETE', '/azure/blob/myaccount/test_copy/file.bin', 202)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
# Test creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:KimVui3ptY9D5ftLlsI7CNOgK36CNAEzsXqcuHskdEY=' or \
'Content-Length' not in h or h['Content-Length'] != '0' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '10' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(10).decode('ascii')
if content != '0123456789':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '6' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(6).decode('ascii')
if content != 'abcdef':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 16:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Test failed creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Test failed writing of a block of an AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
assert f is not None
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
403)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
def seek(self, pos, ref):
gdal.VSIFSeekL(self.f, pos, ref)
def vsis3_4():
if gdaltest.webserver_port == 0:
return 'skip'
with gdaltest.error_handler():
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3', 'wb')
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsis3/s3_fake_bucket3/empty_file.bin').size != 3:
gdaltest.post_reason('fail')
return 'fail'
# Empty file
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
gdal.VSIFCloseL(f)
if gdal.GetLastErrorMsg() != '':
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsis3/s3_fake_bucket3/empty_file.bin').size != 0:
gdaltest.post_reason('fail')
return 'fail'
# Invalid seek
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
# Invalid read
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
if len(ret) != 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
# Error case
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file_error.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
gdal.VSIFCloseL(f)
if gdal.GetLastErrorMsg() == '':
gdaltest.post_reason('fail')
return 'fail'
# Nominal case
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/another_file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFSeekL(f, 0, 1) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFSeekL(f, 0, 2) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFWriteL('foo', 1, 3, f) != 3:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFWriteL('bar', 1, 3, f) != 3:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
gdal.VSIFCloseL(f)
if gdal.GetLastErrorMsg() != '':
gdaltest.post_reason('fail')
return 'fail'
# Redirect case
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/redirect', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFWriteL('foobar', 1, 6, f) != 6:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
gdal.VSIFCloseL(f)
if gdal.GetLastErrorMsg() != '':
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def gdal_cp_single(srcfile, targetfile, progress):
if targetfile.endswith('/'):
stat_res = gdal.VSIStatL(targetfile)
else:
stat_res = gdal.VSIStatL(targetfile + '/')
if (stat_res is None and targetfile.endswith('/')) or \
(stat_res is not None and stat.S_ISDIR(stat_res.mode)):
(_, tail) = os.path.split(srcfile)
if targetfile.endswith('/'):
targetfile = targetfile + tail
else:
targetfile = targetfile + '/' + tail
fin = gdal.VSIFOpenL(srcfile, "rb")
if fin is None:
print('Cannot open %s' % srcfile)
return -1
fout = gdal.VSIFOpenL(targetfile, "wb")
if fout is None:
print('Cannot create %s' % targetfile)
gdal.VSIFCloseL(fin)
return -1
version_num = int(gdal.VersionInfo('VERSION_NUM'))
total_size = 0
if version_num < 1900 or progress is not None:
gdal.VSIFSeekL(fin, 0, 2)
total_size = gdal.VSIFTellL(fin)
gdal.VSIFSeekL(fin, 0, 0)
buf_max_size = 4096
copied = 0
ret = 0
# print('Copying %s...' % srcfile)
if progress is not None:
if not progress.Progress(0.0, 'Copying %s' % srcfile):
print('Copy stopped by user')
ret = -2
while ret == 0:
if total_size != 0 and copied + buf_max_size > total_size:
to_read = total_size - copied
else:
to_read = buf_max_size
buf = gdal.VSIFReadL(1, to_read, fin)
if buf is None:
if copied == 0:
print('Cannot read %d bytes in %s' % (to_read, srcfile))
ret = -1
break
buf_size = len(buf)
if gdal.VSIFWriteL(buf, 1, buf_size, fout) != buf_size:
print('Error writing %d bytes' % buf_size)
ret = -1
break
copied += buf_size
if progress is not None and total_size != 0:
if not progress.Progress(copied * 1.0 / total_size,
'Copying %s' % srcfile):
print('Copy stopped by user')
ret = -2
break
if to_read < buf_max_size or buf_size != buf_max_size:
break
gdal.VSIFCloseL(fin)
gdal.VSIFCloseL(fout)
return ret
def full_check_band(f, band_name, band, errors, block_order_row_major,
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery):
block_size = band.GetBlockSize()
mask_band = None
if mask_interleaved_with_imagery:
mask_band = band.GetMaskBand()
mask_block_size = mask_band.GetBlockSize()
if block_size != mask_block_size:
errors += [
band_name +
': mask block size is different from its imagery band'
]
mask_band = None
yblocks = (band.YSize + block_size[1] - 1) // block_size[1]
xblocks = (band.XSize + block_size[0] - 1) // block_size[0]
last_offset = 0
for y in range(yblocks):
for x in range(xblocks):
offset = int(
band.GetMetadataItem('BLOCK_OFFSET_%d_%d' % (x, y), 'TIFF'))
bytecount = int(
band.GetMetadataItem('BLOCK_SIZE_%d_%d' % (x, y), 'TIFF'))
if block_order_row_major and offset < last_offset:
errors += [
band_name +
': offset of block (%d, %d) is smaller than previous block'
% (x, y)
]
if block_leader_size_as_uint4:
gdal.VSIFSeekL(f, offset - 4, 0)
leader_size = struct.unpack('<I', gdal.VSIFReadL(4, 1, f))[0]
if leader_size != bytecount:
errors += [
band_name +
': for block (%d, %d), size in leader bytes is %d instead of %d'
% (x, y, leader_size, bytecount)
]
if block_trailer_last_4_bytes_repeated:
if bytecount >= 4:
gdal.VSIFSeekL(f, offset + bytecount - 4, 0)
last_bytes = gdal.VSIFReadL(8, 1, f)
if last_bytes[0:4] != last_bytes[4:8]:
errors += [
band_name +
': for block (%d, %d), trailer bytes are invalid' %
(x, y)
]
if mask_band:
offset_mask = int(
mask_band.GetMetadataItem('BLOCK_OFFSET_%d_%d' % (x, y),
'TIFF'))
#bytecount_mask = int(mask_band.GetMetadataItem('BLOCK_SIZE_%d_%d' % (x,y), 'TIFF'))
expected_offset_mask = offset + bytecount + \
(4 if block_leader_size_as_uint4 else 0) + \
(4 if block_trailer_last_4_bytes_repeated else 0)
if offset_mask != expected_offset_mask:
errors += [
'Mask of ' + band_name +
': for block (%d, %d), offset is %d, whereas %d was expected'
% (x, y, offset_mask, expected_offset_mask)
]
last_offset = offset
def extract_tile(ds, src_band_nbr, tile_x, tile_y, jpg_filename):
block_offset = ds.GetRasterBand(src_band_nbr).GetMetadataItem(
'BLOCK_OFFSET_%d_%d' % (tile_x, tile_y), 'TIFF')
block_size = ds.GetRasterBand(src_band_nbr).GetMetadataItem(
'BLOCK_SIZE_%d_%d' % (tile_x, tile_y), 'TIFF')
if block_offset is None or block_size is None:
print('ERROR: Cannot find block (%d,%d)' % (tile_x, tile_y))
return 1
jpegtables = ds.GetRasterBand(src_band_nbr).GetMetadataItem(
'JPEGTABLES', 'TIFF')
if jpegtables is not None:
if (len(jpegtables) % 2
) != 0 or jpegtables[0:4] != 'FFD8' or jpegtables[-2:] != 'D9':
print('ERROR: Invalid JPEG tables')
print(jpegtables)
return 1
# Remove final D9
jpegtables = jpegtables[0:-2]
tiff_f = gdal.VSIFOpenL(ds.GetDescription(), 'rb')
if tiff_f is None:
print('ERROR: Cannot reopen %s' % ds.GetDescription())
return 1
out_f = gdal.VSIFOpenL(jpg_filename, 'wb')
if out_f is None:
print('ERROR: Cannot create %s' % jpg_filename)
gdal.VSIFCloseL(tiff_f)
return 1
# Write JPEG tables
if jpegtables is not None:
for i in range(int(len(jpegtables) / 2)):
c1 = ord(jpegtables[2 * i])
c2 = ord(jpegtables[2 * i + 1])
if c1 >= ord('0') and c1 <= ord('9'):
val = c1 - ord('0')
else:
val = (c1 - ord('A')) + 10
val = val * 16
if c2 >= ord('0') and c2 <= ord('9'):
val = val + (c2 - ord('0'))
else:
val = val + (c2 - ord('A')) + 10
gdal.VSIFWriteL(chr(val), 1, 1, out_f)
else:
gdal.VSIFWriteL(chr(0xFF), 1, 1, out_f)
gdal.VSIFWriteL(chr(0xD8), 1, 1, out_f)
# Write Adobe APP14 marker if necessary
interleave = ds.GetMetadataItem('INTERLEAVE', 'IMAGE_STRUCTURE')
photometric = ds.GetMetadataItem('COMPRESSION', 'IMAGE_STRUCTURE')
if interleave == 'PIXEL' and photometric == 'JPEG' and ds.RasterCount == 3:
adobe_app14 = [
0xFF, 0xEE, 0x00, 0x0E, 0x41, 0x64, 0x6F, 0x62, 0x65, 0x00, 0x64,
0x00, 0x00, 0x00, 0x00, 0x00
]
for c in adobe_app14:
gdal.VSIFWriteL(chr(c), 1, 1, out_f)
# Write JPEG codestream
# skip leading 0xFF 0xD8
gdal.VSIFSeekL(tiff_f, int(block_offset) + 2, 0)
data = gdal.VSIFReadL(1, int(block_size) - 2, tiff_f)
gdal.VSIFCloseL(tiff_f)
gdal.VSIFWriteL(data, 1, len(data), out_f)
gdal.VSIFCloseL(out_f)
aux_xml_filename = '%s.aux.xml' % jpg_filename
gt = ds.GetGeoTransform()
srs = ds.GetProjectionRef()
if srs is not None and srs != '':
sub_gt = [gt[i] for i in range(6)]
(blockxsize, blockysize) = ds.GetRasterBand(1).GetBlockSize()
sub_gt[0] = gt[0] + tile_x * blockxsize * gt[1]
sub_gt[3] = gt[3] + tile_y * blockysize * gt[5]
out_f = gdal.VSIFOpenL(aux_xml_filename, 'wb')
if out_f is None:
print('ERROR: Cannot create %s' % aux_xml_filename)
return 1
content = """<PAMDataset>
<SRS>%s</SRS>
<GeoTransform>%.18g,%.18g,%.18g,%.18g,%.18g,%.18g</GeoTransform>
</PAMDataset>
""" % (srs, sub_gt[0], sub_gt[1], sub_gt[2], sub_gt[3], sub_gt[4],
sub_gt[5])
gdal.VSIFWriteL(content, 1, len(content), out_f)
gdal.VSIFCloseL(out_f)
else:
gdal.Unlink('%s.aux.xml' % jpg_filename)
return 0
def mvt(request):
if not MVT_DRIVER_EXIST:
return HTTPNotFound(explanation='MVT GDAL driver not found')
z = int(request.GET["z"])
x = int(request.GET["x"])
y = int(request.GET["y"])
extent = int(request.GET.get('extent', 4096))
simplification = float(request.GET.get("simplification", extent / 512))
resids = map(
int,
filter(None, request.GET["resource"].split(",")),
)
# web mercator
merc = SRS.filter_by(id=3857).one()
minx, miny, maxx, maxy = merc.tile_extent((z, x, y))
# 5% padding by default
padding = float(request.GET.get("padding", 0.05))
bbox = (
minx - (maxx - minx) * padding,
miny - (maxy - miny) * padding,
maxx + (maxx - minx) * padding,
maxy + (maxy - miny) * padding,
)
bbox = Geometry.from_shape(box(*bbox), srid=merc.id)
options = [
"FORMAT=DIRECTORY",
"TILE_EXTENSION=pbf",
"MINZOOM=%d" % z,
"MAXZOOM=%d" % z,
"EXTENT=%d" % extent,
"COMPRESS=NO",
]
ds = _ogr_ds("MVT", options)
vsibuf = ds.GetName()
for resid in resids:
try:
obj = Resource.filter_by(id=resid).one()
except NoResultFound:
raise ResourceNotFound(resid)
request.resource_permission(PERM_READ, obj)
query = obj.feature_query()
query.intersects(bbox)
query.geom()
if IFeatureQueryClipByBox.providedBy(query):
query.clip_by_box(bbox)
if IFeatureQuerySimplify.providedBy(query):
tolerance = ((obj.srs.maxx - obj.srs.minx) / (1 << z)) / extent
query.simplify(tolerance * simplification)
_ogr_layer_from_features(obj, query(), name="ngw:%d" % obj.id, ds=ds)
# flush changes
ds = None
filepath = os.path.join("%s" % vsibuf, "%d" % z, "%d" % x, "%d.pbf" % y)
try:
f = gdal.VSIFOpenL(ensure_str(filepath), ensure_str("rb"))
if f is not None:
# SEEK_END = 2
gdal.VSIFSeekL(f, 0, 2)
size = gdal.VSIFTellL(f)
# SEEK_SET = 0
gdal.VSIFSeekL(f, 0, 0)
content = gdal.VSIFReadL(1, size, f)
gdal.VSIFCloseL(f)
return Response(
content,
content_type="application/vnd.mapbox-vector-tile",
)
else:
return HTTPNoContent()
finally:
gdal.Unlink(ensure_str(vsibuf))
def validate(ds, check_tiled=True, full_check=False):
"""Check if a file is a (Geo)TIFF with cloud optimized compatible structure.
Args:
ds: GDAL Dataset for the file to inspect.
check_tiled: Set to False to ignore missing tiling.
full_check: Set to TRUe to check tile/strip leader/trailer bytes. Might be slow on remote files
Returns:
A tuple, whose first element is an array of error messages
(empty if there is no error), and the second element, a dictionary
with the structure of the GeoTIFF file.
Raises:
ValidateCloudOptimizedGeoTIFFException: Unable to open the file or the
file is not a Tiff.
"""
if int(gdal.VersionInfo('VERSION_NUM')) < 2020000:
raise ValidateCloudOptimizedGeoTIFFException(
'GDAL 2.2 or above required')
unicode_type = type(''.encode('utf-8').decode('utf-8'))
if isinstance(ds, (str, unicode_type)):
gdal.PushErrorHandler()
ds = gdal.Open(ds)
gdal.PopErrorHandler()
if ds is None:
raise ValidateCloudOptimizedGeoTIFFException(
'Invalid file : %s' % gdal.GetLastErrorMsg())
if ds.GetDriver().ShortName != 'GTiff':
raise ValidateCloudOptimizedGeoTIFFException(
'The file is not a GeoTIFF')
details = {}
errors = []
warnings = []
filename = ds.GetDescription()
main_band = ds.GetRasterBand(1)
ovr_count = main_band.GetOverviewCount()
filelist = ds.GetFileList()
if filelist is not None and filename + '.ovr' in filelist:
errors += [
'Overviews found in external .ovr file. They should be internal'
]
if main_band.XSize > 512 or main_band.YSize > 512:
if check_tiled:
block_size = main_band.GetBlockSize()
if block_size[0] == main_band.XSize and block_size[0] > 1024:
errors += [
'The file is greater than 512xH or Wx512, but is not tiled'
]
if ovr_count == 0:
warnings += [
'The file is greater than 512xH or Wx512, it is recommended '
'to include internal overviews'
]
ifd_offset = int(main_band.GetMetadataItem('IFD_OFFSET', 'TIFF'))
ifd_offsets = [ifd_offset]
block_order_row_major = False
block_leader_size_as_uint4 = False
block_trailer_last_4_bytes_repeated = False
mask_interleaved_with_imagery = False
if ifd_offset not in (8, 16):
# Check if there is GDAL hidden structural metadata
f = gdal.VSIFOpenL(filename, 'rb')
if not f:
raise ValidateCloudOptimizedGeoTIFFException("Cannot open file")
signature = struct.unpack('B' * 4, gdal.VSIFReadL(4, 1, f))
bigtiff = signature in ((0x49, 0x49, 0x2B, 0x00), (0x4D, 0x4D, 0x00,
0x2B))
if bigtiff:
expected_ifd_pos = 16
else:
expected_ifd_pos = 8
gdal.VSIFSeekL(f, expected_ifd_pos, 0)
pattern = "GDAL_STRUCTURAL_METADATA_SIZE=%06d bytes\n" % 0
got = gdal.VSIFReadL(len(pattern), 1, f).decode('LATIN1')
if len(got) == len(pattern) and got.startswith(
'GDAL_STRUCTURAL_METADATA_SIZE='):
size = int(got[len('GDAL_STRUCTURAL_METADATA_SIZE='):][0:6])
extra_md = gdal.VSIFReadL(size, 1, f).decode('LATIN1')
block_order_row_major = 'BLOCK_ORDER=ROW_MAJOR' in extra_md
block_leader_size_as_uint4 = 'BLOCK_LEADER=SIZE_AS_UINT4' in extra_md
block_trailer_last_4_bytes_repeated = 'BLOCK_TRAILER=LAST_4_BYTES_REPEATED' in extra_md
mask_interleaved_with_imagery = 'MASK_INTERLEAVED_WITH_IMAGERY=YES' in extra_md
if 'KNOWN_INCOMPATIBLE_EDITION=YES' in extra_md:
errors += [
"KNOWN_INCOMPATIBLE_EDITION=YES is declared in the file"
]
expected_ifd_pos += len(pattern) + size
expected_ifd_pos += expected_ifd_pos % 2 # IFD offset starts on a 2-byte boundary
gdal.VSIFCloseL(f)
if expected_ifd_pos != ifd_offsets[0]:
errors += [
'The offset of the main IFD should be %d. It is %d instead' %
(expected_ifd_pos, ifd_offsets[0])
]
details['ifd_offsets'] = {}
details['ifd_offsets']['main'] = ifd_offset
for i in range(ovr_count):
# Check that overviews are by descending sizes
ovr_band = ds.GetRasterBand(1).GetOverview(i)
if i == 0:
if (ovr_band.XSize > main_band.XSize
or ovr_band.YSize > main_band.YSize):
errors += [
'First overview has larger dimension than main band'
]
else:
prev_ovr_band = ds.GetRasterBand(1).GetOverview(i - 1)
if (ovr_band.XSize > prev_ovr_band.XSize
or ovr_band.YSize > prev_ovr_band.YSize):
errors += [
'Overview of index %d has larger dimension than '
'overview of index %d' % (i, i - 1)
]
if check_tiled:
block_size = ovr_band.GetBlockSize()
if block_size[0] == ovr_band.XSize and block_size[0] > 1024:
errors += ['Overview of index %d is not tiled' % i]
# Check that the IFD of descending overviews are sorted by increasing
# offsets
ifd_offset = int(ovr_band.GetMetadataItem('IFD_OFFSET', 'TIFF'))
ifd_offsets.append(ifd_offset)
details['ifd_offsets']['overview_%d' % i] = ifd_offset
if ifd_offsets[-1] < ifd_offsets[-2]:
if i == 0:
errors += [
'The offset of the IFD for overview of index %d is %d, '
'whereas it should be greater than the one of the main '
'image, which is at byte %d' %
(i, ifd_offsets[-1], ifd_offsets[-2])
]
else:
errors += [
'The offset of the IFD for overview of index %d is %d, '
'whereas it should be greater than the one of index %d, '
'which is at byte %d' %
(i, ifd_offsets[-1], i - 1, ifd_offsets[-2])
]
# Check that the imagery starts by the smallest overview and ends with
# the main resolution dataset
block_offset = main_band.GetMetadataItem('BLOCK_OFFSET_0_0', 'TIFF')
if not block_offset:
errors += ['Missing BLOCK_OFFSET_0_0']
data_offset = int(block_offset) if block_offset else None
data_offsets = [data_offset]
details['data_offsets'] = {}
details['data_offsets']['main'] = data_offset
for i in range(ovr_count):
ovr_band = ds.GetRasterBand(1).GetOverview(i)
data_offset = int(ovr_band.GetMetadataItem('BLOCK_OFFSET_0_0', 'TIFF'))
data_offsets.append(data_offset)
details['data_offsets']['overview_%d' % i] = data_offset
if data_offsets[-1] < ifd_offsets[-1]:
if ovr_count > 0:
errors += [
'The offset of the first block of the smallest overview '
'should be after its IFD'
]
else:
errors += [
'The offset of the first block of the image should '
'be after its IFD'
]
for i in range(len(data_offsets) - 2, 0, -1):
if data_offsets[i] < data_offsets[i + 1]:
errors += [
'The offset of the first block of overview of index %d should '
'be after the one of the overview of index %d' % (i - 1, i)
]
if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]:
errors += [
'The offset of the first block of the main resolution image'
'should be after the one of the overview of index %d' %
(ovr_count - 1)
]
if full_check and (block_order_row_major or block_leader_size_as_uint4 or \
block_trailer_last_4_bytes_repeated or \
mask_interleaved_with_imagery):
f = gdal.VSIFOpenL(filename, 'rb')
if not f:
raise ValidateCloudOptimizedGeoTIFFException("Cannot open file")
full_check_band(f, 'Main resolution image', main_band, errors,
block_order_row_major, block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery)
if main_band.GetMaskFlags() == gdal.GMF_PER_DATASET and \
(filename + '.msk') not in ds.GetFileList():
full_check_band(f, 'Mask band of main resolution image',
main_band.GetMaskBand(), errors,
block_order_row_major, block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated, False)
for i in range(ovr_count):
ovr_band = ds.GetRasterBand(1).GetOverview(i)
full_check_band(f, 'Overview %d' % i, ovr_band, errors,
block_order_row_major, block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery)
if ovr_band.GetMaskFlags() == gdal.GMF_PER_DATASET and \
(filename + '.msk') not in ds.GetFileList():
full_check_band(f, 'Mask band of overview %d' % i,
ovr_band.GetMaskBand(), errors,
block_order_row_major,
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated, False)
gdal.VSIFCloseL(f)
return warnings, errors, details
def test_vsifile_5():
fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'wb')
ref_data = ''.join(['%08X' % i for i in range(5 * 32768)])
gdal.VSIFWriteL(ref_data, 1, len(ref_data), fp)
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSI_CACHE', 'YES')
for i in range(3):
if i == 0:
gdal.SetConfigOption('VSI_CACHE_SIZE', '0')
elif i == 1:
gdal.SetConfigOption('VSI_CACHE_SIZE', '65536')
else:
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'rb')
gdal.VSIFSeekL(fp, 50000, 0)
if gdal.VSIFTellL(fp) != 50000:
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
pytest.fail()
gdal.VSIFSeekL(fp, 50000, 1)
if gdal.VSIFTellL(fp) != 100000:
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
pytest.fail()
gdal.VSIFSeekL(fp, 0, 2)
if gdal.VSIFTellL(fp) != 5 * 32768 * 8:
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
pytest.fail()
gdal.VSIFReadL(1, 1, fp)
gdal.VSIFSeekL(fp, 0, 0)
data = gdal.VSIFReadL(1, 3 * 32768, fp)
if data.decode('ascii') != ref_data[0:3 * 32768]:
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
pytest.fail()
gdal.VSIFSeekL(fp, 16384, 0)
data = gdal.VSIFReadL(1, 5 * 32768, fp)
if data.decode('ascii') != ref_data[16384:16384 + 5 * 32768]:
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
pytest.fail()
data = gdal.VSIFReadL(1, 50 * 32768, fp)
if data[0:1130496].decode('ascii') != ref_data[16384 + 5 * 32768:]:
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
pytest.fail()
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
gdal.Unlink('tmp/vsifile_5.bin')
def vsifile_generic(filename):
start_time = time.time()
fp = gdal.VSIFOpenL(filename, 'wb+')
assert fp is not None
assert gdal.VSIFWriteL('0123456789', 1, 10, fp) == 10
assert gdal.VSIFFlushL(fp) == 0
assert gdal.VSIFTruncateL(fp, 20) == 0
assert gdal.VSIFTellL(fp) == 10
assert gdal.VSIFTruncateL(fp, 5) == 0
assert gdal.VSIFTellL(fp) == 10
assert gdal.VSIFSeekL(fp, 0, 2) == 0
assert gdal.VSIFTellL(fp) == 5
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
assert statBuf.size == 7
assert start_time == pytest.approx(statBuf.mtime, abs=2)
fp = gdal.VSIFOpenL(filename, 'rb')
buf = gdal.VSIFReadL(1, 7, fp)
assert gdal.VSIFWriteL('a', 1, 1, fp) == 0
assert gdal.VSIFTruncateL(fp, 0) != 0
gdal.VSIFCloseL(fp)
assert buf.decode('ascii') == '01234XX'
# Test append mode on existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
assert statBuf.size == 9
assert gdal.Unlink(filename) == 0
statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG)
assert statBuf is None
# Test append mode on non existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
assert statBuf.size == 2
assert gdal.Unlink(filename) == 0
def test_vsicurl_test_redirect():
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_redirect/', 404)
# Simulate a big time difference between server and local machine
current_time = 1500
def method(request):
response = 'HTTP/1.1 302\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n'
response += 'Location: %s\r\n' % (
'http://localhost:%d/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (gdaltest.webserver_port, current_time + 30))
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add('HEAD', '/test_redirect/test.bin', custom_method=method)
handler.add(
'HEAD',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30), 403, {'Server': 'foo'}, '')
def method(request):
if 'Range' in request.headers:
if request.headers['Range'] == 'bytes=0-16383':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 0-16383/1000000')
request.send_header('Content-Length', 16384)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
elif request.headers['Range'] == 'bytes=16384-49151':
# Test expiration of the signed URL
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
else:
request.send_response(404)
request.send_header('Content-Length', 0)
request.end_headers()
else:
# After a failed attempt on a HEAD, the client should go there
response = 'HTTP/1.1 200\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime(
"%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(current_time)) + '\r\n'
response += 'Content-type: text/plain\r\n'
response += 'Content-Length: 1000000\r\n'
response += 'Connection: close\r\n'
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL(
'/vsicurl/http://localhost:%d/test_redirect/test.bin' %
gdaltest.webserver_port, 'rb')
assert f is not None
gdal.VSIFSeekL(f, 0, 2)
if gdal.VSIFTellL(f) != 1000000:
gdal.VSIFCloseL(f)
pytest.fail(gdal.VSIFTellL(f))
gdal.VSIFSeekL(f, 0, 0)
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
current_time = int(time.time())
def method(request):
# We should go there after expiration of the first signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(302)
# Return a new signed URL
request.send_header(
'Location',
'http://localhost:%d/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d'
% (request.server.port, current_time + 30))
request.send_header('Content-Length', 16384)
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
handler.add('GET', '/test_redirect/test.bin', custom_method=method)
def method(request):
# Second signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 16384-16384/1000000')
request.send_header('Content-Length', 1)
request.end_headers()
request.wfile.write('y'.encode('ascii'))
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
with webserver.install_http_handler(handler):
content = gdal.VSIFReadL(1, 16383, f).decode('ascii')
if len(content) != 16383 or content[0] != 'x':
gdal.VSIFCloseL(f)
pytest.fail(content)
content = gdal.VSIFReadL(1, 2, f).decode('ascii')
if content != 'xy':
gdal.VSIFCloseL(f)
pytest.fail(content)
gdal.VSIFCloseL(f)
def vsifile_generic(filename):
start_time = time.time()
fp = gdal.VSIFOpenL(filename, 'wb+')
if fp is None:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFWriteL('0123456789', 1, 10, fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTruncateL(fp, 20) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTruncateL(fp, 5) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFSeekL(fp, 0, 2) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 5:
gdaltest.post_reason('failure')
return 'fail'
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 7:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if abs(start_time - statBuf.mtime) > 2:
gdaltest.post_reason('failure')
print(statBuf.mtime)
return 'fail'
fp = gdal.VSIFOpenL(filename, 'rb')
buf = gdal.VSIFReadL(1, 7, fp)
if gdal.VSIFWriteL('a', 1, 1, fp) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFTruncateL(fp, 0) == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(fp)
if buf.decode('ascii') != '01234XX':
gdaltest.post_reason('failure')
print(buf.decode('ascii'))
return 'fail'
# Test append mode on existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 9:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if gdal.Unlink(filename) != 0:
gdaltest.post_reason('failure')
return 'fail'
statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG)
if statBuf is not None:
gdaltest.post_reason('failure')
return 'fail'
# Test append mode on non existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 2:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if gdal.Unlink(filename) != 0:
gdaltest.post_reason('failure')
return 'fail'
return 'success'
def vsicurl_streaming_1():
try:
drv = gdal.GetDriverByName('HTTP')
except:
drv = None
if drv is None:
return 'skip'
fp = gdal.VSIFOpenL(
'/vsicurl_streaming/http://download.osgeo.org/gdal/data/usgsdem/cded/114p01_0100_deme.dem',
'rb')
if fp is None:
if gdaltest.gdalurlopen(
'http://download.osgeo.org/gdal/data/usgsdem/cded/114p01_0100_deme.dem'
) is None:
print('cannot open URL')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFTellL(fp) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
data = gdal.VSIFReadL(1, 50, fp)
if data.decode(
'ascii') != ' 114p01DEMe Base Ma':
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
if gdal.VSIFTellL(fp) != 50:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
gdal.VSIFSeekL(fp, 0, 0)
if gdal.VSIFTellL(fp) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
data = gdal.VSIFReadL(1, 50, fp)
if data.decode(
'ascii') != ' 114p01DEMe Base Ma':
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
if gdal.VSIFTellL(fp) != 50:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
time.sleep(0.5)
gdal.VSIFSeekL(fp, 2001, 0)
data_2001 = gdal.VSIFReadL(1, 20, fp)
if data_2001.decode('ascii') != '7-32767-32767-32767-':
print(data_2001)
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
if gdal.VSIFTellL(fp) != 2001 + 20:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
gdal.VSIFSeekL(fp, 0, 2)
if gdal.VSIFTellL(fp) != 9839616:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
nRet = len(gdal.VSIFReadL(1, 10, fp))
if nRet != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
gdal.VSIFSeekL(fp, 2001, 0)
data_2001_2 = gdal.VSIFReadL(1, 20, fp)
if gdal.VSIFTellL(fp) != 2001 + 20:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
if data_2001 != data_2001_2:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
gdal.VSIFSeekL(fp, 1024 * 1024 + 100, 0)
data = gdal.VSIFReadL(1, 20, fp)
if data.decode('ascii') != '67-32767-32767-32767':
print(data)
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
if gdal.VSIFTellL(fp) != 1024 * 1024 + 100 + 20:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
gdal.VSIFCloseL(fp)
return 'success'
def vsicrypt_4():
if not gdaltest.has_vsicrypt:
return 'skip'
test_file = '/vsicrypt/key=DONT_USE_IN_PROD,sector_size=32,file=/vsimem/file_enc.bin'
ref_file = '/vsimem/file.bin'
for seed in range(1000):
gdal.Unlink(test_file)
gdal.Unlink(ref_file)
test_f = gdal.VSIFOpenL(test_file, 'wb+')
ref_f = gdal.VSIFOpenL(ref_file, 'wb+')
import random
random.seed(seed)
for i in range(20):
random_offset = random.randint(0, 1000)
gdal.VSIFSeekL(test_f, random_offset, 0)
gdal.VSIFSeekL(ref_f, random_offset, 0)
random_size = random.randint(1, 80)
random_content = ''.join([
chr(40 + int(10 * random.random())) for i in range(random_size)
])
gdal.VSIFWriteL(random_content, 1, random_size, test_f)
gdal.VSIFWriteL(random_content, 1, random_size, ref_f)
if random.randint(0, 1) == 0:
random_offset = random.randint(0, 1500)
gdal.VSIFSeekL(test_f, random_offset, 0)
gdal.VSIFSeekL(ref_f, random_offset, 0)
random_size = random.randint(1, 80)
test_content = gdal.VSIFReadL(1, random_size, test_f)
ref_content = gdal.VSIFReadL(1, random_size, ref_f)
if test_content != ref_content:
print(seed)
print('Test content (%d):' % len(test_content))
print(test_content)
print('')
print('Ref content (%d):' % len(ref_content))
print(ref_content)
return 'fail'
gdal.VSIFSeekL(test_f, 0, 0)
gdal.VSIFSeekL(ref_f, 0, 0)
test_content = gdal.VSIFReadL(1, 100000, test_f)
ref_content = gdal.VSIFReadL(1, 100000, ref_f)
if test_content != ref_content:
print(seed)
print('Test content (%d):' % len(test_content))
print(test_content)
print('')
print('Ref content (%d):' % len(ref_content))
print(ref_content)
return 'fail'
gdal.Unlink(test_file)
gdal.Unlink(ref_file)
return 'success'
def test_visoss_4():
if gdaltest.webserver_port == 0:
pytest.skip()
with webserver.install_http_handler(webserver.SequentialHandler()):
with gdaltest.error_handler():
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3', 'wb')
assert f is None
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_fake_bucket3/empty_file.bin', 200,
{'Connection': 'close'}, 'foo')
with webserver.install_http_handler(handler):
assert gdal.VSIStatL(
'/vsioss/oss_fake_bucket3/empty_file.bin').size == 3
# Empty file
handler = webserver.SequentialHandler()
def method(request):
if request.headers['Content-Length'] != '0':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/oss_fake_bucket3/empty_file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb')
assert f is not None
gdal.ErrorReset()
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == ''
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_fake_bucket3/empty_file.bin', 200,
{'Connection': 'close'}, '')
with webserver.install_http_handler(handler):
assert gdal.VSIStatL(
'/vsioss/oss_fake_bucket3/empty_file.bin').size == 0
# Invalid seek
handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
assert ret != 0
gdal.VSIFCloseL(f)
# Invalid read
handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
assert not ret
gdal.VSIFCloseL(f)
# Error case
handler = webserver.SequentialHandler()
handler.add('PUT', '/oss_fake_bucket3/empty_file_error.bin', 403)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file_error.bin',
'wb')
assert f is not None
gdal.ErrorReset()
with gdaltest.error_handler():
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() != ''
# Nominal case
with webserver.install_http_handler(webserver.SequentialHandler()):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/another_file.bin', 'wb')
assert f is not None
assert gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) == 0
assert gdal.VSIFSeekL(f, 0, 1) == 0
assert gdal.VSIFSeekL(f, 0, 2) == 0
assert gdal.VSIFWriteL('foo', 1, 3, f) == 3
assert gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) == 0
assert gdal.VSIFWriteL('bar', 1, 3, f) == 3
handler = webserver.SequentialHandler()
def method(request):
if request.headers['Content-Length'] != '6':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(6).decode('ascii')
if content != 'foobar':
sys.stderr.write('Did not get expected content: %s\n' % content)
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/oss_fake_bucket3/another_file.bin',
custom_method=method)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == ''
