def test_vsifile_7():
if gdal.GetConfigOption('SKIP_MEM_INTENSIVE_TEST') is not None:
pytest.skip()
# Test extending file beyond reasonable limits in write mode
fp = gdal.VSIFOpenL('/vsimem/vsifile_7.bin', 'wb')
assert gdal.VSIFSeekL(fp, 0x7FFFFFFFFFFFFFFF, 0) == 0
assert gdal.VSIStatL('/vsimem/vsifile_7.bin').size == 0
gdal.PushErrorHandler()
ret = gdal.VSIFWriteL('a', 1, 1, fp)
gdal.PopErrorHandler()
assert ret == 0
assert gdal.VSIStatL('/vsimem/vsifile_7.bin').size == 0
gdal.VSIFCloseL(fp)
# Test seeking beyond file size in read-only mode
fp = gdal.VSIFOpenL('/vsimem/vsifile_7.bin', 'rb')
assert gdal.VSIFSeekL(fp, 0x7FFFFFFFFFFFFFFF, 0) == 0
assert gdal.VSIFEofL(fp) == 0
assert gdal.VSIFTellL(fp) == 0x7FFFFFFFFFFFFFFF
assert not gdal.VSIFReadL(1, 1, fp)
assert gdal.VSIFEofL(fp) == 1
gdal.VSIFCloseL(fp)
gdal.Unlink('/vsimem/vsifile_7.bin')
def test_vsihdfs_5():
if gdaltest.have_vsihdfs == False:
pytest.skip()
filename = '/vsihdfs/file:' + os.getcwd() + '/data/text.txt'
fp = gdal.VSIFOpenL(filename, 'rb')
assert fp is not None
gdal.VSIFReadL(5, 1, fp)
eof = gdal.VSIFEofL(fp)
assert eof == 0
gdal.VSIFReadL(1000000, 1, fp)
eof = gdal.VSIFEofL(fp)
assert eof == 0
gdal.VSIFReadL(1, 1, fp)
eof = gdal.VSIFEofL(fp)
assert eof == 1
gdal.VSIFSeekL(fp, 0, 0)
eof = gdal.VSIFEofL(fp)
assert eof == 0
gdal.VSIFCloseL(fp)
def vsihdfs_5():
if gdaltest.have_vsihdfs == False:
return 'skip'
filename = '/vsihdfs/file:' + os.getcwd() + '/data/text.txt'
fp = gdal.VSIFOpenL(filename, 'rb')
if fp is None:
return 'fail'
gdal.VSIFReadL(5, 1, fp)
eof = gdal.VSIFEofL(fp)
if eof != 0:
return 'fail'
gdal.VSIFReadL(1000000, 1, fp)
eof = gdal.VSIFEofL(fp)
if eof != 0:
return 'fail'
gdal.VSIFReadL(1, 1, fp)
eof = gdal.VSIFEofL(fp)
if eof != 1:
return 'fail'
gdal.VSIFSeekL(fp, 0, 0)
eof = gdal.VSIFEofL(fp)
if eof != 0:
return 'fail'
gdal.VSIFCloseL(fp)
return 'success'
def test_vsifile_vsizip_stored():
f = gdal.VSIFOpenL('/vsizip/data/stored.zip/foo.txt', 'rb')
assert f
assert gdal.VSIFReadL(1, 5, f) == b'foo\n'
assert gdal.VSIFEofL(f)
gdal.VSIFCloseL(f)
def vsifile_7():
if gdal.GetConfigOption('SKIP_MEM_INTENSIVE_TEST') is not None:
return 'skip'
# Test extending file beyond reasonable limits in write mode
fp = gdal.VSIFOpenL('/vsimem/vsifile_7.bin', 'wb')
if gdal.VSIFSeekL(fp, 0x7FFFFFFFFFFFFFFF, 0) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsimem/vsifile_7.bin').size != 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.PushErrorHandler()
ret = gdal.VSIFWriteL('a', 1, 1, fp)
gdal.PopErrorHandler()
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsimem/vsifile_7.bin').size != 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(fp)
# Test seeking beyond file size in read-only mode
fp = gdal.VSIFOpenL('/vsimem/vsifile_7.bin', 'rb')
if gdal.VSIFSeekL(fp, 0x7FFFFFFFFFFFFFFF, 0) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFEofL(fp) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFTellL(fp) != 0x7FFFFFFFFFFFFFFF:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFReadL(1, 1, fp):
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFEofL(fp) != 1:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(fp)
gdal.Unlink('/vsimem/vsifile_7.bin')
return 'success'
def vsifile_15():
fp = gdal.VSIFOpenL('/vsigzip/data/corrupted_z_buf_error.gz', 'rb')
if fp is None:
return 'fail'
while not gdal.VSIFEofL(fp):
with gdaltest.error_handler():
gdal.VSIFReadL(1, 4, fp)
gdal.VSIFCloseL(fp)
return 'success'
def get_vrt_template(src_ds_file):
src_ds = gdal.Open(args.src_ds)
vrt_tpl_file = '/vsimem/vrt_template.vrt'
gdal.Translate(vrt_tpl_file, src_ds)
src_ds = None
mem_f = gdal.VSIFOpenL(vrt_tpl_file, 'rb')
vrt_template = ''
while not gdal.VSIFEofL(mem_f):
data = gdal.VSIFReadL(1, 10000, mem_f)
vrt_template += data.decode('ascii')
gdal.VSIFCloseL(mem_f)
gdal.Unlink(vrt_tpl_file)
return vrt_template
def test_vsifile_15():
fp = gdal.VSIFOpenL('/vsigzip/data/corrupted_z_buf_error.gz', 'rb')
assert fp is not None
file_len = 0
while not gdal.VSIFEofL(fp):
with gdaltest.error_handler():
file_len += len(gdal.VSIFReadL(1, 4, fp))
assert file_len == 6469
with gdaltest.error_handler():
file_len += len(gdal.VSIFReadL(1, 4, fp))
assert file_len == 6469
with gdaltest.error_handler():
assert gdal.VSIFSeekL(fp, 0, 2) != 0
assert gdal.VSIFSeekL(fp, 0, 0) == 0
len_read = len(gdal.VSIFReadL(1, file_len, fp))
assert len_read == file_len
gdal.VSIFCloseL(fp)
def vsifile_15():
fp = gdal.VSIFOpenL('/vsigzip/data/corrupted_z_buf_error.gz', 'rb')
if fp is None:
return 'fail'
file_len = 0
while not gdal.VSIFEofL(fp):
with gdaltest.error_handler():
file_len += len(gdal.VSIFReadL(1, 4, fp))
if file_len != 6469:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
file_len += len(gdal.VSIFReadL(1, 4, fp))
if file_len != 6469:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
if gdal.VSIFSeekL(fp, 0, 2) == 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFSeekL(fp, 0, 0) != 0:
gdaltest.post_reason('fail')
return 'fail'
len_read = len(gdal.VSIFReadL(1, file_len, fp))
if len_read != file_len:
gdaltest.post_reason('fail')
print(len_read)
return 'fail'
gdal.VSIFCloseL(fp)
return 'success'
def vsiaz_fake_write():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Test creation of BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:AigkrY7q66WCrx3JRKBte56k7kxV2cxB/ZyGNubxk5I=' or \
'Expect' not in h or h['Expect'] != '100-continue' or \
'Content-Length' not in h or h['Content-Length'] != '40000' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'BlockBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(40000).decode('ascii')
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Simulate illegal read
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
if len(ret) != 0:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
gdal.VSIFCloseL(f)
# Simulate illegal seek
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
# Simulate failure when putting BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
if gdal.VSIFSeekL(f, 0, 0) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if gdal.VSIFTellL(f) != 35000:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 35000, 0) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 0, 1) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 0, 2) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFEofL(f) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFCloseL(f)
if ret == 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
# Simulate creation of BlockBob over an existing blob of incompatible type
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 409)
handler.add('DELETE', '/azure/blob/myaccount/test_copy/file.bin', 202)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
# Test creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:KimVui3ptY9D5ftLlsI7CNOgK36CNAEzsXqcuHskdEY=' or \
'Content-Length' not in h or h['Content-Length'] != '0' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '10' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(10).decode('ascii')
if content != '0123456789':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '6' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(6).decode('ascii')
if content != 'abcdef':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 16:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Test failed creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Test failed writing of a block of an AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
403)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
return 'success'
def test_vsistdin_5():
f = open('tmp/test_vsistdin_5.bin', 'wb')
f.write(b'0123456789' * (1024 * 1024))
f.close()
with gdaltest.config_options({
'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin',
'CPL_VSISTDIN_RESET_POSITION': 'YES',
'CPL_VSISTDIN_FILE_CLOSE': 'YES'
}):
f = gdal.VSIFOpenL('/vsistdin?buffer_limit=10MB', 'rb')
assert f is not None
assert gdal.VSIFEofL(f) == 0
assert gdal.VSIFReadL(10, 1, f) == b'0123456789'
assert gdal.VSIFTellL(f) == 10
assert gdal.VSIFSeekL(f, 5, 0) == 0
assert gdal.VSIFTellL(f) == 5
assert gdal.VSIFReadL(3, 1, f) == b'567'
assert gdal.VSIFTellL(f) == 8
assert gdal.VSIFReadL(4, 1, f) == b'8901'
assert gdal.VSIFTellL(f) == 12
assert gdal.VSIFSeekL(f, 0, 2) == 0
assert gdal.VSIFTellL(f) == 10 * 1024 * 1024
assert gdal.VSIFReadL(1, 1, f) == b''
assert gdal.VSIFEofL(f) == 1
assert gdal.VSIFTellL(f) == 10 * 1024 * 1024
assert gdal.VSIFSeekL(f, 5, 0) == 0
assert gdal.VSIFTellL(f) == 5
assert gdal.VSIFReadL(3, 1, f) == b'567'
assert gdal.VSIFTellL(f) == 8
assert gdal.VSIFSeekL(f, 10 * 1024 * 1024 - 10 + 1, 0) == 0
assert gdal.VSIFReadL(3, 1, f) == b'123'
gdal.VSIFCloseL(f)
with gdaltest.config_options({
'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin',
'CPL_VSISTDIN_RESET_POSITION': 'YES',
'CPL_VSISTDIN_FILE_CLOSE': 'YES'
}):
f = gdal.VSIFOpenL('/vsistdin?buffer_limit=-1', 'rb')
assert f is not None
assert gdal.VSIFSeekL(f, 0, 2) == 0
assert gdal.VSIFTellL(f) == 10 * 1024 * 1024
gdal.VSIFCloseL(f)
assert gdal.VSIStatL(
'/vsistdin?buffer_limit=-1').size == 10 * 1024 * 1024
with gdaltest.config_options({
'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin',
'CPL_VSISTDIN_RESET_POSITION': 'YES',
'CPL_VSISTDIN_FILE_CLOSE': 'YES'
}):
f = gdal.VSIFOpenL('/vsistdin?buffer_limit=1GB', 'rb')
assert f is not None
assert gdal.VSIFSeekL(f, 0, 2) == 0
assert gdal.VSIFTellL(f) == 10 * 1024 * 1024
gdal.VSIFCloseL(f)
assert gdal.VSIStatL(
'/vsistdin?buffer_limit=-1').size == 10 * 1024 * 1024
with gdaltest.config_options({
'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin',
'CPL_VSISTDIN_RESET_POSITION': 'YES',
'CPL_VSISTDIN_FILE_CLOSE': 'YES'
}):
f = gdal.VSIFOpenL('/vsistdin?buffer_limit=10', 'rb')
assert f is not None
assert gdal.VSIFSeekL(f, 0, 2) == 0
assert gdal.VSIFTellL(f) == 10 * 1024 * 1024
assert gdal.VSIFSeekL(f, 0, 0) == 0
assert gdal.VSIFReadL(5, 1, f) == b'01234'
assert gdal.VSIFReadL(3, 1, f) == b'567'
with gdaltest.error_handler():
assert gdal.VSIFReadL(3, 1, f) == b''
gdal.VSIFCloseL(f)
os.unlink('tmp/test_vsistdin_5.bin')
