def vsiswift_fake_auth_storage_url_and_auth_token():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
gdal.SetConfigOption('SWIFT_AUTH_V1_URL', '')
gdal.SetConfigOption('SWIFT_USER', '')
gdal.SetConfigOption('SWIFT_KEY', '')
gdal.SetConfigOption(
'SWIFT_STORAGE_URL',
'http://127.0.0.1:%d/v1/AUTH_something' % gdaltest.webserver_port)
gdal.SetConfigOption('SWIFT_AUTH_TOKEN', 'my_auth_token')
# Failure
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 501)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiswift/foo/bar')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
gdal.VSICurlClearCache()
# Success
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'x-auth-token' not in h or \
h['x-auth-token'] != 'my_auth_token':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiswift/foo/bar')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
return 'success'
def vsiwebhdfs_unlink():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Success
handler = webserver.SequentialHandler()
handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE', 200,
{}, '{"boolean":true}')
with webserver.install_http_handler(handler):
ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar')
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSICurlClearCache()
# With permissions
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE&user.name=root&delegation=token', 200,
{}, '{"boolean":true}')
with gdaltest.config_options({'WEBHDFS_USERNAME': '******',
'WEBHDFS_DELEGATION': 'token'}):
with webserver.install_http_handler(handler):
ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar')
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Failure
handler = webserver.SequentialHandler()
handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE', 200,
{}, '{"boolean":false}')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar')
if ret != -1:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSICurlClearCache()
# Failure
handler = webserver.SequentialHandler()
handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE', 404,
{})
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar')
if ret != -1:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def test_vsiadls_fake_sync_copyobject():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/test_bucket/src.txt', 200, {'Content-Length': '3', 'x-ms-permissions': 'rwxrwxrwx', 'x-ms-resource-type': 'file'})
handler.add('HEAD', '/azure/blob/myaccount/test_bucket/dst.txt', 404)
handler.add('PUT', '/azure/blob/myaccount/test_bucket/dst.txt', 202,
expected_headers={'x-ms-copy-source': 'http://127.0.0.1:%d/azure/blob/myaccount/test_bucket/src.txt' % gdaltest.webserver_port})
with webserver.install_http_handler(handler):
assert gdal.Sync('/vsiadls/test_bucket/src.txt',
'/vsiadls/test_bucket/dst.txt')
gdal.VSICurlClearCache()
# Error case
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/test_bucket/src.txt', 200, {'Content-Length': '3', 'x-ms-permissions': 'rwxrwxrwx', 'x-ms-resource-type': 'file'})
handler.add('HEAD', '/azure/blob/myaccount/test_bucket/dst.txt', 404)
handler.add('PUT', '/azure/blob/myaccount/test_bucket/dst.txt', 400)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
assert not gdal.Sync('/vsiadls/test_bucket/src.txt',
'/vsiadls/test_bucket/dst.txt')
def vsicurl_test_retry():
if gdaltest.webserver_port == 0:
return 'skip'
f = gdal.VSIFOpenL(
'/vsicurl/http://localhost:%d/test_retry_reset_counter' %
gdaltest.webserver_port, 'rb')
if f:
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
f = gdal.VSIFOpenL(
'/vsicurl/http://localhost:%d/test_retry/test.txt' %
gdaltest.webserver_port, 'rb')
data_len = 0
if f:
data_len = len(gdal.VSIFReadL(1, 1, f))
gdal.VSIFCloseL(f)
if data_len != 0:
gdaltest.post_reason('fail')
print(data_len)
return 'fail'
gdal.VSICurlClearCache()
f = gdal.VSIFOpenL(
'/vsicurl/http://localhost:%d/test_retry_reset_counter' %
gdaltest.webserver_port, 'rb')
if f:
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
gdal.VSICurlClearCache()
f = gdal.VSIFOpenL(
'/vsicurl/max_retry=1,retry_delay=0.1,url=http://localhost:%d/test_retry/test.txt'
% gdaltest.webserver_port, 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
error_msg = gdal.GetLastErrorMsg()
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
if error_msg.find('502') < 0:
gdaltest.post_reason('fail')
print(error_msg)
return 'fail'
return 'success'
def test_vsiwebhdfs_open():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
# Download without redirect (not nominal)
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1/foo/bar?op=OPEN&offset=9999990784&length=16384', 200,
{}, '0123456789data')
with webserver.install_http_handler(handler):
f = open_for_read(gdaltest.webhdfs_base_connection + '/foo/bar')
assert f is not None
gdal.VSIFSeekL(f, 9999990784 + 10, 0)
assert gdal.VSIFReadL(1, 4, f).decode('ascii') == 'data'
gdal.VSIFCloseL(f)
# Download with redirect (nominal) and permissions
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384&user.name=root&delegation=token', 307,
{'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384'})
handler.add('GET', '/redirected/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384', 200,
{}, 'yeah')
with gdaltest.config_options({'WEBHDFS_USERNAME': '******',
'WEBHDFS_DELEGATION': 'token',
'WEBHDFS_DATANODE_HOST': 'localhost'}):
with webserver.install_http_handler(handler):
f = open_for_read(gdaltest.webhdfs_base_connection + '/foo/bar')
assert f is not None
assert gdal.VSIFReadL(1, 4, f).decode('ascii') == 'yeah'
gdal.VSIFCloseL(f)
# Test error
gdal.VSICurlClearCache()
f = open_for_read(gdaltest.webhdfs_base_connection + '/foo/bar')
assert f is not None
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384', 404)
with webserver.install_http_handler(handler):
assert len(gdal.VSIFReadL(1, 4, f)) == 0
# Retry: shouldn't not cause network access
assert len(gdal.VSIFReadL(1, 4, f)) == 0
gdal.VSIFCloseL(f)
def test_vsicurl_test_fallback_from_head_to_get():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('HEAD', '/test_fallback_from_head_to_get', 405)
handler.add('GET', '/test_fallback_from_head_to_get', 200, {}, 'foo')
with webserver.install_http_handler(handler):
statres = gdal.VSIStatL('/vsicurl/http://localhost:%d/test_fallback_from_head_to_get' % gdaltest.webserver_port)
assert statres.size == 3
gdal.VSICurlClearCache()
def test_vsiadls_fake_sync_multithreaded_upload_single_file():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
gdal.Mkdir('/vsimem/test', 0)
gdal.FileFromMemBuffer('/vsimem/test/foo', 'foo\n')
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/test_bucket?resource=filesystem', 200)
handler.add('HEAD', '/azure/blob/myaccount/test_bucket/foo', 404)
handler.add('PUT', '/azure/blob/myaccount/test_bucket/foo?resource=file', 201)
handler.add('PATCH', '/azure/blob/myaccount/test_bucket/foo?action=append&position=0', 202, expected_body = b'foo')
handler.add('PATCH', '/azure/blob/myaccount/test_bucket/foo?action=append&position=3', 202, expected_body = b'\n')
handler.add('PATCH', '/azure/blob/myaccount/test_bucket/foo?action=flush&close=true&position=4', 200)
with gdaltest.config_option('VSIS3_SIMULATE_THREADING', 'YES'):
with webserver.install_http_handler(handler):
assert gdal.Sync('/vsimem/test/foo',
'/vsiadls/test_bucket',
options=['NUM_THREADS=1', 'CHUNK_SIZE=3'])
gdal.RmdirRecursive('/vsimem/test')
def vsiswift_fake_unlink():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Success
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 206,
{'Content-Range': 'bytes 0-0/1'} , 'x')
handler.add('DELETE', '/v1/AUTH_something/foo/bar', 202, {'Connection':'close'})
with webserver.install_http_handler(handler):
ret = gdal.Unlink('/vsiswift/foo/bar')
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Failure
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 206,
{'Content-Range': 'bytes 0-0/1'} , 'x')
handler.add('DELETE', '/v1/AUTH_something/foo/bar', 400, {'Connection':'close'})
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink('/vsiswift/foo/bar')
if ret != -1:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def test_vsiaz_write_appendblob_retry():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
with gdaltest.config_options({
'GDAL_HTTP_MAX_RETRY': '2',
'GDAL_HTTP_RETRY_DELAY': '0.01',
'VSIAZ_CHUNK_SIZE_BYTES': '10'
}):
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 502)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
handler.add(
'PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
502)
handler.add(
'PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
201)
handler.add(
'PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
502)
handler.add(
'PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
201)
with webserver.install_http_handler(handler):
assert gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) == 16
gdal.VSIFCloseL(f)
def vsigs_read_credentials_file():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '/vsimem/.boto')
gdal.VSICurlClearCache()
gdal.FileFromMemBuffer('/vsimem/.boto', """
[unrelated]
gs_access_key_id = foo
gs_secret_access_key = bar
[Credentials]
gs_access_key_id = GS_ACCESS_KEY_ID
gs_secret_access_key = GS_SECRET_ACCESS_KEY
[unrelated]
gs_access_key_id = foo
gs_secret_access_key = bar
""")
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'GOOG1 GS_ACCESS_KEY_ID:8tndu9//BfmN+Kg4AFLdUMZMBDQ='
if request.headers['Authorization'] != expected_authorization :
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET', '/gs_fake_bucket/resource', custom_method = method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '')
gdal.Unlink('/vsimem/.boto')
return 'success'
def test_vsicurl_test_retry():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_retry/', 404)
handler.add('HEAD', '/test_retry/test.txt', 200, {'Content-Length': '3'})
handler.add('GET', '/test_retry/test.txt', 502)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsicurl/http://localhost:%d/test_retry/test.txt' % gdaltest.webserver_port, 'rb')
data_len = 0
if f:
data_len = len(gdal.VSIFReadL(1, 1, f))
gdal.VSIFCloseL(f)
assert data_len == 0
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_retry/', 404)
handler.add('HEAD', '/test_retry/test.txt', 200, {'Content-Length': '3'})
handler.add('GET', '/test_retry/test.txt', 502)
handler.add('GET', '/test_retry/test.txt', 429)
handler.add('GET', '/test_retry/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsicurl?max_retry=2&retry_delay=0.01&url=http://localhost:%d/test_retry/test.txt' % gdaltest.webserver_port, 'rb')
assert f is not None
gdal.ErrorReset()
with gdaltest.error_handler():
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
error_msg = gdal.GetLastErrorMsg()
gdal.VSIFCloseL(f)
assert data == 'foo'
assert '429' in error_msg
def test_vsiaz_write_blockblob_retry():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
# Test creation of BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
with gdaltest.config_options({'GDAL_HTTP_MAX_RETRY': '2',
'GDAL_HTTP_RETRY_DELAY': '0.01'}):
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(3).decode('ascii')
if len(content) != 3:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 502)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
assert gdal.VSIFWriteL('foo', 1, 3, f) == 3
gdal.VSIFCloseL(f)
def test_vsiaz_fake_rename():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/test/source.txt', 200,
{'Content-Length': '3'})
def method(request):
if request.headers['Content-Length'] != '0':
sys.stderr.write('Did not get expected headers: %s\n' % str(request.headers))
request.send_response(400)
return
expected = 'http://127.0.0.1:%d/azure/blob/myaccount/test/source.txt' % gdaltest.webserver_port
if request.headers['x-ms-copy-source'] != expected:
sys.stderr.write('Did not get expected headers: %s\n' % str(request.headers))
request.send_response(400)
return
request.send_response(202)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/azure/blob/myaccount/test/target.txt', custom_method=method)
handler.add('DELETE', '/azure/blob/myaccount/test/source.txt', 202)
with webserver.install_http_handler(handler):
assert gdal.Rename( '/vsiaz/test/source.txt', '/vsiaz/test/target.txt') == 0
def gdalRead(infile):
src = gdal.Open(infile)
if infile.split('.')[-1] in ['jpg', 'png']:
Info = {
'projection': '-',
'geotransform': '-',
'EPSG': '-',
'datatype': gdal.GetDataTypeName(src.GetRasterBand(1).DataType),
'columns': src.RasterXSize,
'rows': src.RasterYSize,
'numofbands': src.RasterCount
}
else:
proj = osr.SpatialReference(wkt=src.GetProjection())
Info = {
'projection': src.GetProjection(),
'geotransform': src.GetGeoTransform(),
'EPSG': proj.GetAttrValue('AUTHORITY', 1),
'datatype': gdal.GetDataTypeName(src.GetRasterBand(1).DataType),
'columns': src.RasterXSize,
'rows': src.RasterYSize,
'numofbands': src.RasterCount
}
Image = np.zeros((src.RasterYSize, src.RasterXSize, src.RasterCount),
dtype=NP2GDAL_CONVERSION[Info['datatype']])
for band in range(Image.shape[2]):
Image[:, :, band] = src.GetRasterBand(band + 1).ReadAsArray()
del src
gdal.VSICurlClearCache()
return Info, np.squeeze(Image)
def test_vsiwebhdfs_readdir():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/webhdfs/v1/foo/?op=LISTSTATUS', 200, {},
'{"FileStatuses":{"FileStatus":[{"type":"FILE","modificationTime":1000,"pathSuffix":"bar.baz","length":123456},{"type":"DIRECTORY","pathSuffix":"mysubdir","length":0}]}}'
)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir(gdaltest.webhdfs_base_connection + '/foo')
assert dir_contents == ['bar.baz', 'mysubdir']
stat_res = gdal.VSIStatL(gdaltest.webhdfs_base_connection + '/foo/bar.baz')
assert stat_res.size == 123456
assert stat_res.mtime == 1
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir(gdaltest.webhdfs_base_connection +
'/foo/bar.baz')
assert dir_contents is None
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1foo/error_test/?op=LISTSTATUS', 404)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir(gdaltest.webhdfs_base_connection +
'foo/error_test/')
assert dir_contents is None
def test_vsiwebhdfs_stat():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1/foo/bar?op=GETFILESTATUS', 200, {},
'{"FileStatus":{"type":"FILE","length":1000000}}')
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL(gdaltest.webhdfs_base_connection + '/foo/bar')
if stat_res is None or stat_res.size != 1000000:
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
pytest.fail()
# Test caching
stat_res = gdal.VSIStatL(gdaltest.webhdfs_base_connection + '/foo/bar')
assert stat_res.size == 1000000
# Test missing file
handler = webserver.SequentialHandler()
handler.add(
'GET', '/webhdfs/v1/unexisting?op=GETFILESTATUS', 404, {},
'{"RemoteException":{"exception":"FileNotFoundException","javaClassName":"java.io.FileNotFoundException","message":"File does not exist: /unexisting"}}'
)
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL(gdaltest.webhdfs_base_connection +
'/unexisting')
assert stat_res is None
def test_vsiswift_fake_unlink():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
# Success
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 206,
{'Content-Range': 'bytes 0-0/1'}, 'x')
handler.add('DELETE', '/v1/AUTH_something/foo/bar', 202,
{'Connection': 'close'})
with webserver.install_http_handler(handler):
ret = gdal.Unlink('/vsiswift/foo/bar')
assert ret == 0
# Failure
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 206,
{'Content-Range': 'bytes 0-0/1'}, 'x')
handler.add('DELETE', '/v1/AUTH_something/foo/bar', 400,
{'Connection': 'close'})
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink('/vsiswift/foo/bar')
assert ret == -1
def test_vsiadls_fake_unlink():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
# Success
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_unlink/myfile',
200, {'Content-Length': '1'})
handler.add('DELETE', '/azure/blob/myaccount/az_bucket_test_unlink/myfile',
200, {'Connection': 'close'})
with webserver.install_http_handler(handler):
ret = gdal.Unlink('/vsiadls/az_bucket_test_unlink/myfile')
assert ret == 0
# Failure
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_unlink/myfile',
200, {'Content-Length': '1'})
handler.add('DELETE', '/azure/blob/myaccount/az_bucket_test_unlink/myfile',
400, {'Connection': 'close'})
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink('/vsiadls/az_bucket_test_unlink/myfile')
assert ret == -1
def test_eedai_gce_credentials():
if gdaltest.eedai_drv is None:
pytest.skip()
gdaltest.webserver_process = None
gdaltest.webserver_port = 0
if not gdaltest.built_against_curl():
pytest.skip()
(gdaltest.webserver_process, gdaltest.webserver_port) = webserver.launch(handler=webserver.DispatcherHttpHandler)
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL',
'http://localhost:%d/computeMetadata/v1/instance/service-accounts/default/token' % gdaltest.webserver_port)
# Disable hypervisor related check to test if we are really on EC2
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', 'NO')
gdal.VSICurlClearCache()
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET', '/computeMetadata/v1/instance/service-accounts/default/token', 200, {},
"""{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}""")
with webserver.install_http_handler(handler):
gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/')
ds = gdal.Open('EEDAI:image')
gdal.SetConfigOption('EEDA_URL', None)
gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL', None)
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', None)
webserver.server_stop(gdaltest.webserver_process, gdaltest.webserver_port)
assert ds is not None
def vsis3_read_credentials_config_file_non_default():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('AWS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('AWS_ACCESS_KEY_ID', '')
gdal.SetConfigOption('CPL_AWS_CREDENTIALS_FILE', '/vsimem/aws_credentials')
gdal.SetConfigOption('AWS_CONFIG_FILE', '/vsimem/aws_config')
gdal.SetConfigOption('AWS_DEFAULT_PROFILE', 'myprofile')
gdal.VSICurlClearCache()
gdal.FileFromMemBuffer(
'/vsimem/aws_credentials', """
[unrelated]
aws_access_key_id = foo
aws_secret_access_key = bar
[myprofile]
aws_access_key_id = AWS_ACCESS_KEY_ID
aws_secret_access_key = AWS_SECRET_ACCESS_KEY
[default]
aws_access_key_id = foo
aws_secret_access_key = bar
""")
gdal.FileFromMemBuffer(
'/vsimem/aws_config', """
[unrelated]
aws_access_key_id = foo
aws_secret_access_key = bar
[profile myprofile]
region = us-east-1
[default]
aws_access_key_id = foo
aws_secret_access_key = bar
""")
f = open_for_read('/vsis3/s3_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.SetConfigOption('CPL_AWS_CREDENTIALS_FILE', '')
gdal.Unlink('/vsimem/aws_credentials')
gdal.SetConfigOption('AWS_CONFIG_FILE', '')
gdal.Unlink('/vsimem/aws_config')
gdal.SetConfigOption('AWS_DEFAULT_PROFILE', '')
return 'success'
def vsicurl_test_fallback_from_head_to_get():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('HEAD', '/test_fallback_from_head_to_get', 405)
handler.add('GET', '/test_fallback_from_head_to_get', 200, {}, 'foo' )
with webserver.install_http_handler(handler):
statres = gdal.VSIStatL('/vsicurl/http://localhost:%d/test_fallback_from_head_to_get' % gdaltest.webserver_port)
if statres.size != 3:
return 'fail'
gdal.VSICurlClearCache()
return 'success'
def vsiswift_fake_write():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Test creation of BlockBob
f = gdal.VSIFOpenL('/vsiswift/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'x-auth-token' not in h or \
h['x-auth-token'] != 'my_auth_token' or \
'Transfer-Encoding' not in h or h['Transfer-Encoding'] != 'chunked':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = ''
while True:
numchars = int(request.rfile.readline().strip(), 16)
content += request.rfile.read(numchars).decode('ascii')
request.rfile.read(2)
if numchars == 0:
break
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/v1/AUTH_something/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
return 'success'
def test_vsiadls_fake_mkdir_rmdir():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
# Invalid name
ret = gdal.Mkdir('/vsiadls', 0)
assert ret != 0
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_mkdir/dir', 404, {'Connection': 'close'})
handler.add('PUT', '/azure/blob/myaccount/az_bucket_test_mkdir/dir?resource=directory', 201)
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsiadls/az_bucket_test_mkdir/dir', 0)
assert ret == 0
# Try creating already existing directory
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_mkdir/dir', 200, {'x-ms-permissions': 'rwxrwxrwx', 'x-ms-resource-type': 'directory' } )
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsiadls/az_bucket_test_mkdir/dir', 0)
assert ret != 0
# Invalid name
ret = gdal.Rmdir('/vsiadls')
assert ret != 0
# Not a directory
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_mkdir/it_is_a_file', 200, {'x-ms-permissions': 'rwxrwxrwx', 'x-ms-resource-type': 'file' } )
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiadls/az_bucket_test_mkdir/it_is_a_file')
assert ret != 0
# Valid
handler = webserver.SequentialHandler()
handler.add('DELETE', '/azure/blob/myaccount/az_bucket_test_mkdir/dir?recursive=false', 200)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiadls/az_bucket_test_mkdir/dir')
assert ret == 0
# Try deleting already deleted directory
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_mkdir/dir', 404 )
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiadls/az_bucket_test_mkdir/dir')
assert ret != 0
# RmdirRecursive
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_mkdir/dir_rec', 200, {'x-ms-permissions': 'rwxrwxrwx', 'x-ms-resource-type': 'directory' } )
handler.add('DELETE', '/azure/blob/myaccount/az_bucket_test_mkdir/dir_rec?recursive=true', 200)
with webserver.install_http_handler(handler):
ret = gdal.RmdirRecursive('/vsiadls/az_bucket_test_mkdir/dir_rec')
assert ret == 0
def test_vsicurl_stop_webserver():
if gdaltest.webserver_port == 0:
pytest.skip()
# Clearcache needed to close all connections, since the Python server
# can only handle one connection at a time
gdal.VSICurlClearCache()
webserver.server_stop(gdaltest.webserver_process, gdaltest.webserver_port)
def test_vsiadls_fake_basic():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
signed_url = gdal.GetSignedURL('/vsiadls/az_fake_bucket/resource',
['START_DATE=20180213T123456'])
assert (signed_url in (
'http://127.0.0.1:8080/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12',
'http://127.0.0.1:8081/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12'
))
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:C0sSaBzGbvadfuuMMjQiHCXCUzsGWj3uuE+UO8dDl0U=' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET',
'/azure/blob/myaccount/az_fake_bucket/resource',
custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiadls/az_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin',
200, {'Content-Length': '1000000'})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsiadls/az_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
pytest.fail()
def test_vsicurl_test_CPL_CURL_VERBOSE():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
class MyHandler:
def __init__(self):
self.found_CURL_INFO = False
self.found_CURL_INFO_HEADER_IN = False
self.found_CURL_INFO_HEADER_OUT = False
def handler(self, err_type, err_no, err_msg):
if 'CURL_INFO_TEXT:' in err_msg:
self.found_CURL_INFO_TEXT = True
if 'CURL_INFO_HEADER_IN:' in err_msg:
self.found_CURL_INFO_HEADER_IN = True
if 'CURL_INFO_HEADER_OUT:' in err_msg:
self.found_CURL_INFO_HEADER_OUT = True
handler = webserver.SequentialHandler()
handler.add('HEAD', '/test_vsicurl_test_CPL_CURL_VERBOSE', 200,
{'Content-Length': '3'})
my_error_handler = MyHandler()
with gdaltest.config_options({
'CPL_CURL_VERBOSE': 'YES',
'CPL_DEBUG': 'ON'
}):
with gdaltest.error_handler(my_error_handler.handler):
with webserver.install_http_handler(handler):
statres = gdal.VSIStatL(
'/vsicurl/http://localhost:%d/test_vsicurl_test_CPL_CURL_VERBOSE'
% gdaltest.webserver_port)
assert statres.size == 3
assert my_error_handler.found_CURL_INFO_TEXT
assert my_error_handler.found_CURL_INFO_HEADER_IN
assert my_error_handler.found_CURL_INFO_HEADER_OUT
gdal.VSICurlClearCache()
def vsiaz_stop_webserver():
if gdaltest.webserver_port == 0:
return 'skip'
# Clearcache needed to close all connections, since the Python server
# can only handle one connection at a time
gdal.VSICurlClearCache()
webserver.server_stop(gdaltest.webserver_process, gdaltest.webserver_port)
return 'success'
def vsicurl_test_retry():
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add('GET', '/test_retry/', 404)
handler.add('HEAD', '/test_retry/test.txt', 200, {'Content-Length': '3'})
handler.add('GET', '/test_retry/test.txt', 502)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL(
'/vsicurl/http://localhost:%d/test_retry/test.txt' %
gdaltest.webserver_port, 'rb')
data_len = 0
if f:
data_len = len(gdal.VSIFReadL(1, 1, f))
gdal.VSIFCloseL(f)
if data_len != 0:
gdaltest.post_reason('fail')
print(data_len)
return 'fail'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_retry/', 404)
handler.add('HEAD', '/test_retry/test.txt', 200, {'Content-Length': '3'})
handler.add('GET', '/test_retry/test.txt', 502)
handler.add('GET', '/test_retry/test.txt', 429)
handler.add('GET', '/test_retry/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL(
'/vsicurl?max_retry=2&retry_delay=0.01&url=http://localhost:%d/test_retry/test.txt'
% gdaltest.webserver_port, 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
error_msg = gdal.GetLastErrorMsg()
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
if error_msg.find('429') < 0:
gdaltest.post_reason('fail')
print(error_msg)
return 'fail'
return 'success'
def test_vsiadls_fake_metadata():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/test/foo.bin', 200, {'Content-Length': '3', 'x-ms-permissions': 'rwxrwxrwx', 'x-ms-resource-type': 'file'})
with webserver.install_http_handler(handler):
md = gdal.GetFileMetadata('/vsiadls/test/foo.bin', 'HEADERS')
assert 'x-ms-permissions' in md
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/test/foo.bin?action=getStatus', 200, {'foo': 'bar'})
with webserver.install_http_handler(handler):
md = gdal.GetFileMetadata('/vsiadls/test/foo.bin', 'STATUS')
assert 'foo' in md
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/test/foo.bin?action=getAccessControl', 200, {'x-ms-acl': 'some_acl'})
with webserver.install_http_handler(handler):
md = gdal.GetFileMetadata('/vsiadls/test/foo.bin', 'ACL')
assert 'x-ms-acl' in md
# Error case
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/test/foo.bin?action=getStatus', 404)
with webserver.install_http_handler(handler):
assert gdal.GetFileMetadata('/vsiadls/test/foo.bin', 'STATUS') == {}
# SetMetadata()
handler = webserver.SequentialHandler()
handler.add('PATCH', '/azure/blob/myaccount/test/foo.bin?action=setProperties', 200, expected_headers={'x-ms-properties': 'foo=bar'})
with webserver.install_http_handler(handler):
assert gdal.SetFileMetadata('/vsiadls/test/foo.bin', {'x-ms-properties': 'foo=bar'}, 'PROPERTIES')
handler = webserver.SequentialHandler()
handler.add('PATCH', '/azure/blob/myaccount/test/foo.bin?action=setAccessControl', 200, expected_headers={'x-ms-acl': 'foo'})
with webserver.install_http_handler(handler):
assert gdal.SetFileMetadata('/vsiadls/test/foo.bin', {'x-ms-acl': 'foo'}, 'ACL')
handler = webserver.SequentialHandler()
handler.add('PATCH', '/azure/blob/myaccount/test/foo.bin?action=setAccessControlRecursive&mode=set', 200, expected_headers={'x-ms-acl': 'foo'})
with webserver.install_http_handler(handler):
assert gdal.SetFileMetadata('/vsiadls/test/foo.bin', {'x-ms-acl': 'foo'}, 'ACL', ['RECURSIVE=TRUE', 'MODE=set'])
# Error case
handler = webserver.SequentialHandler()
handler.add('PATCH', '/azure/blob/myaccount/test/foo.bin?action=setProperties', 404)
with webserver.install_http_handler(handler):
assert not gdal.SetFileMetadata('/vsiadls/test/foo.bin', {'x-ms-properties': 'foo=bar'}, 'PROPERTIES')
def test_vsiswift_stat():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 206,
{'Content-Range': 'bytes 0-0/1000000'}, 'x')
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsiswift/foo/bar')
if stat_res is None or stat_res.size != 1000000:
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
pytest.fail()
handler = webserver.SequentialHandler()
handler.add('HEAD', '/v1/AUTH_something/foo/bar', 200,
{'Content-Length': '1000000'})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsiswift_streaming/foo/bar')
if stat_res is None or stat_res.size != 1000000:
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
pytest.fail()
# Test stat on container
handler = webserver.SequentialHandler()
# GET on the container URL returns something, but we must hack this back
# to a directory
handler.add('GET', '/v1/AUTH_something/foo', 200, {}, "blabla")
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsiswift/foo')
assert stat_res is not None and stat.S_ISDIR(stat_res.mode)
# No network access done
s = gdal.VSIStatL(
'/vsiswift/foo', gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG | gdal.VSI_STAT_CACHE_ONLY)
assert s
assert stat.S_ISDIR(s.mode)
# No network access done
assert gdal.VSIStatL(
'/vsiswift/i_do_not_exist',
gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG | gdal.VSI_STAT_CACHE_ONLY) is None