def ogr_wfs3_empty_layer():
if gdaltest.wfs3_drv is None:
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200,
{'Content-Type': 'application/json'},
'{ "collections" : [ { "name": "foo" }] }')
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' % gdaltest.webserver_port)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayerCount() != 1:
gdaltest.post_reason('fail')
return 'fail'
lyr = ds.GetLayer(0)
if lyr.GetName() != 'foo':
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
'{ "type": "FeatureCollection", "features": [] }')
with webserver.install_http_handler(handler):
if lyr.GetLayerDefn().GetFieldCount() != 0:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsiwebhdfs_unlink():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Success
handler = webserver.SequentialHandler()
handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE', 200,
{}, '{"boolean":true}')
with webserver.install_http_handler(handler):
ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar')
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSICurlClearCache()
# With permissions
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE&user.name=root&delegation=token', 200,
{}, '{"boolean":true}')
with gdaltest.config_options({'WEBHDFS_USERNAME': '******',
'WEBHDFS_DELEGATION': 'token'}):
with webserver.install_http_handler(handler):
ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar')
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Failure
handler = webserver.SequentialHandler()
handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE', 200,
{}, '{"boolean":false}')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar')
if ret != -1:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSICurlClearCache()
# Failure
handler = webserver.SequentialHandler()
handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE', 404,
{})
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar')
if ret != -1:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def test_ogr_wfs3_get_feature_count():
if gdaltest.wfs3_drv is None:
pytest.skip()
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200, {'Content-Type': 'application/json'},
"""{ "collections" : [ {
"name": "foo",
"extent": {
"spatial": [ -10, 40, 15, 50 ]
}
}] }""")
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' % gdaltest.webserver_port)
lyr = ds.GetLayer(0)
handler = webserver.SequentialHandler()
# Fake openapi response
handler.add('GET', '/wfs3/api', 200, {'Content-Type': 'application/json'},
"""{
"openapi": "3.0.0",
"paths" : {
"/collections/foo/items": {
"get": {
"parameters": [
{
"$ref" : "#/components/parameters/resultType"
}
]
}
}
},
"components": {
"parameters": {
"resultType": {
"name": "resultType",
"in": "query",
"schema" : {
"type" : "string",
"enum" : [ "hits", "results" ]
}
}
}
}
}""")
handler.add('GET', '/wfs3/collections/foo/items?resultType=hits', 200,
{'Content-Type': 'application/json'},
'{ "numberMatched": 1234 }')
with webserver.install_http_handler(handler):
assert lyr.GetFeatureCount() == 1234
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections/foo/items?resultType=hits', 200,
{'Content-Type': 'application/json'},
'{ "numberMatched": 1234 }')
with webserver.install_http_handler(handler):
assert lyr.GetFeatureCount() == 1234
def test_vsicurl_test_retry():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_retry/', 404)
handler.add('HEAD', '/test_retry/test.txt', 200, {'Content-Length': '3'})
handler.add('GET', '/test_retry/test.txt', 502)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsicurl/http://localhost:%d/test_retry/test.txt' % gdaltest.webserver_port, 'rb')
data_len = 0
if f:
data_len = len(gdal.VSIFReadL(1, 1, f))
gdal.VSIFCloseL(f)
assert data_len == 0
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_retry/', 404)
handler.add('HEAD', '/test_retry/test.txt', 200, {'Content-Length': '3'})
handler.add('GET', '/test_retry/test.txt', 502)
handler.add('GET', '/test_retry/test.txt', 429)
handler.add('GET', '/test_retry/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsicurl?max_retry=2&retry_delay=0.01&url=http://localhost:%d/test_retry/test.txt' % gdaltest.webserver_port, 'rb')
assert f is not None
gdal.ErrorReset()
with gdaltest.error_handler():
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
error_msg = gdal.GetLastErrorMsg()
gdal.VSIFCloseL(f)
assert data == 'foo'
assert '429' in error_msg
def test_vsiwebhdfs_stat():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1/foo/bar?op=GETFILESTATUS', 200,
{}, '{"FileStatus":{"type":"FILE","length":1000000}}')
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL(gdaltest.webhdfs_base_connection + '/foo/bar')
if stat_res is None or stat_res.size != 1000000:
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
pytest.fail()
# Test caching
stat_res = gdal.VSIStatL(gdaltest.webhdfs_base_connection + '/foo/bar')
assert stat_res.size == 1000000
# Test missing file
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1/unexisting?op=GETFILESTATUS', 404, {},
'{"RemoteException":{"exception":"FileNotFoundException","javaClassName":"java.io.FileNotFoundException","message":"File does not exist: /unexisting"}}')
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL(
gdaltest.webhdfs_base_connection + '/unexisting')
assert stat_res is None
def test_vsiwebhdfs_readdir():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1/foo/?op=LISTSTATUS', 200,
{}, '{"FileStatuses":{"FileStatus":[{"type":"FILE","modificationTime":1000,"pathSuffix":"bar.baz","length":123456},{"type":"DIRECTORY","pathSuffix":"mysubdir","length":0}]}}')
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir(gdaltest.webhdfs_base_connection + '/foo')
assert dir_contents == ['bar.baz', 'mysubdir']
stat_res = gdal.VSIStatL(gdaltest.webhdfs_base_connection + '/foo/bar.baz')
assert stat_res.size == 123456
assert stat_res.mtime == 1
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir(
gdaltest.webhdfs_base_connection + '/foo/bar.baz')
assert dir_contents is None
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1foo/error_test/?op=LISTSTATUS', 404)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir(
gdaltest.webhdfs_base_connection + 'foo/error_test/')
assert dir_contents is None
def test_vsiswift_fake_unlink():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
# Success
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 206,
{'Content-Range': 'bytes 0-0/1'}, 'x')
handler.add('DELETE', '/v1/AUTH_something/foo/bar', 202, {'Connection': 'close'})
with webserver.install_http_handler(handler):
ret = gdal.Unlink('/vsiswift/foo/bar')
assert ret == 0
# Failure
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 206,
{'Content-Range': 'bytes 0-0/1'}, 'x')
handler.add('DELETE', '/v1/AUTH_something/foo/bar', 400, {'Connection': 'close'})
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink('/vsiswift/foo/bar')
assert ret == -1
def vsiswift_fake_unlink():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Success
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 206,
{'Content-Range': 'bytes 0-0/1'}, 'x')
handler.add('DELETE', '/v1/AUTH_something/foo/bar', 202, {'Connection': 'close'})
with webserver.install_http_handler(handler):
ret = gdal.Unlink('/vsiswift/foo/bar')
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Failure
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 206,
{'Content-Range': 'bytes 0-0/1'}, 'x')
handler.add('DELETE', '/v1/AUTH_something/foo/bar', 400, {'Connection': 'close'})
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink('/vsiswift/foo/bar')
if ret != -1:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsiaz_fake_unlink():
if gdaltest.webserver_port == 0:
return 'skip'
# Success
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_unlink/myfile', 200, {'Content-Length': '1'})
handler.add('DELETE', '/azure/blob/myaccount/az_bucket_test_unlink/myfile', 202, {'Connection': 'close'})
with webserver.install_http_handler(handler):
ret = gdal.Unlink('/vsiaz/az_bucket_test_unlink/myfile')
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Failure
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_unlink/myfile', 200, {'Content-Length': '1'})
handler.add('DELETE', '/azure/blob/myaccount/az_bucket_test_unlink/myfile', 400, {'Connection': 'close'})
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink('/vsiaz/az_bucket_test_unlink/myfile')
if ret != -1:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def test_vsicurl_test_parse_html_filelist_apache():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/mydir/', 200, {}, """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<html>
<head>
<title>Index of /mydir</title>
</head>
<body>
<h1>Index of /mydir</h1>
<table><tr><th><img src="/icons/blank.gif" alt="[ICO]"></th><th><a href="?C=N;O=D">Name</a></th><th><a href="?C=M;O=A">Last modified</a></th><th><a href="?C=S;O=A">Size</a></th><th><a href="?C=D;O=A">Description</a></th></tr><tr><th colspan="5"><hr></th></tr>
<tr><td valign="top"><img src="/icons/back.gif" alt="[DIR]"></td><td><a href="/gdal/data/">Parent Directory</a></td><td> </td><td align="right"> - </td><td> </td></tr>
<tr><td valign="top"><img src="/icons/image2.gif" alt="[IMG]"></td><td><a href="foo.tif">foo.tif</a></td><td align="right">17-May-2010 12:26 </td><td align="right"> 90K</td><td> </td></tr>
<tr><td valign="top"><img src="/icons/image2.gif" alt="[IMG]"></td><td><a href="foo%20with%20space.tif">foo with space.tif</a></td><td align="right">15-Jan-2007 11:02 </td><td align="right">736 </td><td> </td></tr>
<tr><th colspan="5"><hr></th></tr>
</table>
</body></html>""")
with webserver.install_http_handler(handler):
fl = gdal.ReadDir('/vsicurl/http://localhost:%d/mydir' % gdaltest.webserver_port)
assert fl == ['foo.tif', 'foo%20with%20space.tif']
assert gdal.VSIStatL('/vsicurl/http://localhost:%d/mydir/foo%%20with%%20space.tif' % gdaltest.webserver_port, gdal.VSI_STAT_EXISTS_FLAG) is not None
handler = webserver.SequentialHandler()
handler.add('HEAD', '/mydir/i_dont_exist', 404, {})
with webserver.install_http_handler(handler):
assert gdal.VSIStatL('/vsicurl/http://localhost:%d/mydir/i_dont_exist' % gdaltest.webserver_port, gdal.VSI_STAT_EXISTS_FLAG) is None
def test_visoss_8():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/visoss_8/?delimiter=%2F', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix></Prefix>
<Contents>
<Key>test</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>40</Size>
</Contents>
<CommonPrefixes>
<Prefix>test/</Prefix>
</CommonPrefixes>
</ListBucketResult>
""")
with webserver.install_http_handler(handler):
listdir = gdal.ReadDir('/vsioss/visoss_8', 0)
assert listdir == ['test', 'test/']
handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
assert not stat.S_ISDIR(gdal.VSIStatL('/vsioss/visoss_8/test').mode)
handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
assert stat.S_ISDIR(gdal.VSIStatL('/vsioss/visoss_8/test/').mode)
def vsiswift_fake_auth_storage_url_and_auth_token():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
gdal.SetConfigOption('SWIFT_AUTH_V1_URL', '')
gdal.SetConfigOption('SWIFT_USER', '')
gdal.SetConfigOption('SWIFT_KEY', '')
gdal.SetConfigOption('SWIFT_STORAGE_URL', 'http://127.0.0.1:%d/v1/AUTH_something' % gdaltest.webserver_port)
gdal.SetConfigOption('SWIFT_AUTH_TOKEN', 'my_auth_token')
# Failure
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 501)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiswift/foo/bar')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
gdal.VSICurlClearCache()
# Success
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'x-auth-token' not in h or \
h['x-auth-token'] != 'my_auth_token':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiswift/foo/bar')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
return 'success'
def test_vsiwebhdfs_open():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
# Download without redirect (not nominal)
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1/foo/bar?op=OPEN&offset=9999990784&length=16384', 200,
{}, '0123456789data')
with webserver.install_http_handler(handler):
f = open_for_read(gdaltest.webhdfs_base_connection + '/foo/bar')
assert f is not None
gdal.VSIFSeekL(f, 9999990784 + 10, 0)
assert gdal.VSIFReadL(1, 4, f).decode('ascii') == 'data'
gdal.VSIFCloseL(f)
# Download with redirect (nominal) and permissions
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384&user.name=root&delegation=token', 307,
{'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384'})
handler.add('GET', '/redirected/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384', 200,
{}, 'yeah')
with gdaltest.config_options({'WEBHDFS_USERNAME': '******',
'WEBHDFS_DELEGATION': 'token',
'WEBHDFS_DATANODE_HOST': 'localhost'}):
with webserver.install_http_handler(handler):
f = open_for_read(gdaltest.webhdfs_base_connection + '/foo/bar')
assert f is not None
assert gdal.VSIFReadL(1, 4, f).decode('ascii') == 'yeah'
gdal.VSIFCloseL(f)
# Test error
gdal.VSICurlClearCache()
f = open_for_read(gdaltest.webhdfs_base_connection + '/foo/bar')
assert f is not None
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384', 404)
with webserver.install_http_handler(handler):
assert len(gdal.VSIFReadL(1, 4, f)) == 0
# Retry: shouldn't not cause network access
assert len(gdal.VSIFReadL(1, 4, f)) == 0
gdal.VSIFCloseL(f)
def test_visoss_7():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_bucket_test_mkdir/dir/', 404, {'Connection': 'close'})
handler.add('GET', '/oss_bucket_test_mkdir/?delimiter=%2F&max-keys=100&prefix=dir%2F', 404, {'Connection': 'close'})
handler.add('PUT', '/oss_bucket_test_mkdir/dir/', 200)
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsioss/oss_bucket_test_mkdir/dir', 0)
assert ret == 0
# Try creating already existing directory
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_bucket_test_mkdir/dir/', 416)
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsioss/oss_bucket_test_mkdir/dir', 0)
assert ret != 0
handler = webserver.SequentialHandler()
handler.add('DELETE', '/oss_bucket_test_mkdir/dir/', 204)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsioss/oss_bucket_test_mkdir/dir')
assert ret == 0
# Try deleting already deleted directory
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_bucket_test_mkdir/dir/', 404)
handler.add('GET', '/oss_bucket_test_mkdir/?delimiter=%2F&max-keys=100&prefix=dir%2F', 404, {'Connection': 'close'})
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsioss/oss_bucket_test_mkdir/dir')
assert ret != 0
# Try deleting non-empty directory
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_bucket_test_mkdir/dir_nonempty/', 416)
handler.add('GET', '/oss_bucket_test_mkdir/?delimiter=%2F&max-keys=100&prefix=dir_nonempty%2F', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>dir_nonempty/</Prefix>
<Contents>
<Key>dir_nonempty/test.txt</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>40</Size>
</Contents>
</ListBucketResult>
""")
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsioss/oss_bucket_test_mkdir/dir_nonempty')
assert ret != 0
def vsicurl_test_retry():
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add('GET', '/test_retry/', 404)
handler.add('HEAD', '/test_retry/test.txt', 200, {'Content-Length': '3'})
handler.add('GET', '/test_retry/test.txt', 502)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsicurl/http://localhost:%d/test_retry/test.txt' % gdaltest.webserver_port, 'rb')
data_len = 0
if f:
data_len = len(gdal.VSIFReadL(1, 1, f))
gdal.VSIFCloseL(f)
if data_len != 0:
gdaltest.post_reason('fail')
print(data_len)
return 'fail'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_retry/', 404)
handler.add('HEAD', '/test_retry/test.txt', 200, {'Content-Length': '3'})
handler.add('GET', '/test_retry/test.txt', 502)
handler.add('GET', '/test_retry/test.txt', 429)
handler.add('GET', '/test_retry/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsicurl?max_retry=2&retry_delay=0.01&url=http://localhost:%d/test_retry/test.txt' % gdaltest.webserver_port, 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
error_msg = gdal.GetLastErrorMsg()
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
if error_msg.find('429') < 0:
gdaltest.post_reason('fail')
print(error_msg)
return 'fail'
return 'success'
def test_vsiaz_write_blockblob_retry():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
# Test creation of BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
with gdaltest.config_options({'GDAL_HTTP_MAX_RETRY': '2',
'GDAL_HTTP_RETRY_DELAY': '0.01'}):
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(3).decode('ascii')
if len(content) != 3:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 502)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
assert gdal.VSIFWriteL('foo', 1, 3, f) == 3
gdal.VSIFCloseL(f)
def test_vsiaz_write_appendblob_retry():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
with gdaltest.config_options({'GDAL_HTTP_MAX_RETRY': '2',
'GDAL_HTTP_RETRY_DELAY': '0.01',
'VSIAZ_CHUNK_SIZE_BYTES': '10'}):
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 502)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 502)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 201)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 502)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 201)
with webserver.install_http_handler(handler):
assert gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) == 16
gdal.VSIFCloseL(f)
def vsiwebhdfs_readdir():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1/foo/?op=LISTSTATUS', 200,
{}, '{"FileStatuses":{"FileStatus":[{"type":"FILE","modificationTime":1000,"pathSuffix":"bar.baz","length":123456},{"type":"DIRECTORY","pathSuffix":"mysubdir","length":0}]}}')
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir(gdaltest.webhdfs_base_connection + '/foo')
if dir_contents != ['bar.baz', 'mysubdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
stat_res = gdal.VSIStatL(gdaltest.webhdfs_base_connection + '/foo/bar.baz')
if stat_res.size != 123456:
gdaltest.post_reason('fail')
print(stat_res.size)
return 'fail'
if stat_res.mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir(
gdaltest.webhdfs_base_connection + '/foo/bar.baz')
if dir_contents is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add('GET', '/webhdfs/v1foo/error_test/?op=LISTSTATUS', 404)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir(
gdaltest.webhdfs_base_connection + 'foo/error_test/')
if dir_contents is not None:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
return 'success'
def vsiswift_stat():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/bar', 206,
{'Content-Range': 'bytes 0-0/1000000'}, 'x')
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsiswift/foo/bar')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/v1/AUTH_something/foo/bar', 200, {'Content-Length': '1000000'})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsiswift_streaming/foo/bar')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
# Test stat on container
handler = webserver.SequentialHandler()
# GET on the container URL returns something, but we must hack this back
# to a directory
handler.add('GET', '/v1/AUTH_something/foo', 200, {}, "blabla")
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsiswift/foo')
if stat_res is None or not stat.S_ISDIR(stat_res.mode):
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsiswift_fake_write():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Test creation of BlockBob
f = gdal.VSIFOpenL('/vsiswift/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'x-auth-token' not in h or \
h['x-auth-token'] != 'my_auth_token' or \
'Transfer-Encoding' not in h or h['Transfer-Encoding'] != 'chunked':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = ''
while True:
numchars = int(request.rfile.readline().strip(), 16)
content += request.rfile.read(numchars).decode('ascii')
request.rfile.read(2)
if numchars == 0:
break
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/v1/AUTH_something/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
return 'success'
def test_vsiaz_fake_unlink():
if gdaltest.webserver_port == 0:
pytest.skip()
# Success
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_unlink/myfile', 200, {'Content-Length': '1'})
handler.add('DELETE', '/azure/blob/myaccount/az_bucket_test_unlink/myfile', 202, {'Connection': 'close'})
with webserver.install_http_handler(handler):
ret = gdal.Unlink('/vsiaz/az_bucket_test_unlink/myfile')
assert ret == 0
# Failure
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_unlink/myfile', 200, {'Content-Length': '1'})
handler.add('DELETE', '/azure/blob/myaccount/az_bucket_test_unlink/myfile', 400, {'Connection': 'close'})
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink('/vsiaz/az_bucket_test_unlink/myfile')
assert ret == -1
def test_vsigs_read_credentials_file():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '/vsimem/.boto')
gdal.VSICurlClearCache()
gdal.FileFromMemBuffer('/vsimem/.boto', """
[unrelated]
gs_access_key_id = foo
gs_secret_access_key = bar
[Credentials]
gs_access_key_id = GS_ACCESS_KEY_ID
gs_secret_access_key = GS_SECRET_ACCESS_KEY
[unrelated]
gs_access_key_id = foo
gs_secret_access_key = bar
""")
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'GOOG1 GS_ACCESS_KEY_ID:8tndu9//BfmN+Kg4AFLdUMZMBDQ='
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '')
gdal.Unlink('/vsimem/.boto')
def test_vsicurl_test_fallback_from_head_to_get():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('HEAD', '/test_fallback_from_head_to_get', 405)
handler.add('GET', '/test_fallback_from_head_to_get', 200, {}, 'foo')
with webserver.install_http_handler(handler):
statres = gdal.VSIStatL('/vsicurl/http://localhost:%d/test_fallback_from_head_to_get' % gdaltest.webserver_port)
assert statres.size == 3
gdal.VSICurlClearCache()
def test_ogr_wfs3_empty_layer():
if gdaltest.wfs3_drv is None:
pytest.skip()
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200,
{'Content-Type': 'application/json'},
'{ "collections" : [ { "name": "foo" }] }')
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' % gdaltest.webserver_port)
assert ds is not None
assert ds.GetLayerCount() == 1
lyr = ds.GetLayer(0)
assert lyr.GetName() == 'foo'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
'{ "type": "FeatureCollection", "features": [] }')
with webserver.install_http_handler(handler):
assert lyr.GetLayerDefn().GetFieldCount() == 0
def vsicurl_test_fallback_from_head_to_get():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('HEAD', '/test_fallback_from_head_to_get', 405)
handler.add('GET', '/test_fallback_from_head_to_get', 200, {}, 'foo')
with webserver.install_http_handler(handler):
statres = gdal.VSIStatL('/vsicurl/http://localhost:%d/test_fallback_from_head_to_get' % gdaltest.webserver_port)
if statres.size != 3:
return 'fail'
gdal.VSICurlClearCache()
return 'success'
def test_vsigs_write_retry():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
with gdaltest.config_options({'GDAL_HTTP_MAX_RETRY': '2',
'GDAL_HTTP_RETRY_DELAY': '0.01'}):
f = gdal.VSIFOpenL('/vsigs/test_write_retry/put_with_retry.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = ''
while True:
numchars = int(request.rfile.readline().strip(), 16)
content += request.rfile.read(numchars).decode('ascii')
request.rfile.read(2)
if numchars == 0:
break
if len(content) != 3:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/test_write_retry/put_with_retry.bin', 502)
handler.add('PUT', '/test_write_retry/put_with_retry.bin', custom_method=method)
with webserver.install_http_handler(handler):
assert gdal.VSIFWriteL('foo', 1, 3, f) == 3
gdal.VSIFCloseL(f)
def mbtiles_http_png():
if gdaltest.mbtiles_drv is None:
return 'skip'
if gdal.GetDriverByName('HTTP') is None:
return 'skip'
if gdal.GetDriverByName('PNG') is None:
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.FileHandler(
{'/byte.mbtiles': open('data/byte.mbtiles', 'rb').read()})
with webserver.install_http_handler(handler):
ds = gdal.Open('/vsicurl/http://localhost:%d/byte.mbtiles' % gdaltest.webserver_port)
if ds is None:
return 'fail'
return 'success'
def test_visoss_5():
if gdaltest.webserver_port == 0:
pytest.skip()
with webserver.install_http_handler(webserver.SequentialHandler()):
with gdaltest.error_handler():
ret = gdal.Unlink('/vsioss/foo')
assert ret != 0
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_delete_bucket/delete_file', 200, {'Connection': 'close'}, 'foo')
with webserver.install_http_handler(handler):
assert gdal.VSIStatL('/vsioss/oss_delete_bucket/delete_file').size == 3
with webserver.install_http_handler(webserver.SequentialHandler()):
assert gdal.VSIStatL('/vsioss/oss_delete_bucket/delete_file').size == 3
handler = webserver.SequentialHandler()
handler.add('DELETE', '/oss_delete_bucket/delete_file', 204)
with webserver.install_http_handler(handler):
ret = gdal.Unlink('/vsioss/oss_delete_bucket/delete_file')
assert ret == 0
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_delete_bucket/delete_file', 404, {'Connection': 'close'}, 'foo')
handler.add('GET', '/oss_delete_bucket/?delimiter=%2F&max-keys=100&prefix=delete_file%2F', 404, {'Connection': 'close'}, 'foo')
with webserver.install_http_handler(handler):
assert gdal.VSIStatL('/vsioss/oss_delete_bucket/delete_file') is None
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_delete_bucket/delete_file_error', 200)
handler.add('DELETE', '/oss_delete_bucket/delete_file_error', 403)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.Unlink('/vsioss/oss_delete_bucket/delete_file_error')
assert ret != 0
def ogr_wfs3_attribute_filter():
if gdaltest.wfs3_drv is None:
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200, {'Content-Type': 'application/json'},
"""{ "collections" : [ {
"name": "foo"
}] }""")
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' % gdaltest.webserver_port)
lyr = ds.GetLayer(0)
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"attr1": "",
"attr2": 0,
"attr3": ""
}
}
] }""")
# Fake openapi response
handler.add('GET', '/wfs3/api', 200, {'Content-Type': 'application/json'},
"""{
"openapi": "3.0.0",
"paths" : {
"/collections/foo/items": {
"get": {
"parameters": [
{
"name": "attr1",
"in": "query"
},
{
"name": "attr2",
"in": "query"
}
]
}
}
}
}""")
with webserver.install_http_handler(handler):
lyr.SetAttributeFilter("(attr1 = 'foo' AND attr2 = 2) AND attr3 = 'bar'")
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections/foo/items?limit=10&attr1=foo&attr2=2', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"attr1": "foo",
"attr2": 2,
"attr3": "foo"
}
},
{
"type": "Feature",
"properties": {
"attr1": "foo",
"attr2": 2,
"attr3": "bar"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f is None:
gdaltest.post_reason('fail')
return 'fail'
lyr.ResetReading()
lyr.SetAttributeFilter("attr1 = 'foo' OR attr3 = 'bar'")
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"attr1": "foo",
"attr3": "foo"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f is None:
gdaltest.post_reason('fail')
return 'fail'
lyr.ResetReading()
lyr.SetAttributeFilter(None)
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"attr1": "foo",
"attr3": "foo"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f is None:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsiaz_fake_mkdir_rmdir():
if gdaltest.webserver_port == 0:
return 'skip'
# Invalid name
ret = gdal.Mkdir('/vsiaz', 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_mkdir/dir/', 404,
{'Connection': 'close'})
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=/&maxresults=1&prefix=dir/&restype=container',
200, {'Connection': 'close'})
handler.add(
'PUT',
'/azure/blob/myaccount/az_bucket_test_mkdir/dir/.gdal_marker_for_dir',
201)
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsiaz/az_bucket_test_mkdir/dir', 0)
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Try creating already existing directory
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_mkdir/dir/', 404)
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=/&maxresults=1&prefix=dir/&restype=container',
200, {
'Connection': 'close',
'Content-type': 'application/xml'
}, """<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>dir/</Prefix>
<Blobs>
<Blob>
<Name>dir/.gdal_marker_for_dir</Name>
</Blob>
</Blobs>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsiaz/az_bucket_test_mkdir/dir', 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Invalid name
ret = gdal.Rmdir('/vsiaz')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Not a directory
handler = webserver.SequentialHandler()
handler.add('HEAD',
'/azure/blob/myaccount/az_bucket_test_mkdir/it_is_a_file/',
404)
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=/&maxresults=1&prefix=it_is_a_file/&restype=container',
200, {
'Connection': 'close',
'Content-type': 'application/xml'
}, """<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>it_is_a_file/</Prefix>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiaz/az_bucket_test_mkdir/it_is_a_file')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Valid
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=/&maxresults=1&prefix=dir/&restype=container',
200, {
'Connection': 'close',
'Content-type': 'application/xml'
}, """<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>dir/</Prefix>
<Blobs>
<Blob>
<Name>dir/.gdal_marker_for_dir</Name>
</Blob>
</Blobs>
</EnumerationResults>
""")
handler.add(
'DELETE',
'/azure/blob/myaccount/az_bucket_test_mkdir/dir/.gdal_marker_for_dir',
202)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiaz/az_bucket_test_mkdir/dir')
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Try deleting already deleted directory
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_mkdir/dir/', 404)
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=/&maxresults=1&prefix=dir/&restype=container',
200)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiaz/az_bucket_test_mkdir/dir')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Try deleting non-empty directory
handler = webserver.SequentialHandler()
handler.add('HEAD',
'/azure/blob/myaccount/az_bucket_test_mkdir/dir_nonempty/',
404)
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=/&maxresults=1&prefix=dir_nonempty/&restype=container',
200, {
'Connection': 'close',
'Content-type': 'application/xml'
}, """<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>dir_nonempty/</Prefix>
<Blobs>
<Blob>
<Name>dir_nonempty/foo</Name>
</Blob>
</Blobs>
</EnumerationResults>
""")
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=/&maxresults=1&prefix=dir_nonempty/&restype=container',
200, {
'Connection': 'close',
'Content-type': 'application/xml'
}, """<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>dir_nonempty/</Prefix>
<Blobs>
<Blob>
<Name>dir_nonempty/foo</Name>
</Blob>
</Blobs>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiaz/az_bucket_test_mkdir/dir_nonempty')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsiswift_fake_auth_v1_url():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
gdal.SetConfigOption(
'SWIFT_AUTH_V1_URL',
'http://127.0.0.1:%d/auth/1.0' % gdaltest.webserver_port)
gdal.SetConfigOption('SWIFT_USER', 'my_user')
gdal.SetConfigOption('SWIFT_KEY', 'my_key')
gdal.SetConfigOption('SWIFT_STORAGE_URL', '')
gdal.SetConfigOption('SWIFT_AUTH_TOKEN', '')
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'X-Auth-User' not in h or h['X-Auth-User'] != 'my_user' or \
'X-Auth-Key' not in h or h['X-Auth-Key'] != 'my_key':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.send_header(
'X-Storage-Url',
'http://127.0.0.1:%d/v1/AUTH_something' % gdaltest.webserver_port)
request.send_header('X-Auth-Token', 'my_auth_token')
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/auth/1.0', custom_method=method)
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'x-auth-token' not in h or \
h['x-auth-token'] != 'my_auth_token':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/v1/AUTH_something/foo/bar', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiswift/foo/bar')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
# authentication is reused
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'x-auth-token' not in h or \
h['x-auth-token'] != 'my_auth_token':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""bar""".encode('ascii'))
handler.add('GET', '/v1/AUTH_something/foo/baz', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiswift/foo/baz')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'bar':
gdaltest.post_reason('fail')
print(data)
return 'fail'
return 'success'
def vsigs_read_credentials_refresh_token_custom_app():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption(
'GOA2_AUTH_URL_TOKEN',
'http://localhost:%d/accounts.google.com/o/oauth2/token' %
gdaltest.webserver_port)
gdal.SetConfigOption('GS_OAUTH2_REFRESH_TOKEN', 'REFRESH_TOKEN')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_ID', 'CLIENT_ID')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_SECRET', 'CLIENT_SECRET')
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(
request.headers['Content-Length'])).decode('ascii')
if content != 'refresh_token=REFRESH_TOKEN&client_id=CLIENT_ID&client_secret=CLIENT_SECRET&grant_type=refresh_token':
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST',
'/accounts.google.com/o/oauth2/token',
custom_method=method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" %
str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN', None)
gdal.SetConfigOption('GS_OAUTH2_REFRESH_TOKEN', '')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_ID', '')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_SECRET', '')
return 'success'
def vsiswift_fake_readdir():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=1', 200,
{'Content-type': 'application/json'}, """[
{
"last_modified": "1970-01-01T00:00:01",
"bytes": 123456,
"name": "bar.baz"
}
]""")
handler.add('GET',
'/v1/AUTH_something/foo?delimiter=%2F&limit=1&marker=bar.baz',
200, {'Content-type': 'application/json'}, """[
{
"subdir": "mysubdir/"
}
]""")
handler.add(
'GET',
'/v1/AUTH_something/foo?delimiter=%2F&limit=1&marker=mysubdir%2F', 200,
{'Content-type': 'application/json'}, """[
]""")
with gdaltest.config_option('SWIFT_MAX_KEYS', '1'):
with webserver.install_http_handler(handler):
f = open_for_read('/vsiswift/foo/bar.baz')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsiswift/foo')
if dir_contents != ['bar.baz', 'mysubdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
stat_res = gdal.VSIStatL('/vsiswift/foo/bar.baz')
if stat_res.size != 123456:
gdaltest.post_reason('fail')
print(stat_res.size)
return 'fail'
if stat_res.mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsiswift/foo/bar.baz')
if dir_contents is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/v1/AUTH_something/foo?delimiter=%2F&limit=10000&prefix=error_test%2F',
500)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/foo/error_test/')
if dir_contents is not None:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# List containers (empty result)
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something', 200,
{'Content-type': 'application/json'}, """[]
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/')
if dir_contents != ['.']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# List containers
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/v1/AUTH_something', 200, {'Content-type': 'application/json'},
"""[ { "name": "mycontainer1", "count": 0, "bytes": 0 },
{ "name": "mycontainer2", "count": 0, "bytes": 0}
] """)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/')
if dir_contents != ['mycontainer1', 'mycontainer2']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# ReadDir() with a file and directory of same names
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/v1/AUTH_something', 200, {'Content-type': 'application/json'},
"""[ {
"last_modified": "1970-01-01T00:00:01",
"bytes": 123456,
"name": "foo"
},
{ "subdir": "foo/"} ] """)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/')
if dir_contents != ['foo', 'foo/']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200,
{'Content-type': 'application/json'}, "[]")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/foo/')
if dir_contents != ['.']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
return 'success'
def vsiswift_fake_mkdir_rmdir():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Invalid name
ret = gdal.Mkdir('/vsiswift', 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/dir/', 404,
{'Connection': 'close'})
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200,
{'Connection': 'close'}, "[]")
handler.add('PUT', '/v1/AUTH_something/foo/dir/', 201)
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsiswift/foo/dir', 0)
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Try creating already existing directory
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/dir/', 404,
{'Connection': 'close'})
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200,
{
'Connection': 'close',
'Content-type': 'application/json'
}, """[ { "subdir": "dir/" } ]""")
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsiswift/foo/dir', 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Invalid name
ret = gdal.Rmdir('/vsiswift')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSICurlClearCache()
# Not a directory
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/it_is_a_file/', 404)
handler.add(
'GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200, {
'Connection': 'close',
'Content-type': 'application/json'
},
"""[ { "name": "it_is_a_file/", "bytes": 0, "last_modified": "1970-01-01T00:00:01" } ]"""
)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiswift/foo/it_is_a_file')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Valid
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/dir/', 200)
handler.add('GET',
'/v1/AUTH_something/foo?delimiter=%2F&limit=2&prefix=dir%2F',
200, {
'Connection': 'close',
'Content-type': 'application/json'
}, """[]
""")
handler.add('DELETE', '/v1/AUTH_something/foo/dir/', 204)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiswift/foo/dir')
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Try deleting already deleted directory
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/dir/', 404)
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiswift/foo/dir')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSICurlClearCache()
# Try deleting non-empty directory
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/dir_nonempty/', 404)
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200,
{
'Connection': 'close',
'Content-type': 'application/json'
}, """[ { "subdir": "dir_nonempty/" } ]""")
handler.add(
'GET',
'/v1/AUTH_something/foo?delimiter=%2F&limit=2&prefix=dir_nonempty%2F',
200, {
'Connection': 'close',
'Content-type': 'application/json'
},
"""[ { "name": "dir_nonempty/some_file", "bytes": 0, "last_modified": "1970-01-01T00:00:01" } ]"""
)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiswift/foo/dir_nonempty')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsigs_read_credentials_gce_expiration():
if gdaltest.webserver_port == 0:
return 'skip'
if sys.platform not in ('linux', 'linux2', 'win32'):
return 'skip'
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '')
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption(
'CPL_GCE_CREDENTIALS_URL',
'http://localhost:%d/computeMetadata/v1/instance/service-accounts/default/token'
% gdaltest.webserver_port)
# Disable hypervisor related check to test if we are really on EC2
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', 'NO')
gdal.VSICurlClearCache()
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" %
str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
# First time is used when trying to establish if GCE authentication is available
handler.add(
'GET', '/computeMetadata/v1/instance/service-accounts/default/token',
200, {}, """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 0,
}""")
# Second time is needed because f the access to th file
handler.add(
'GET', '/computeMetadata/v1/instance/service-accounts/default/token',
200, {}, """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 0,
}""")
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL', '')
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', None)
return 'success'
def vsigs_read_credentials_oauth2_service_account():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
# Generated with 'openssl genrsa -out rsa-openssl.pem 1024' and
# 'openssl pkcs8 -nocrypt -in rsa-openssl.pem -inform PEM -topk8 -outform PEM -out rsa-openssl.pkcs8.pem'
# DO NOT USE in production !!!!
key = """-----BEGIN PRIVATE KEY-----
MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk
VNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm
i2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ
DVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy
TabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I
2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd
DrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+
bpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg
Wz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3
4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5
Kvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj
j+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI
yzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq
gwE6fxOLyJDxuWRf
-----END PRIVATE KEY-----
"""
for i in range(2):
gdal.SetConfigOption(
'GO2A_AUD',
'http://localhost:%d/oauth2/v4/token' % gdaltest.webserver_port)
gdal.SetConfigOption('GOA2_NOW', '123456')
if i == 0:
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY', key)
else:
gdal.FileFromMemBuffer('/vsimem/pkey', key)
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY_FILE', '/vsimem/pkey')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_EMAIL', 'CLIENT_EMAIL')
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(
request.headers['Content-Length'])).decode('ascii')
content_8080 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODAvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.DAhqWtBgKpObxZ%2BGiXqwF%2Fa4SS%2FNWQRhLCI7DYZCuOTuf2w7dL8j4CdpiwwzQg1diIus7dyViRfzpsFmuZKAXwL%2B84iBoVVqnJJZ4TgwH49NdfMAnc4Rgm%2Bo2a2nEcMjX%2FbQ3jRY%2B9WNVl96hzULGvLrVeyego2f06wivqmvxHA%3D'
content_8081 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODEvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.0abOEg4%2FRApWTSeAs6YTHaNzdwOgZLm8DTMO2MKlOA%2Fiagyb4cBJxDpkD5gECPvi7qhkg7LsyFuj0a%2BK48Bsuj%2FgLHOU4MpB0dHwYnDO2UXzH%2FUPdgFCVak1P1V%2ByiDA%2B%2Ft4aDI5fD9qefKQiu3wsMDHzP71MNLzayrjqaqKKS4%3D'
if content not in [content_8080, content_8081]:
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST', '/oauth2/v4/token', custom_method=method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" %
str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
try:
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
except:
if gdal.GetLastErrorMsg().find(
'CPLRSASHA256Sign() not implemented') >= 0:
return 'skip'
finally:
gdal.SetConfigOption('GO2A_AUD', None)
gdal.SetConfigOption('GO2A_NOW', None)
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY', '')
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY_FILE', '')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_EMAIL', '')
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.Unlink('/vsimem/pkey')
return 'success'
def vsigs_2():
if gdaltest.webserver_port == 0:
return 'skip'
# header file
gdal.FileFromMemBuffer('/vsimem/my_headers.txt', 'foo: bar')
handler = webserver.SequentialHandler()
handler.add('GET',
'/gs_fake_bucket_http_header_file/resource',
200, {'Content-type': 'text/plain'},
'Y',
expected_headers={'foo': 'bar'})
with webserver.install_http_handler(handler):
with gdaltest.config_option('GDAL_HTTP_HEADER_FILE',
'/vsimem/my_headers.txt'):
f = open_for_read(
'/vsigs/gs_fake_bucket_http_header_file/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(data) != 1:
gdaltest.post_reason('fail')
return 'fail'
gdal.Unlink('/vsimem/my_headers.txt')
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID')
gdal.SetConfigOption('CPL_GS_TIMESTAMP', 'my_timestamp')
handler = webserver.SequentialHandler()
handler.add('GET',
'/gs_fake_bucket/resource',
200, {'Content-type': 'text/plain'},
'foo',
expected_headers={
'Authorization':
'GOOG1 GS_ACCESS_KEY_ID:8tndu9//BfmN+Kg4AFLdUMZMBDQ='
})
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET',
'/gs_fake_bucket/resource',
200, {'Content-type': 'text/plain'},
'foo',
expected_headers={
'Authorization':
'GOOG1 GS_ACCESS_KEY_ID:8tndu9//BfmN+Kg4AFLdUMZMBDQ='
})
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs_streaming/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET', '/gs_fake_bucket/resource2.bin', 206,
{'Content-Range': 'bytes 0-0/1000000'}, 'x')
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsigs/gs_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/gs_fake_bucket/resource2.bin', 200,
{'Content-Length': 1000000})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL(
'/vsigs_streaming/gs_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
return 'success'
def vsigs_read_credentials_file():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '/vsimem/.boto')
gdal.VSICurlClearCache()
gdal.FileFromMemBuffer(
'/vsimem/.boto', """
[unrelated]
gs_access_key_id = foo
gs_secret_access_key = bar
[Credentials]
gs_access_key_id = GS_ACCESS_KEY_ID
gs_secret_access_key = GS_SECRET_ACCESS_KEY
[unrelated]
gs_access_key_id = foo
gs_secret_access_key = bar
""")
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'GOOG1 GS_ACCESS_KEY_ID:8tndu9//BfmN+Kg4AFLdUMZMBDQ='
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" %
str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '')
gdal.Unlink('/vsimem/.boto')
return 'success'
def vsiaz_fake_write():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Test creation of BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:AigkrY7q66WCrx3JRKBte56k7kxV2cxB/ZyGNubxk5I=' or \
'Expect' not in h or h['Expect'] != '100-continue' or \
'Content-Length' not in h or h['Content-Length'] != '40000' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'BlockBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(40000).decode('ascii')
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Simulate illegal read
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
if len(ret) != 0:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
gdal.VSIFCloseL(f)
# Simulate illegal seek
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
# Simulate failure when putting BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
if gdal.VSIFSeekL(f, 0, 0) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if gdal.VSIFTellL(f) != 35000:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 35000, 0) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 0, 1) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 0, 2) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFEofL(f) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFCloseL(f)
if ret == 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
# Simulate creation of BlockBob over an existing blob of incompatible type
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 409)
handler.add('DELETE', '/azure/blob/myaccount/test_copy/file.bin', 202)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
# Test creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:KimVui3ptY9D5ftLlsI7CNOgK36CNAEzsXqcuHskdEY=' or \
'Content-Length' not in h or h['Content-Length'] != '0' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '10' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(10).decode('ascii')
if content != '0123456789':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '6' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(6).decode('ascii')
if content != 'abcdef':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 16:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Test failed creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Test failed writing of a block of an AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
403)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
return 'success'
def vsigs_readdir():
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add(
'GET', '/gs_fake_bucket2/?delimiter=%2F&prefix=a_dir%2F', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Contents>
<Key>a_dir/resource3.bin</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>123456</Size>
</Contents>
</ListBucketResult>
""")
handler.add(
'GET', '/gs_fake_bucket2/?delimiter=%2F&marker=bla&prefix=a_dir%2F',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<Contents>
<Key>a_dir/resource4.bin</Key>
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
</Contents>
<CommonPrefixes>
<Prefix>a_dir/subdir/</Prefix>
</CommonPrefixes>
</ListBucketResult>
""")
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL(
'/vsigs/gs_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir/resource3.bin')
if dir_contents is not None:
gdaltest.post_reason('fail')
return 'fail'
# List buckets
handler = webserver.SequentialHandler()
handler.add(
'GET', '/', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult>
<Buckets>
<Bucket>
<Name>mybucket</Name>
</Bucket>
</Buckets>
</ListAllMyBucketsResult>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsigs/')
if dir_contents != ['mybucket']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
return 'success'
def vsigs_write():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = ''
while True:
numchars = int(request.rfile.readline().strip(), 16)
content += request.rfile.read(numchars).decode('ascii')
request.rfile.read(2)
if numchars == 0:
break
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Simulate failure while transmitting
f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Simulate failure at end of transfer
f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = ''
while True:
numchars = int(request.rfile.readline().strip(), 16)
content += request.rfile.read(numchars).decode('ascii')
request.rfile.read(2)
if numchars == 0:
break
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if ret != 35000:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFCloseL(f)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsiaz_fake_basic():
if gdaltest.webserver_port == 0:
return 'skip'
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:zKb0EXnM/RinBjcUE9EU+qfRIGaIItoUElSWc+FE24E=' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET',
'/azure/blob/myaccount/az_fake_bucket/resource',
custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz/az_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:8d6IEeOsl7qGpKAxaTTxx2xMNpvqWq8DGlFE67lsmQ4=' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'Accept-Encoding' not in h or h['Accept-Encoding'] != 'gzip':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET',
'/azure/blob/myaccount/az_fake_bucket/resource',
custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz_streaming/az_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin',
200, {'Content-Length': '1000000'})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsiaz/az_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin',
200, {'Content-Length': 1000000})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL(
'/vsiaz_streaming/az_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
return 'success'
def vsiaz_fake_readdir():
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=/&prefix=a_dir/&restype=container',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Blobs>
<Blob>
<Name>a_dir/resource3.bin</Name>
<Properties>
<Last-Modified>01 Jan 1970 00:00:01</Last-Modified>
<Content-Length>123456</Content-Length>
</Properties>
</Blob>
</Blobs>
</EnumerationResults>
""")
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=/&marker=bla&prefix=a_dir/&restype=container',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<Blobs>
<Blob>
<Name>a_dir/resource4.bin</Name>
<Properties>
<Last-Modified>16 Oct 2016 12:34:56</Last-Modified>
<Content-Length>456789</Content-Length>
</Properties>
</Blob>
<BlobPrefix>
<Name>a_dir/subdir/</Name>
</BlobPrefix>
</Blobs>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL(
'/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
print(gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size)
return 'fail'
if gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
if dir_contents is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=/&prefix=error_test/&restype=container',
500)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/error_test/')
if dir_contents is not None:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# List containers (empty result)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/azure/blob/myaccount/?comp=list', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults ServiceEndpoint="https://myaccount.blob.core.windows.net">
<Containers/>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
if dir_contents != ['.']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
gdal.VSICurlClearCache()
# List containers
handler = webserver.SequentialHandler()
handler.add(
'GET', '/azure/blob/myaccount/?comp=list', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer1</Name>
</Container>
</Containers>
<NextMarker>bla</NextMarker>
</EnumerationResults>
""")
handler.add(
'GET', '/azure/blob/myaccount/?comp=list&marker=bla', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer2</Name>
</Container>
</Containers>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
if dir_contents != ['mycontainer1', 'mycontainer2']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
return 'success'
def ogr_wfs3_errors():
if gdaltest.wfs3_drv is None:
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 404)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' % gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# No Content-Type
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200, {}, 'foo')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' % gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# Unexpected Content-Type
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200,
{'Content-Type': 'text/html'}, 'foo')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' % gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid JSON
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200,
{'Content-Type': 'application/json'}, 'foo bar')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' % gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# Valid JSON but not collections array
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200,
{'Content-Type': 'application/json'}, '{}')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' % gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# Valid JSON but collections is not an array
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200,
{'Content-Type': 'application/json'},
'{ "collections" : null }')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' % gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200,
{'Content-Type': 'application/json'},
'{ "collections" : [ null, {} ] }')
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' % gdaltest.webserver_port)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayerCount() != 0:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayer(-1) is not None:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayer(0) is not None:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def test_vsicurl_test_redirect_x_amz():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_redirect/', 404)
# Simulate a big time difference between server and local machine
current_time = 1500
def method(request):
response = 'HTTP/1.1 302\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n'
response += 'Location: %s\r\n' % (
'http://localhost:%d/foo.s3.amazonaws.com/test_redirected/test.bin?X-Amz-Signature=foo&X-Amz-Expires=30&X-Amz-Date=%s'
% (gdaltest.webserver_port,
time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(current_time))))
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add('HEAD', '/test_redirect/test.bin', custom_method=method)
handler.add(
'HEAD',
'/foo.s3.amazonaws.com/test_redirected/test.bin?X-Amz-Signature=foo&X-Amz-Expires=30&X-Amz-Date=%s'
% time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(current_time)), 403,
{'Server': 'foo'}, '')
def method(request):
if 'Range' in request.headers:
if request.headers['Range'] == 'bytes=0-16383':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 0-16383/1000000')
request.send_header('Content-Length', 16384)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
elif request.headers['Range'] == 'bytes=16384-49151':
# Test expiration of the signed URL
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
else:
request.send_response(404)
request.send_header('Content-Length', 0)
request.end_headers()
else:
# After a failed attempt on a HEAD, the client should go there
response = 'HTTP/1.1 200\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime(
"%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(current_time)) + '\r\n'
response += 'Content-type: text/plain\r\n'
response += 'Content-Length: 1000000\r\n'
response += 'Connection: close\r\n'
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?X-Amz-Signature=foo&X-Amz-Expires=30&X-Amz-Date=%s'
% time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(current_time)),
custom_method=method)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL(
'/vsicurl/http://localhost:%d/test_redirect/test.bin' %
gdaltest.webserver_port, 'rb')
assert f is not None
gdal.VSIFSeekL(f, 0, 2)
try:
assert gdal.VSIFTellL(f) == 1000000
except:
gdal.VSIFCloseL(f)
raise
gdal.VSIFSeekL(f, 0, 0)
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?X-Amz-Signature=foo&X-Amz-Expires=30&X-Amz-Date=%s'
% time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(current_time)),
custom_method=method)
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?X-Amz-Signature=foo&X-Amz-Expires=30&X-Amz-Date=%s'
% time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(current_time)),
custom_method=method)
current_time = int(time.time())
def method(request):
# We should go there after expiration of the first signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(302)
# Return a new signed URL
request.send_header(
'Location',
'http://localhost:%d/foo.s3.amazonaws.com/test_redirected2/test.bin?X-Amz-Signature=foo&X-Amz-Expires=30&X-Amz-Date=%s'
% (request.server.port,
time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(current_time))))
request.send_header('Content-Length', 16384)
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
handler.add('GET', '/test_redirect/test.bin', custom_method=method)
def method(request):
# Second signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 16384-16384/1000000')
request.send_header('Content-Length', 1)
request.end_headers()
request.wfile.write('y'.encode('ascii'))
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected2/test.bin?X-Amz-Signature=foo&X-Amz-Expires=30&X-Amz-Date=%s'
% time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(current_time)),
custom_method=method)
with webserver.install_http_handler(handler):
content = gdal.VSIFReadL(1, 16383, f).decode('ascii')
if len(content) != 16383 or content[0] != 'x':
gdal.VSIFCloseL(f)
pytest.fail(content)
content = gdal.VSIFReadL(1, 2, f).decode('ascii')
if content != 'xy':
gdal.VSIFCloseL(f)
pytest.fail(content)
gdal.VSIFCloseL(f)