def test_visoss_3():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'application/xml')
response = """<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Contents>
<Key>a_dir/resource3.bin</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>123456</Size>
</Contents>
</ListBucketResult>
"""
request.send_header('Content-Length', len(response))
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET',
'/oss_fake_bucket2/?delimiter=%2F&prefix=a_dir%2F',
custom_method=method)
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'application/xml')
response = """<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<Contents>
<Key>a_dir/resource4.bin</Key>
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
</Contents>
<CommonPrefixes>
<Prefix>a_dir/subdir/</Prefix>
</CommonPrefixes>
</ListBucketResult>
"""
request.send_header('Content-Length', len(response))
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET',
'/oss_fake_bucket2/?delimiter=%2F&marker=bla&prefix=a_dir%2F',
custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
pytest.fail()
gdal.VSIFCloseL(f)
with webserver.install_http_handler(webserver.SequentialHandler()):
dir_contents = gdal.ReadDir('/vsioss/oss_fake_bucket2/a_dir')
assert dir_contents == ['resource3.bin', 'resource4.bin', 'subdir']
assert gdal.VSIStatL(
'/vsioss/oss_fake_bucket2/a_dir/resource3.bin').size == 123456
assert gdal.VSIStatL(
'/vsioss/oss_fake_bucket2/a_dir/resource3.bin').mtime == 1
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsioss/oss_fake_bucket2/a_dir/resource3.bin')
assert dir_contents is None
# Test CPL_VSIL_CURL_NON_CACHED
for config_option_value in [
'/vsioss/oss_non_cached/test.txt', '/vsioss/oss_non_cached',
'/vsioss/oss_non_cached:/vsioss/unrelated',
'/vsioss/unrelated:/vsioss/oss_non_cached',
'/vsioss/unrelated:/vsioss/oss_non_cached:/vsioss/unrelated'
]:
with gdaltest.config_option('CPL_VSIL_CURL_NON_CACHED',
config_option_value):
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo')
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo', config_option_value
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'bar2')
with webserver.install_http_handler(handler):
size = gdal.VSIStatL('/vsioss/oss_non_cached/test.txt').size
assert size == 4, config_option_value
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
size = gdal.VSIStatL('/vsioss/oss_non_cached/test.txt').size
if size != 3:
print(config_option_value)
pytest.fail(data)
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'bar2')
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'bar2')
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'bar2', config_option_value
# Retry without option
for config_option_value in [None, '/vsioss/oss_non_cached/bar.txt']:
with gdaltest.config_option('CPL_VSIL_CURL_NON_CACHED',
config_option_value):
handler = webserver.SequentialHandler()
if config_option_value is None:
handler.add(
'GET', '/oss_non_cached/?delimiter=%2F', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>/</Prefix>
<Contents>
<Key>/test.txt</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>40</Size>
</Contents>
<Contents>
<Key>/test2.txt</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>40</Size>
</Contents>
</ListBucketResult>
""")
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo', config_option_value
handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
# We should still get foo because of caching
assert data == 'foo', config_option_value
# List buckets (empty result)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult>
<Buckets>
</Buckets>
</ListAllMyBucketsResult>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsioss/')
assert dir_contents == ['.']
gdal.VSICurlClearCache()
# List buckets
handler = webserver.SequentialHandler()
handler.add(
'GET', '/', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult>
<Buckets>
<Bucket>
<Name>mybucket</Name>
</Bucket>
</Buckets>
</ListAllMyBucketsResult>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsioss/')
assert dir_contents == ['mybucket']
def test_ogr_opaif_fc_links_next_geojson():
if gdaltest.opaif_drv is None:
pytest.skip()
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections', 200, {'Content-Type': 'application/json'},
'{ "collections" : [ { "name": "foo" }] }')
with webserver.install_http_handler(handler):
ds = ogr.Open('OAPIF:http://localhost:%d/oapif' % gdaltest.webserver_port)
lyr = ds.GetLayer(0)
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar"
}
}
] }""")
with webserver.install_http_handler(handler):
assert lyr.GetLayerDefn().GetFieldCount() == 1
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection",
"links" : [
{ "rel": "next", "type": "application/geo+json", "href": "http://localhost:%d/oapif/foo_next" }
],
"features": [
{
"type": "Feature",
"properties": {
"foo": "bar"
}
}
] }""" % gdaltest.webserver_port)
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f['foo'] != 'bar':
f.DumpReadable()
pytest.fail()
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/foo_next', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"foo": "baz"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f['foo'] != 'baz':
f.DumpReadable()
pytest.fail()
def test_ogr_opaif_spatial_filter():
if gdaltest.opaif_drv is None:
pytest.skip()
if gdaltest.webserver_port == 0:
pytest.skip()
# Deprecated API
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections', 200, {'Content-Type': 'application/json'},
"""{ "collections" : [ {
"name": "foo",
"extent": {
"spatial": [ -10, 40, 15, 50 ]
}
}] }""")
with webserver.install_http_handler(handler):
ds = ogr.Open('OAPIF:http://localhost:%d/oapif' % gdaltest.webserver_port)
lyr = ds.GetLayer(0)
assert lyr.TestCapability(ogr.OLCFastGetExtent)
assert lyr.GetExtent() == (-10.0, 15.0, 40.0, 50.0)
# Nominal API
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections', 200, {'Content-Type': 'application/json'},
"""{ "collections" : [ {
"id": "foo",
"extent": {
"spatial": {
"bbox": [
[ -10, 40, -100, 15, 50, 100 ]
]
}
}
}] }""")
with webserver.install_http_handler(handler):
ds = ogr.Open('OAPIF:http://localhost:%d/oapif' % gdaltest.webserver_port)
lyr = ds.GetLayer(0)
assert lyr.GetExtent() == (-10.0, 15.0, 40.0, 50.0)
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar"
}
}
] }""")
with webserver.install_http_handler(handler):
assert lyr.GetLayerDefn().GetFieldCount() == 1
lyr.SetSpatialFilterRect(2, 49, 3, 50)
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections/foo/items?limit=10&bbox=2,49,3,50', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar",
},
"geometry": {
"type": "Point",
"coordinates": [ 2.5, 49.5 ]
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
assert f is not None
# Test clamping of bounds
lyr.SetSpatialFilterRect(-200, 49, 200, 50)
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections/foo/items?limit=10&bbox=-180,49,180,50', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar",
},
"geometry": {
"type": "Point",
"coordinates": [ 2.5, 49.5 ]
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
assert f is not None
lyr.SetSpatialFilterRect(2, -100, 3, 100)
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections/foo/items?limit=10&bbox=2,-90,3,90', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar",
},
"geometry": {
"type": "Point",
"coordinates": [ 2.5, 49.5 ]
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
assert f is not None
lyr.SetSpatialFilterRect(-200, -100, 200, 100)
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar",
},
"geometry": {
"type": "Point",
"coordinates": [ 2.5, 49.5 ]
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
assert f is not None
lyr.SetSpatialFilter(None)
lyr.ResetReading()
handler.add('GET', '/oapif/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar",
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
assert f is not None
def vsigs_read_credentials_file_refresh_token():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '/vsimem/.boto')
gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN',
'http://localhost:%d/accounts.google.com/o/oauth2/token' % gdaltest.webserver_port)
gdal.VSICurlClearCache()
gdal.FileFromMemBuffer('/vsimem/.boto', """
[Credentials]
gs_oauth2_refresh_token = REFRESH_TOKEN
[OAuth2]
client_id = CLIENT_ID
client_secret = CLIENT_SECRET
""")
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(request.headers['Content-Length'])).decode('ascii')
if content != 'refresh_token=REFRESH_TOKEN&client_id=CLIENT_ID&client_secret=CLIENT_SECRET&grant_type=refresh_token':
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST', '/accounts.google.com/o/oauth2/token', custom_method = method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization :
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method = method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '')
gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN', None)
gdal.Unlink('/vsimem/.boto')
return 'success'
def test_vsicurl_test_redirect():
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_redirect/', 404)
# Simulate a big time difference between server and local machine
current_time = 1500
def method(request):
response = 'HTTP/1.1 302\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n'
response += 'Location: %s\r\n' % (
'http://localhost:%d/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (gdaltest.webserver_port, current_time + 30))
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add('HEAD', '/test_redirect/test.bin', custom_method=method)
handler.add(
'HEAD',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30), 403, {'Server': 'foo'}, '')
def method(request):
if 'Range' in request.headers:
if request.headers['Range'] == 'bytes=0-16383':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 0-16383/1000000')
request.send_header('Content-Length', 16384)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
elif request.headers['Range'] == 'bytes=16384-49151':
# Test expiration of the signed URL
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
else:
request.send_response(404)
request.send_header('Content-Length', 0)
request.end_headers()
else:
# After a failed attempt on a HEAD, the client should go there
response = 'HTTP/1.1 200\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime(
"%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(current_time)) + '\r\n'
response += 'Content-type: text/plain\r\n'
response += 'Content-Length: 1000000\r\n'
response += 'Connection: close\r\n'
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL(
'/vsicurl/http://localhost:%d/test_redirect/test.bin' %
gdaltest.webserver_port, 'rb')
assert f is not None
gdal.VSIFSeekL(f, 0, 2)
if gdal.VSIFTellL(f) != 1000000:
gdal.VSIFCloseL(f)
pytest.fail(gdal.VSIFTellL(f))
gdal.VSIFSeekL(f, 0, 0)
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
current_time = int(time.time())
def method(request):
# We should go there after expiration of the first signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(302)
# Return a new signed URL
request.send_header(
'Location',
'http://localhost:%d/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d'
% (request.server.port, current_time + 30))
request.send_header('Content-Length', 16384)
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
handler.add('GET', '/test_redirect/test.bin', custom_method=method)
def method(request):
# Second signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 16384-16384/1000000')
request.send_header('Content-Length', 1)
request.end_headers()
request.wfile.write('y'.encode('ascii'))
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
with webserver.install_http_handler(handler):
content = gdal.VSIFReadL(1, 16383, f).decode('ascii')
if len(content) != 16383 or content[0] != 'x':
gdal.VSIFCloseL(f)
pytest.fail(content)
content = gdal.VSIFReadL(1, 2, f).decode('ascii')
if content != 'xy':
gdal.VSIFCloseL(f)
pytest.fail(content)
gdal.VSIFCloseL(f)
def test_ogr_wfs3_attribute_filter():
if gdaltest.wfs3_drv is None:
pytest.skip()
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/collections', 200, {'Content-Type': 'application/json'},
"""{ "collections" : [ {
"name": "foo"
}] }""")
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
lyr = ds.GetLayer(0)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"attr1": "",
"attr2": 0,
"attr3": ""
}
}
] }""")
# Fake openapi response
handler.add(
'GET', '/wfs3/api', 200, {'Content-Type': 'application/json'}, """{
"openapi": "3.0.0",
"paths" : {
"/collections/foo/items": {
"get": {
"parameters": [
{
"name": "attr1",
"in": "query"
},
{
"name": "attr2",
"in": "query"
}
]
}
}
}
}""")
with webserver.install_http_handler(handler):
lyr.SetAttributeFilter(
"(attr1 = 'foo' AND attr2 = 2) AND attr3 = 'bar'")
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/collections/foo/items?limit=10&attr1=foo&attr2=2', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"attr1": "foo",
"attr2": 2,
"attr3": "foo"
}
},
{
"type": "Feature",
"properties": {
"attr1": "foo",
"attr2": 2,
"attr3": "bar"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
assert f is not None
lyr.ResetReading()
lyr.SetAttributeFilter("attr1 = 'foo' OR attr3 = 'bar'")
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"attr1": "foo",
"attr3": "foo"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
assert f is not None
lyr.ResetReading()
lyr.SetAttributeFilter(None)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"attr1": "foo",
"attr3": "foo"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
assert f is not None
def vsigs_2():
if gdaltest.webserver_port == 0:
return 'skip'
# header file
gdal.FileFromMemBuffer('/vsimem/my_headers.txt', 'foo: bar')
def method(request):
if 'foo' not in request.headers or request.headers['foo'] != 'bar':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 1)
request.end_headers()
request.wfile.write("""Y""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET', '/gs_fake_bucket_http_header_file/resource', custom_method = method)
with webserver.install_http_handler(handler):
with gdaltest.config_option('GDAL_HTTP_HEADER_FILE', '/vsimem/my_headers.txt'):
f = open_for_read('/vsigs/gs_fake_bucket_http_header_file/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(data) != 1:
gdaltest.post_reason('fail')
return 'fail'
gdal.Unlink('/vsimem/my_headers.txt')
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID')
gdal.SetConfigOption('CPL_GS_TIMESTAMP', 'my_timestamp')
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'GOOG1 GS_ACCESS_KEY_ID:8tndu9//BfmN+Kg4AFLdUMZMBDQ='
if request.headers['Authorization'] != expected_authorization :
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET', '/gs_fake_bucket/resource', custom_method = method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET', '/gs_fake_bucket/resource', custom_method = method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs_streaming/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/gs_fake_bucket/resource2.bin', 200,
{'Content-Length': 1000000})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsigs/gs_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/gs_fake_bucket/resource2.bin', 200,
{'Content-Length': 1000000})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsigs_streaming/gs_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
return 'success'
def ogr_wfs3_errors():
if gdaltest.wfs3_drv is None:
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3', 404)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# No Content-Type
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3', 200, {}, 'foo')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# Unexpected Content-Type
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3', 200, {'Content-Type': 'text/html'}, 'foo')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid JSON
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3', 200, {'Content-Type': 'application/json'},
'foo bar')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# Valid JSON but not collections array
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3', 200, {'Content-Type': 'application/json'},
'{}')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# Valid JSON but collections is not an array
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3', 200, {'Content-Type': 'application/json'},
'{ "collections" : null }')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3', 200, {'Content-Type': 'application/json'},
'{ "collections" : [ null, {} ] }')
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayerCount() != 0:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayer(-1) is not None:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayer(0) is not None:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def ogr_wfs3_spatial_filter():
if gdaltest.wfs3_drv is None:
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3', 200, {'Content-Type': 'application/json'},
"""{ "collections" : [ {
"name": "foo",
"extent": {
"bbox": [ -10, 40, 15, 50 ]
}
}] }""")
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
lyr = ds.GetLayer(0)
if lyr.GetExtent() != (-10.0, 15.0, 40.0, 50.0):
gdaltest.post_reason('fail')
print(lyr.GetExtent())
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/api', 200, {'Content-Type': 'application/json'},
'{}')
handler.add(
'GET', '/wfs3/foo?count=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar"
}
}
] }""")
with webserver.install_http_handler(handler):
if lyr.GetLayerDefn().GetFieldCount() != 1:
gdaltest.post_reason('fail')
return 'fail'
lyr.SetSpatialFilterRect(2, 49, 3, 50)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/foo?count=10&bbox=2,49,3,50', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar",
},
"geometry": {
"type": "Point",
"coordinates": [ 2.5, 49.5 ]
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f is None:
gdaltest.post_reason('fail')
return 'fail'
lyr.SetSpatialFilter(None)
lyr.ResetReading()
handler.add(
'GET', '/wfs3/foo?count=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar",
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f is None:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def ogr_wfs3_fc_links_next_geojson():
if gdaltest.wfs3_drv is None:
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3', 200, {'Content-Type': 'application/json'},
'{ "collections" : [ { "name": "foo" }] }')
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
lyr = ds.GetLayer(0)
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/api', 200, {'Content-Type': 'application/json'},
'{}')
handler.add(
'GET', '/wfs3/foo?count=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar"
}
}
] }""")
with webserver.install_http_handler(handler):
if lyr.GetLayerDefn().GetFieldCount() != 1:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/foo?count=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection",
"links" : [
{ "rel": "next", "type": "application/geo+json", "href": "http://localhost:%d/wfs3/foo_next" }
],
"features": [
{
"type": "Feature",
"properties": {
"foo": "bar"
}
}
] }""" % gdaltest.webserver_port)
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f['foo'] != 'bar':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/foo_next', 200, {'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"foo": "baz"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f['foo'] != 'baz':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
return 'success'
def ogr_wfs3_fc_links_next_headers():
if gdaltest.wfs3_drv is None:
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3', 200, {'Content-Type': 'application/json'},
'{ "collections" : [ { "name": "foo" }] }')
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
lyr = ds.GetLayer(0)
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/api', 200, {'Content-Type': 'application/json'},
'{}')
handler.add(
'GET', '/wfs3/foo?count=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar"
}
}
] }""")
with webserver.install_http_handler(handler):
if lyr.GetLayerDefn().GetFieldCount() != 1:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
link_val = '<http://data.example.org/buildings.json>; rel="self"; type="application/geo+json"\r\nLink: <http://localhost:%d/wfs3/foo_next>; rel="next"; type="application/geo+json"' % gdaltest.webserver_port
handler.add(
'GET', '/wfs3/foo?count=10', 200, {
'Content-Type': 'application/geo+json',
'Link': link_val
}, """{ "type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"foo": "bar"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f['foo'] != 'bar':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/foo_next', 200, {'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"foo": "baz"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f['foo'] != 'baz':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
return 'success'
def ogr_wfs3_schema_from_api():
if gdaltest.wfs3_drv is None:
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3', 200, {'Content-Type': 'application/json'},
'{ "collections" : [ { "name": "foo" }] }')
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayerCount() != 1:
gdaltest.post_reason('fail')
return 'fail'
lyr = ds.GetLayer(0)
if lyr.GetName() != 'foo':
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
# Fake openapi response
handler.add(
'GET', '/wfs3/api', 200, {'Content-Type': 'application/json'}, """{
"openapi": "3.0.0",
"paths" : {
"/foo/{id}": {
"get": {
"responses": {
"200": {
"description": "A feature.",
"content": {
"application/geo+json": {
"schema": {
"$ref": "#/components/schemas/foo"
}
}
}
}
}
}
}
},
"components": {
"schemas": {
"foo": {
"type": "object",
"required": [
"id",
"type",
"geometry",
"properties"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Feature"
]
},
"geometry": {
"$ref": "#/components/schemas/geometryGeoJSON"
},
"id": {
"type": "string"
},
"properties": {
"type": "object",
"properties": {
"date_attr": {
"type": "string",
"format": "date"
},
"date_time_attr": {
"type": "string",
"format": "date-time"
},
"real_attr": {
"type": "number"
},
"float_attr": {
"type": "number",
"format": "float"
},
"int_attr": {
"type": "integer"
},
"int64_attr": {
"type": "integer",
"format": "int64"
},
"boolean_attr": {
"type": "boolean"
},
"string_attr": {
"type": "string"
}
}
}
}
}
}
}
}""")
with webserver.install_http_handler(handler):
if lyr.GetLayerDefn().GetFieldCount() != 8:
gdaltest.post_reason('fail')
return 'fail'
expected_attrs = [
("date_attr", ogr.OFTDate, ogr.OFSTNone),
("date_time_attr", ogr.OFTDateTime, ogr.OFSTNone),
("real_attr", ogr.OFTReal, ogr.OFSTNone),
("float_attr", ogr.OFTReal, ogr.OFSTFloat32),
("int_attr", ogr.OFTInteger, ogr.OFSTNone),
("int64_attr", ogr.OFTInteger64, ogr.OFSTNone),
("boolean_attr", ogr.OFTInteger, ogr.OFSTBoolean),
("string_attr", ogr.OFTString, ogr.OFSTNone),
]
for (attr_name, type, subtype) in expected_attrs:
fld_idx = lyr.GetLayerDefn().GetFieldIndex(attr_name)
if fld_idx < 0:
gdaltest.post_reason('fail')
print(attr_name)
return 'fail'
fld_defn = lyr.GetLayerDefn().GetFieldDefn(fld_idx)
if fld_defn.GetType() != type and fld_defn.GetSubType() != subtype:
gdaltest.post_reason('fail')
print(attr_name, fld_defn.GetType(), fld_defn.GetSubType())
return 'fail'
return 'success'
def test_visoss_6():
if gdaltest.webserver_port == 0:
pytest.skip()
with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB
with webserver.install_http_handler(webserver.SequentialHandler()):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket4/large_file.bin', 'wb')
assert f is not None
size = 1024 * 1024 + 1
big_buffer = 'a' * size
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
response = '<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>'
request.send_response(200)
request.send_header('Content-type', 'application/xml')
request.send_header('Content-Length', len(response))
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('POST',
'/oss_fake_bucket4/large_file.bin?uploads',
custom_method=method)
def method(request):
if request.headers['Content-Length'] != '1048576':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('ETag', '"first_etag"')
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/oss_fake_bucket4/large_file.bin?partNumber=1&uploadId=my_id',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL(big_buffer, 1, size, f)
assert ret == size
handler = webserver.SequentialHandler()
def method(request):
if request.headers['Content-Length'] != '1':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
return
request.send_response(200)
request.send_header('ETag', '"second_etag"')
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/oss_fake_bucket4/large_file.bin?partNumber=2&uploadId=my_id',
custom_method=method)
def method(request):
if request.headers['Content-Length'] != '186':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
content = request.rfile.read(186).decode('ascii')
if content != """<CompleteMultipartUpload>
<Part>
<PartNumber>1</PartNumber><ETag>"first_etag"</ETag></Part>
<Part>
<PartNumber>2</PartNumber><ETag>"second_etag"</ETag></Part>
</CompleteMultipartUpload>
""":
sys.stderr.write('Did not get expected content: %s\n' % content)
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('POST',
'/oss_fake_bucket4/large_file.bin?uploadId=my_id',
custom_method=method)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == ''
handler = webserver.SequentialHandler()
handler.add('POST',
'/oss_fake_bucket4/large_file_initiate_403_error.bin?uploads',
403)
handler.add(
'POST',
'/oss_fake_bucket4/large_file_initiate_empty_result.bin?uploads', 200)
handler.add(
'POST',
'/oss_fake_bucket4/large_file_initiate_invalid_xml_result.bin?uploads',
200, {}, 'foo')
handler.add(
'POST',
'/oss_fake_bucket4/large_file_initiate_no_uploadId.bin?uploads', 200,
{}, '<foo/>')
with webserver.install_http_handler(handler):
for filename in [
'/vsioss/oss_fake_bucket4/large_file_initiate_403_error.bin',
'/vsioss/oss_fake_bucket4/large_file_initiate_empty_result.bin',
'/vsioss/oss_fake_bucket4/large_file_initiate_invalid_xml_result.bin',
'/vsioss/oss_fake_bucket4/large_file_initiate_no_uploadId.bin'
]:
with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB
f = gdal.VSIFOpenL(filename, 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFWriteL(big_buffer, 1, size, f)
assert ret == 0
gdal.ErrorReset()
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == ''
handler = webserver.SequentialHandler()
handler.add(
'POST',
'/oss_fake_bucket4/large_file_upload_part_403_error.bin?uploads', 200,
{},
'<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>'
)
handler.add(
'PUT',
'/oss_fake_bucket4/large_file_upload_part_403_error.bin?partNumber=1&uploadId=my_id',
403)
handler.add(
'DELETE',
'/oss_fake_bucket4/large_file_upload_part_403_error.bin?uploadId=my_id',
204)
handler.add(
'POST', '/oss_fake_bucket4/large_file_upload_part_no_etag.bin?uploads',
200, {},
'<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>'
)
handler.add(
'PUT',
'/oss_fake_bucket4/large_file_upload_part_no_etag.bin?partNumber=1&uploadId=my_id',
200)
handler.add(
'DELETE',
'/oss_fake_bucket4/large_file_upload_part_no_etag.bin?uploadId=my_id',
204)
with webserver.install_http_handler(handler):
for filename in [
'/vsioss/oss_fake_bucket4/large_file_upload_part_403_error.bin',
'/vsioss/oss_fake_bucket4/large_file_upload_part_no_etag.bin'
]:
with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB
f = gdal.VSIFOpenL(filename, 'wb')
assert f is not None, filename
with gdaltest.error_handler():
ret = gdal.VSIFWriteL(big_buffer, 1, size, f)
assert ret == 0, filename
gdal.ErrorReset()
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == '', filename
# Simulate failure in AbortMultipart stage
handler = webserver.SequentialHandler()
handler.add(
'POST',
'/oss_fake_bucket4/large_file_abortmultipart_403_error.bin?uploads',
200, {},
'<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>'
)
handler.add(
'PUT',
'/oss_fake_bucket4/large_file_abortmultipart_403_error.bin?partNumber=1&uploadId=my_id',
403)
handler.add(
'DELETE',
'/oss_fake_bucket4/large_file_abortmultipart_403_error.bin?uploadId=my_id',
403)
filename = '/vsioss/oss_fake_bucket4/large_file_abortmultipart_403_error.bin'
with webserver.install_http_handler(handler):
with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB
f = gdal.VSIFOpenL(filename, 'wb')
assert f is not None, filename
with gdaltest.error_handler():
ret = gdal.VSIFWriteL(big_buffer, 1, size, f)
assert ret == 0, filename
gdal.ErrorReset()
with gdaltest.error_handler():
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() != '', filename
# Simulate failure in CompleteMultipartUpload stage
handler = webserver.SequentialHandler()
handler.add(
'POST',
'/oss_fake_bucket4/large_file_completemultipart_403_error.bin?uploads',
200, {},
'<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>'
)
handler.add(
'PUT',
'/oss_fake_bucket4/large_file_completemultipart_403_error.bin?partNumber=1&uploadId=my_id',
200, {'ETag': 'first_etag'}, '')
handler.add(
'PUT',
'/oss_fake_bucket4/large_file_completemultipart_403_error.bin?partNumber=2&uploadId=my_id',
200, {'ETag': 'second_etag'}, '')
handler.add(
'POST',
'/oss_fake_bucket4/large_file_completemultipart_403_error.bin?uploadId=my_id',
403)
# handler.add('DELETE', '/oss_fake_bucket4/large_file_completemultipart_403_error.bin?uploadId=my_id', 204)
filename = '/vsioss/oss_fake_bucket4/large_file_completemultipart_403_error.bin'
with webserver.install_http_handler(handler):
with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB
f = gdal.VSIFOpenL(filename, 'wb')
assert f is not None, filename
ret = gdal.VSIFWriteL(big_buffer, 1, size, f)
assert ret == size, filename
gdal.ErrorReset()
with gdaltest.error_handler():
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() != '', filename
def test_visoss_4():
if gdaltest.webserver_port == 0:
pytest.skip()
with webserver.install_http_handler(webserver.SequentialHandler()):
with gdaltest.error_handler():
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3', 'wb')
assert f is None
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_fake_bucket3/empty_file.bin', 200,
{'Connection': 'close'}, 'foo')
with webserver.install_http_handler(handler):
assert gdal.VSIStatL(
'/vsioss/oss_fake_bucket3/empty_file.bin').size == 3
# Empty file
handler = webserver.SequentialHandler()
def method(request):
if request.headers['Content-Length'] != '0':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/oss_fake_bucket3/empty_file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb')
assert f is not None
gdal.ErrorReset()
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == ''
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_fake_bucket3/empty_file.bin', 200,
{'Connection': 'close'}, '')
with webserver.install_http_handler(handler):
assert gdal.VSIStatL(
'/vsioss/oss_fake_bucket3/empty_file.bin').size == 0
# Invalid seek
handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
assert ret != 0
gdal.VSIFCloseL(f)
# Invalid read
handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
assert not ret
gdal.VSIFCloseL(f)
# Error case
handler = webserver.SequentialHandler()
handler.add('PUT', '/oss_fake_bucket3/empty_file_error.bin', 403)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file_error.bin',
'wb')
assert f is not None
gdal.ErrorReset()
with gdaltest.error_handler():
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() != ''
# Nominal case
with webserver.install_http_handler(webserver.SequentialHandler()):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/another_file.bin', 'wb')
assert f is not None
assert gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) == 0
assert gdal.VSIFSeekL(f, 0, 1) == 0
assert gdal.VSIFSeekL(f, 0, 2) == 0
assert gdal.VSIFWriteL('foo', 1, 3, f) == 3
assert gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) == 0
assert gdal.VSIFWriteL('bar', 1, 3, f) == 3
handler = webserver.SequentialHandler()
def method(request):
if request.headers['Content-Length'] != '6':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(6).decode('ascii')
if content != 'foobar':
sys.stderr.write('Did not get expected content: %s\n' % content)
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/oss_fake_bucket3/another_file.bin',
custom_method=method)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == ''
def test_ogr_wfs3_spatial_filter():
if gdaltest.wfs3_drv is None:
pytest.skip()
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/collections', 200, {'Content-Type': 'application/json'},
"""{ "collections" : [ {
"name": "foo",
"extent": {
"spatial": [ -10, 40, 15, 50 ]
}
}] }""")
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
lyr = ds.GetLayer(0)
assert lyr.GetExtent() == (-10.0, 15.0, 40.0, 50.0)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar"
}
}
] }""")
with webserver.install_http_handler(handler):
assert lyr.GetLayerDefn().GetFieldCount() == 1
lyr.SetSpatialFilterRect(2, 49, 3, 50)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/collections/foo/items?limit=10&bbox=2,49,3,50', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar",
},
"geometry": {
"type": "Point",
"coordinates": [ 2.5, 49.5 ]
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
assert f is not None
lyr.SetSpatialFilter(None)
lyr.ResetReading()
handler.add(
'GET', '/wfs3/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar",
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
assert f is not None
def vsiaz_fake_basic():
if gdaltest.webserver_port == 0:
return 'skip'
signed_url = gdal.GetSignedURL('/vsiaz/az_fake_bucket/resource',
['START_DATE=20180213T123456'])
if signed_url not in (
'http://127.0.0.1:8080/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12',
'http://127.0.0.1:8081/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12'
):
gdaltest.post_reason('fail')
print(signed_url)
return 'fail'
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:zKb0EXnM/RinBjcUE9EU+qfRIGaIItoUElSWc+FE24E=' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET',
'/azure/blob/myaccount/az_fake_bucket/resource',
custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz/az_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:8d6IEeOsl7qGpKAxaTTxx2xMNpvqWq8DGlFE67lsmQ4=' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'Accept-Encoding' not in h or h['Accept-Encoding'] != 'gzip':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET',
'/azure/blob/myaccount/az_fake_bucket/resource',
custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz_streaming/az_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin',
200, {'Content-Length': '1000000'})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsiaz/az_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin',
200, {'Content-Length': 1000000})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL(
'/vsiaz_streaming/az_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
return 'success'
def test_ogr_wfs3_get_feature_count():
if gdaltest.wfs3_drv is None:
pytest.skip()
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/wfs3/collections', 200, {'Content-Type': 'application/json'},
"""{ "collections" : [ {
"name": "foo",
"extent": {
"spatial": [ -10, 40, 15, 50 ]
}
}] }""")
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
lyr = ds.GetLayer(0)
handler = webserver.SequentialHandler()
# Fake openapi response
handler.add(
'GET', '/wfs3/api', 200, {'Content-Type': 'application/json'}, """{
"openapi": "3.0.0",
"paths" : {
"/collections/foo/items": {
"get": {
"parameters": [
{
"$ref" : "#/components/parameters/resultType"
}
]
}
}
},
"components": {
"parameters": {
"resultType": {
"name": "resultType",
"in": "query",
"schema" : {
"type" : "string",
"enum" : [ "hits", "results" ]
}
}
}
}
}""")
handler.add('GET', '/wfs3/collections/foo/items?resultType=hits', 200,
{'Content-Type': 'application/json'},
'{ "numberMatched": 1234 }')
with webserver.install_http_handler(handler):
assert lyr.GetFeatureCount() == 1234
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections/foo/items?resultType=hits', 200,
{'Content-Type': 'application/json'},
'{ "numberMatched": 1234 }')
with webserver.install_http_handler(handler):
assert lyr.GetFeatureCount() == 1234
def vsiaz_fake_readdir():
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%2F&restype=container',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Blobs>
<Blob>
<Name>a_dir/resource3.bin</Name>
<Properties>
<Last-Modified>01 Jan 1970 00:00:01</Last-Modified>
<Content-Length>123456</Content-Length>
</Properties>
</Blob>
</Blobs>
</EnumerationResults>
""")
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla&prefix=a_dir%2F&restype=container',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<Blobs>
<Blob>
<Name>a_dir/resource4.bin</Name>
<Properties>
<Last-Modified>16 Oct 2016 12:34:56</Last-Modified>
<Content-Length>456789</Content-Length>
</Properties>
</Blob>
<BlobPrefix>
<Name>a_dir/subdir/</Name>
</BlobPrefix>
</Blobs>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL(
'/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
print(gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size)
return 'fail'
if gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
if dir_contents is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=error_test%2F&restype=container',
500)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/error_test/')
if dir_contents is not None:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# List containers (empty result)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/azure/blob/myaccount/?comp=list', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults ServiceEndpoint="https://myaccount.blob.core.windows.net">
<Containers/>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
if dir_contents != ['.']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
gdal.VSICurlClearCache()
# List containers
handler = webserver.SequentialHandler()
handler.add(
'GET', '/azure/blob/myaccount/?comp=list', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer1</Name>
</Container>
</Containers>
<NextMarker>bla</NextMarker>
</EnumerationResults>
""")
handler.add(
'GET', '/azure/blob/myaccount/?comp=list&marker=bla', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer2</Name>
</Container>
</Containers>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
if dir_contents != ['mycontainer1', 'mycontainer2']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
return 'success'
def test_ogr_wfs3_errors():
if gdaltest.wfs3_drv is None:
pytest.skip()
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 404)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
assert ds is None
# No Content-Type
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200, {}, 'foo')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
assert ds is None
# Unexpected Content-Type
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200, {'Content-Type': 'text/html'},
'foo')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
assert ds is None
# Invalid JSON
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200,
{'Content-Type': 'application/json'}, 'foo bar')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
assert ds is None
# Valid JSON but not collections array
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200,
{'Content-Type': 'application/json'}, '{}')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
assert ds is None
# Valid JSON but collections is not an array
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200,
{'Content-Type': 'application/json'},
'{ "collections" : null }')
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
assert ds is None
handler = webserver.SequentialHandler()
handler.add('GET', '/wfs3/collections', 200,
{'Content-Type': 'application/json'},
'{ "collections" : [ null, {} ] }')
with webserver.install_http_handler(handler):
ds = ogr.Open('WFS3:http://localhost:%d/wfs3' %
gdaltest.webserver_port)
assert ds is not None
assert ds.GetLayerCount() == 0
assert ds.GetLayer(-1) is None
assert ds.GetLayer(0) is None
def vsiaz_fake_write():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Test creation of BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:AigkrY7q66WCrx3JRKBte56k7kxV2cxB/ZyGNubxk5I=' or \
'Expect' not in h or h['Expect'] != '100-continue' or \
'Content-Length' not in h or h['Content-Length'] != '40000' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'BlockBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(40000).decode('ascii')
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Simulate illegal read
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
if ret:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
gdal.VSIFCloseL(f)
# Simulate illegal seek
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
# Simulate failure when putting BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
if gdal.VSIFSeekL(f, 0, 0) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if gdal.VSIFTellL(f) != 35000:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 35000, 0) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 0, 1) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 0, 2) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFEofL(f) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFCloseL(f)
if ret == 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
# Simulate creation of BlockBob over an existing blob of incompatible type
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 409)
handler.add('DELETE', '/azure/blob/myaccount/test_copy/file.bin', 202)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
# Test creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:KimVui3ptY9D5ftLlsI7CNOgK36CNAEzsXqcuHskdEY=' or \
'Content-Length' not in h or h['Content-Length'] != '0' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '10' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(10).decode('ascii')
if content != '0123456789':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '6' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(6).decode('ascii')
if content != 'abcdef':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 16:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Test failed creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Test failed writing of a block of an AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
403)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
return 'success'
def vsigs_read_credentials_oauth2_service_account():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
# Generated with 'openssl genrsa -out rsa-openssl.pem 1024' and
# 'openssl pkcs8 -nocrypt -in rsa-openssl.pem -inform PEM -topk8 -outform PEM -out rsa-openssl.pkcs8.pem'
# DO NOT USE in production !!!!
key = """-----BEGIN PRIVATE KEY-----
MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk
VNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm
i2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ
DVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy
TabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I
2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd
DrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+
bpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg
Wz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3
4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5
Kvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj
j+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI
yzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq
gwE6fxOLyJDxuWRf
-----END PRIVATE KEY-----
"""
for i in range(2):
gdal.SetConfigOption('GO2A_AUD',
'http://localhost:%d/oauth2/v4/token' % gdaltest.webserver_port)
gdal.SetConfigOption('GOA2_NOW', '123456')
if i == 0:
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY', key)
else:
gdal.FileFromMemBuffer('/vsimem/pkey', key)
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY_FILE', '/vsimem/pkey')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_EMAIL', 'CLIENT_EMAIL')
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(request.headers['Content-Length'])).decode('ascii')
if content != 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODAvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.DAhqWtBgKpObxZ%2BGiXqwF%2Fa4SS%2FNWQRhLCI7DYZCuOTuf2w7dL8j4CdpiwwzQg1diIus7dyViRfzpsFmuZKAXwL%2B84iBoVVqnJJZ4TgwH49NdfMAnc4Rgm%2Bo2a2nEcMjX%2FbQ3jRY%2B9WNVl96hzULGvLrVeyego2f06wivqmvxHA%3D':
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST', '/oauth2/v4/token', custom_method = method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization :
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method = method)
try:
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
except:
if gdal.GetLastErrorMsg().find('CPLRSASHA256Sign() not implemented') >= 0:
return 'skip'
finally:
gdal.SetConfigOption('GO2A_AUD', None)
gdal.SetConfigOption('GO2A_NOW', None)
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY', '')
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY_FILE', '')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_EMAIL', '')
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.Unlink('/vsimem/pkey')
return 'success'
def vsiaz_fake_mkdir_rmdir():
if gdaltest.webserver_port == 0:
return 'skip'
# Invalid name
ret = gdal.Mkdir('/vsiaz', 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_mkdir/dir/', 404,
{'Connection': 'close'})
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=%2F&maxresults=1&prefix=dir%2F&restype=container',
200, {'Connection': 'close'})
handler.add(
'PUT',
'/azure/blob/myaccount/az_bucket_test_mkdir/dir/.gdal_marker_for_dir',
201)
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsiaz/az_bucket_test_mkdir/dir', 0)
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Try creating already existing directory
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_mkdir/dir/', 404)
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=%2F&maxresults=1&prefix=dir%2F&restype=container',
200, {
'Connection': 'close',
'Content-type': 'application/xml'
}, """<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>dir/</Prefix>
<Blobs>
<Blob>
<Name>dir/.gdal_marker_for_dir</Name>
</Blob>
</Blobs>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsiaz/az_bucket_test_mkdir/dir', 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Invalid name
ret = gdal.Rmdir('/vsiaz')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Not a directory
handler = webserver.SequentialHandler()
handler.add('HEAD',
'/azure/blob/myaccount/az_bucket_test_mkdir/it_is_a_file/',
404)
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=%2F&maxresults=1&prefix=it_is_a_file%2F&restype=container',
200, {
'Connection': 'close',
'Content-type': 'application/xml'
}, """<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>it_is_a_file/</Prefix>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiaz/az_bucket_test_mkdir/it_is_a_file')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Valid
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=%2F&maxresults=1&prefix=dir%2F&restype=container',
200, {
'Connection': 'close',
'Content-type': 'application/xml'
}, """<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>dir/</Prefix>
<Blobs>
<Blob>
<Name>dir/.gdal_marker_for_dir</Name>
</Blob>
</Blobs>
</EnumerationResults>
""")
handler.add(
'DELETE',
'/azure/blob/myaccount/az_bucket_test_mkdir/dir/.gdal_marker_for_dir',
202)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiaz/az_bucket_test_mkdir/dir')
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Try deleting already deleted directory
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_bucket_test_mkdir/dir/', 404)
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=%2F&maxresults=1&prefix=dir%2F&restype=container',
200)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiaz/az_bucket_test_mkdir/dir')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Try deleting non-empty directory
handler = webserver.SequentialHandler()
handler.add('HEAD',
'/azure/blob/myaccount/az_bucket_test_mkdir/dir_nonempty/',
404)
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=%2F&maxresults=1&prefix=dir_nonempty%2F&restype=container',
200, {
'Connection': 'close',
'Content-type': 'application/xml'
}, """<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>dir_nonempty/</Prefix>
<Blobs>
<Blob>
<Name>dir_nonempty/foo</Name>
</Blob>
</Blobs>
</EnumerationResults>
""")
handler.add(
'GET',
'/azure/blob/myaccount/az_bucket_test_mkdir?comp=list&delimiter=%2F&maxresults=1&prefix=dir_nonempty%2F&restype=container',
200, {
'Connection': 'close',
'Content-type': 'application/xml'
}, """<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>dir_nonempty/</Prefix>
<Blobs>
<Blob>
<Name>dir_nonempty/foo</Name>
</Blob>
</Blobs>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiaz/az_bucket_test_mkdir/dir_nonempty')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsigs_read_credentials_gce_expiration():
if gdaltest.webserver_port == 0:
return 'skip'
if sys.platform not in ('linux', 'linux2', 'win32'):
return 'skip'
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '')
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL',
'http://localhost:%d/computeMetadata/v1/instance/service-accounts/default/token' % gdaltest.webserver_port)
# Disable hypervisor related check to test if we are really on EC2
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', 'NO')
gdal.VSICurlClearCache()
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization :
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
# First time is used when trying to establish if GCE authentication is available
handler.add('GET', '/computeMetadata/v1/instance/service-accounts/default/token', 200, {},
"""{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 0,
}""")
# Second time is needed because f the access to th file
handler.add('GET', '/computeMetadata/v1/instance/service-accounts/default/token', 200, {},
"""{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 0,
}""")
handler.add('GET', '/gs_fake_bucket/resource', custom_method = method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL','')
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', None)
return 'success'
def vsiswift_fake_auth_v1_url():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
gdal.SetConfigOption(
'SWIFT_AUTH_V1_URL',
'http://127.0.0.1:%d/auth/1.0' % gdaltest.webserver_port)
gdal.SetConfigOption('SWIFT_USER', 'my_user')
gdal.SetConfigOption('SWIFT_KEY', 'my_key')
gdal.SetConfigOption('SWIFT_STORAGE_URL', '')
gdal.SetConfigOption('SWIFT_AUTH_TOKEN', '')
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'X-Auth-User' not in h or h['X-Auth-User'] != 'my_user' or \
'X-Auth-Key' not in h or h['X-Auth-Key'] != 'my_key':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.send_header(
'X-Storage-Url',
'http://127.0.0.1:%d/v1/AUTH_something' % gdaltest.webserver_port)
request.send_header('X-Auth-Token', 'my_auth_token')
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/auth/1.0', custom_method=method)
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'x-auth-token' not in h or \
h['x-auth-token'] != 'my_auth_token':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/v1/AUTH_something/foo/bar', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiswift/foo/bar')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
# authentication is reused
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'x-auth-token' not in h or \
h['x-auth-token'] != 'my_auth_token':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""bar""".encode('ascii'))
handler.add('GET', '/v1/AUTH_something/foo/baz', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiswift/foo/baz')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'bar':
gdaltest.post_reason('fail')
print(data)
return 'fail'
return 'success'
def test_eedai_gce_credentials():
if gdaltest.eedai_drv is None:
pytest.skip()
if sys.platform not in ('linux', 'linux2', 'win32'):
pytest.skip()
gdaltest.webserver_process = None
gdaltest.webserver_port = 0
if not gdaltest.built_against_curl():
pytest.skip()
(gdaltest.webserver_process, gdaltest.webserver_port) = webserver.launch(handler=webserver.DispatcherHttpHandler)
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL',
'http://localhost:%d/computeMetadata/v1/instance/service-accounts/default/token' % gdaltest.webserver_port)
# Disable hypervisor related check to test if we are really on EC2
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', 'NO')
gdal.VSICurlClearCache()
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET', '/computeMetadata/v1/instance/service-accounts/default/token', 200, {},
"""{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}""")
with webserver.install_http_handler(handler):
gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/')
ds = gdal.Open('EEDAI:image')
gdal.SetConfigOption('EEDA_URL', None)
gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL', None)
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', None)
webserver.server_stop(gdaltest.webserver_process, gdaltest.webserver_port)
assert ds is not None
def vsiswift_fake_readdir():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=1', 200,
{'Content-type': 'application/json'}, """[
{
"last_modified": "1970-01-01T00:00:01",
"bytes": 123456,
"name": "bar.baz"
}
]""")
handler.add('GET',
'/v1/AUTH_something/foo?delimiter=%2F&limit=1&marker=bar.baz',
200, {'Content-type': 'application/json'}, """[
{
"subdir": "mysubdir/"
}
]""")
handler.add(
'GET',
'/v1/AUTH_something/foo?delimiter=%2F&limit=1&marker=mysubdir%2F', 200,
{'Content-type': 'application/json'}, """[
]""")
with gdaltest.config_option('SWIFT_MAX_KEYS', '1'):
with webserver.install_http_handler(handler):
f = open_for_read('/vsiswift/foo/bar.baz')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsiswift/foo')
if dir_contents != ['bar.baz', 'mysubdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
stat_res = gdal.VSIStatL('/vsiswift/foo/bar.baz')
if stat_res.size != 123456:
gdaltest.post_reason('fail')
print(stat_res.size)
return 'fail'
if stat_res.mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsiswift/foo/bar.baz')
if dir_contents is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/v1/AUTH_something/foo?delimiter=%2F&limit=10000&prefix=error_test%2F',
500)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/foo/error_test/')
if dir_contents is not None:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# List containers (empty result)
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something', 200,
{'Content-type': 'application/json'}, """[]
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/')
if dir_contents != ['.']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# List containers
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/v1/AUTH_something', 200, {'Content-type': 'application/json'},
"""[ { "name": "mycontainer1", "count": 0, "bytes": 0 },
{ "name": "mycontainer2", "count": 0, "bytes": 0}
] """)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/')
if dir_contents != ['mycontainer1', 'mycontainer2']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# ReadDir() with a file and directory of same names
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/v1/AUTH_something', 200, {'Content-type': 'application/json'},
"""[ {
"last_modified": "1970-01-01T00:00:01",
"bytes": 123456,
"name": "foo"
},
{ "subdir": "foo/"} ] """)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/')
if dir_contents != ['foo', 'foo/']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200,
{'Content-type': 'application/json'}, "[]")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/foo/')
if dir_contents != ['.']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
return 'success'
def NO_LONGER_USED_test_ogr_opaif_fc_links_next_headers():
if gdaltest.opaif_drv is None:
pytest.skip()
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections', 200, {'Content-Type': 'application/json'},
'{ "collections" : [ { "name": "foo" }] }')
with webserver.install_http_handler(handler):
ds = ogr.Open('OAPIF:http://localhost:%d/oapif' % gdaltest.webserver_port)
lyr = ds.GetLayer(0)
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection", "features": [
{
"type": "Feature",
"properties": {
"foo": "bar"
}
}
] }""")
with webserver.install_http_handler(handler):
assert lyr.GetLayerDefn().GetFieldCount() == 1
handler = webserver.SequentialHandler()
link_val = '<http://data.example.org/buildings.json>; rel="self"; type="application/geo+json"\r\nLink: <http://localhost:%d/oapif/foo_next>; rel="next"; type="application/geo+json"' % gdaltest.webserver_port
handler.add('GET', '/oapif/collections/foo/items?limit=10', 200,
{'Content-Type': 'application/geo+json',
'Link': link_val},
"""{ "type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"foo": "bar"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f['foo'] != 'bar':
f.DumpReadable()
pytest.fail()
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/foo_next', 200,
{'Content-Type': 'application/geo+json'},
"""{ "type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"foo": "baz"
}
}
] }""")
with webserver.install_http_handler(handler):
f = lyr.GetNextFeature()
if f['foo'] != 'baz':
f.DumpReadable()
pytest.fail()
def vsiswift_fake_mkdir_rmdir():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Invalid name
ret = gdal.Mkdir('/vsiswift', 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/dir/', 404,
{'Connection': 'close'})
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200,
{'Connection': 'close'}, "[]")
handler.add('PUT', '/v1/AUTH_something/foo/dir/', 201)
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsiswift/foo/dir', 0)
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Try creating already existing directory
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/dir/', 404,
{'Connection': 'close'})
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200,
{
'Connection': 'close',
'Content-type': 'application/json'
}, """[ { "subdir": "dir/" } ]""")
with webserver.install_http_handler(handler):
ret = gdal.Mkdir('/vsiswift/foo/dir', 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Invalid name
ret = gdal.Rmdir('/vsiswift')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSICurlClearCache()
# Not a directory
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/it_is_a_file/', 404)
handler.add(
'GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200, {
'Connection': 'close',
'Content-type': 'application/json'
},
"""[ { "name": "it_is_a_file/", "bytes": 0, "last_modified": "1970-01-01T00:00:01" } ]"""
)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiswift/foo/it_is_a_file')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# Valid
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/dir/', 200)
handler.add(
'GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=101&prefix=dir%2F',
200, {
'Connection': 'close',
'Content-type': 'application/json'
}, """[]
""")
handler.add('DELETE', '/v1/AUTH_something/foo/dir/', 204)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiswift/foo/dir')
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
# Try deleting already deleted directory
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/dir/', 404)
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiswift/foo/dir')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSICurlClearCache()
# Try deleting non-empty directory
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo/dir_nonempty/', 404)
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200,
{
'Connection': 'close',
'Content-type': 'application/json'
}, """[ { "subdir": "dir_nonempty/" } ]""")
handler.add(
'GET',
'/v1/AUTH_something/foo?delimiter=%2F&limit=101&prefix=dir_nonempty%2F',
200, {
'Connection': 'close',
'Content-type': 'application/json'
},
"""[ { "name": "dir_nonempty/some_file", "bytes": 0, "last_modified": "1970-01-01T00:00:01" } ]"""
)
with webserver.install_http_handler(handler):
ret = gdal.Rmdir('/vsiswift/foo/dir_nonempty')
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def test_ogr_opaif_get_feature_count():
if gdaltest.opaif_drv is None:
pytest.skip()
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections', 200, {'Content-Type': 'application/json'},
"""{ "collections" : [ {
"id": "foo"
}] }""")
with webserver.install_http_handler(handler):
ds = ogr.Open('OAPIF:http://localhost:%d/oapif' % gdaltest.webserver_port)
lyr = ds.GetLayer(0)
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections/foo/items?limit=10', 200, {'Content-Type': 'application/json'}, '{}')
handler.add('GET', '/oapif', 200, {'Content-Type': 'application/json'},
"""{ "links" : [ { "rel": "service-desc",
"href" : "http://localhost:%d/oapif/my_api",
"type": "application/vnd.oai.openapi+json;version=3.0" } ] }""" % gdaltest.webserver_port )
# Fake openapi response
handler.add('GET', '/oapif/my_api', 200, {'Content-Type': 'application/vnd.oai.openapi+json; charset=utf-8; version=3.0'},
"""{
"openapi": "3.0.0",
"paths" : {
"/collections/foo/items": {
"get": {
"parameters": [
{
"$ref" : "#/components/parameters/resultType"
}
]
}
}
},
"components": {
"parameters": {
"resultType": {
"name": "resultType",
"in": "query",
"schema" : {
"type" : "string",
"enum" : [ "hits", "results" ]
}
}
}
}
}""")
handler.add('GET', '/oapif/collections/foo/items?resultType=hits', 200,
{'Content-Type': 'application/json'},
'{ "numberMatched": 1234 }')
with webserver.install_http_handler(handler):
assert lyr.GetFeatureCount() == 1234
handler = webserver.SequentialHandler()
handler.add('GET', '/oapif/collections/foo/items?resultType=hits', 200,
{'Content-Type': 'application/json'},
'{ "numberMatched": 1234 }')
with webserver.install_http_handler(handler):
assert lyr.GetFeatureCount() == 1234
def test_visoss_2():
if gdaltest.webserver_port == 0:
pytest.skip()
signed_url = gdal.GetSignedURL('/vsioss/oss_fake_bucket/resource',
['START_DATE=20180212T123456Z'])
assert (signed_url in (
'http://127.0.0.1:8080/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D',
'http://127.0.0.1:8081/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D'
))
handler = webserver.SequentialHandler()
handler.add('GET',
'/oss_fake_bucket/resource',
custom_method=get_oss_fake_bucket_resource_method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
handler.add('GET',
'/oss_fake_bucket/resource',
custom_method=get_oss_fake_bucket_resource_method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss_streaming/oss_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(400)
response = """<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>AccessDenied</Code>
<Message>The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.</Message>
<HostId>unused</HostId>
<Bucket>unuset</Bucket>
<Endpoint>localhost:%d</Endpoint>
</Error>""" % request.server.port
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
request.send_header('Content-type', 'application/xml')
request.send_header('Transfer-Encoding', 'chunked')
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method)
def method(request):
request.protocol_version = 'HTTP/1.1'
if request.headers['Host'].startswith('localhost'):
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
else:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method)
# Test region and endpoint 'redirects'
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_fake_bucket/redirect')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
pytest.fail(data)
# Test region and endpoint 'redirects'
handler.req_count = 0
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss_streaming/oss_fake_bucket/redirect')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
def method(request):
# /vsioss_streaming/ should have remembered the change of region and endpoint
if not request.headers['Host'].startswith('localhost'):
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.protocol_version = 'HTTP/1.1'
request.send_response(400)
response = 'bla'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
request.send_header('Content-type', 'application/xml')
request.send_header('Transfer-Encoding', 'chunked')
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET', '/oss_fake_bucket/non_xml_error', custom_method=method)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/non_xml_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('bla') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><oops>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/invalid_xml_error', 400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/invalid_xml_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<oops>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error/>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/no_code_in_error', 400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_code_in_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error/>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>AuthorizationHeaderMalformed</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET',
'/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error',
400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error'
)
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>PermanentRedirect</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error', 400,
{
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error'
)
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>bla</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/no_message_in_error', 400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_message_in_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0