def vsigs_extra_1():
if not gdaltest.built_against_curl():
return 'skip'
if gdal.GetConfigOption('GS_SECRET_ACCESS_KEY') is None:
print('Missing GS_SECRET_ACCESS_KEY for running gdaltest_list_extra')
return 'skip'
elif gdal.GetConfigOption('GS_ACCESS_KEY_ID') is None:
print('Missing GS_ACCESS_KEY_ID for running gdaltest_list_extra')
return 'skip'
elif gdal.GetConfigOption('GS_RESOURCE') is None:
print('Missing GS_RESOURCE for running gdaltest_list_extra')
return 'skip'
f = open_for_read('/vsigs/' + gdal.GetConfigOption('GS_RESOURCE'))
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Same with /vsigs_streaming/
f = open_for_read('/vsigs_streaming/' + gdal.GetConfigOption('GS_RESOURCE'))
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsigs/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if gdal.VSIGetLastErrorMsg() == '':
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsigs_streaming/' + gdal.GetConfigOption('GS_RESOURCE') + '/invalid_resource.baz')
if f is not None:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def test_visoss_1():
if not gdaltest.built_against_curl():
pytest.skip()
# Missing OSS_SECRET_ACCESS_KEY
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsioss/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'OSS_SECRET_ACCESS_KEY') >= 0
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsioss_streaming/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'OSS_SECRET_ACCESS_KEY') >= 0
gdal.SetConfigOption('OSS_SECRET_ACCESS_KEY', 'OSS_SECRET_ACCESS_KEY')
# Missing OSS_ACCESS_KEY_ID
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsioss/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'OSS_ACCESS_KEY_ID') >= 0
gdal.SetConfigOption('OSS_ACCESS_KEY_ID', 'OSS_ACCESS_KEY_ID')
def test_vsiaz_real_server_errors():
if not gdaltest.built_against_curl():
pytest.skip()
# Missing AZURE_STORAGE_ACCOUNT
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiaz/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'AZURE_STORAGE_ACCOUNT') >= 0
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiaz_streaming/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'AZURE_STORAGE_ACCOUNT') >= 0
# Invalid AZURE_STORAGE_CONNECTION_STRING
with gdaltest.config_option('AZURE_STORAGE_CONNECTION_STRING', 'invalid'):
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiaz/foo/bar')
assert f is None
# Missing AZURE_STORAGE_ACCESS_KEY
gdal.ErrorReset()
with gdaltest.config_options({
'AZURE_STORAGE_ACCOUNT': 'AZURE_STORAGE_ACCOUNT',
'CPL_AZURE_VM_API_ROOT_URL': 'disabled'
}):
with gdaltest.error_handler():
f = open_for_read('/vsiaz/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'AZURE_STORAGE_ACCESS_KEY') >= 0
# AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_ACCESS_KEY but invalid
gdal.ErrorReset()
with gdaltest.config_options({
'AZURE_STORAGE_ACCOUNT':
'AZURE_STORAGE_ACCOUNT',
'AZURE_STORAGE_ACCESS_KEY':
'AZURE_STORAGE_ACCESS_KEY'
}):
with gdaltest.error_handler():
f = open_for_read('/vsiaz/foo/bar.baz')
if f is not None:
if f is not None:
gdal.VSIFCloseL(f)
if gdal.GetConfigOption('APPVEYOR') is not None:
return
pytest.fail(gdal.VSIGetLastErrorMsg())
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiaz_streaming/foo/bar.baz')
assert f is None, gdal.VSIGetLastErrorMsg()
def vsifile_22():
# VSIOpenL doesn't set errorno
gdal.VSIErrorReset()
if gdal.VSIGetLastErrorNo() != 0:
gdaltest.post_reason("Expected Err=0 after VSIErrorReset(), got %d" % gdal.VSIGetLastErrorNo())
return 'fail'
fp = gdal.VSIFOpenL('tmp/not-existing', 'r')
if fp is not None:
gdaltest.post_reason("Expected None from VSIFOpenL")
return 'fail'
if gdal.VSIGetLastErrorNo() != 0:
gdaltest.post_reason("Expected Err=0 from VSIFOpenL, got %d" % gdal.VSIGetLastErrorNo())
return 'fail'
# VSIOpenExL does
fp = gdal.VSIFOpenExL('tmp/not-existing', 'r', 1)
if fp is not None:
gdaltest.post_reason("Expected None from VSIFOpenExL")
return 'fail'
if gdal.VSIGetLastErrorNo() != 1:
gdaltest.post_reason("Expected Err=1 from VSIFOpenExL, got %d" % gdal.VSIGetLastErrorNo())
return 'fail'
if len(gdal.VSIGetLastErrorMsg()) == 0:
gdaltest.post_reason("Expected a VSI error message")
return 'fail'
gdal.VSIErrorReset()
if gdal.VSIGetLastErrorNo() != 0:
gdaltest.post_reason("Expected Err=0 after VSIErrorReset(), got %d" % gdal.VSIGetLastErrorNo())
return 'fail'
return 'success'
def vsiswift_real_server_errors():
if not gdaltest.built_against_curl():
return 'skip'
# Nothing set
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiswift/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find(
'SWIFT_STORAGE_URL') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiswift_streaming/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find(
'SWIFT_STORAGE_URL') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.SetConfigOption('SWIFT_STORAGE_URL', 'http://0.0.0.0')
# Missing SWIFT_AUTH_TOKEN
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiswift/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find('SWIFT_AUTH_TOKEN') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.SetConfigOption('SWIFT_AUTH_TOKEN', 'SWIFT_AUTH_TOKEN')
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiswift/foo/bar.baz')
if f is not None:
if f is not None:
gdal.VSIFCloseL(f)
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiswift_streaming/foo/bar.baz')
if f is not None:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def test_vsiswift_real_server_errors():
if not gdaltest.built_against_curl():
pytest.skip()
# Nothing set
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiswift/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'SWIFT_STORAGE_URL') >= 0
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiswift_streaming/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'SWIFT_STORAGE_URL') >= 0
gdal.SetConfigOption('SWIFT_STORAGE_URL', 'http://0.0.0.0')
# Missing SWIFT_AUTH_TOKEN
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiswift/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'SWIFT_AUTH_TOKEN') >= 0
gdal.SetConfigOption('SWIFT_AUTH_TOKEN', 'SWIFT_AUTH_TOKEN')
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiswift/foo/bar.baz')
if f is not None:
if f is not None:
gdal.VSIFCloseL(f)
pytest.fail(gdal.VSIGetLastErrorMsg())
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiswift_streaming/foo/bar.baz')
assert f is None, gdal.VSIGetLastErrorMsg()
def test_visoss_real_test():
if not gdaltest.built_against_curl():
pytest.skip()
if gdaltest.skip_on_travis():
pytest.skip()
# ERROR 1: The OSS Access Key Id you provided does not exist in our records.
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsioss/foo/bar.baz')
if f is not None or gdal.VSIGetLastErrorMsg() == '':
if f is not None:
gdal.VSIFCloseL(f)
if gdal.GetConfigOption('APPVEYOR') is not None:
return
pytest.fail(gdal.VSIGetLastErrorMsg())
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsioss_streaming/foo/bar.baz')
assert f is None and gdal.VSIGetLastErrorMsg() != ''
def test_vsifile_22():
# VSIOpenL doesn't set errorno
gdal.VSIErrorReset()
assert gdal.VSIGetLastErrorNo() == 0, \
("Expected Err=0 after VSIErrorReset(), got %d" % gdal.VSIGetLastErrorNo())
fp = gdal.VSIFOpenL('tmp/not-existing', 'r')
assert fp is None, "Expected None from VSIFOpenL"
assert gdal.VSIGetLastErrorNo() == 0, \
("Expected Err=0 from VSIFOpenL, got %d" % gdal.VSIGetLastErrorNo())
# VSIOpenExL does
fp = gdal.VSIFOpenExL('tmp/not-existing', 'r', 1)
assert fp is None, "Expected None from VSIFOpenExL"
assert gdal.VSIGetLastErrorNo() == 1, \
("Expected Err=1 from VSIFOpenExL, got %d" % gdal.VSIGetLastErrorNo())
assert len(gdal.VSIGetLastErrorMsg()) != 0, "Expected a VSI error message"
gdal.VSIErrorReset()
assert gdal.VSIGetLastErrorNo() == 0, \
("Expected Err=0 after VSIErrorReset(), got %d" % gdal.VSIGetLastErrorNo())
def vsigs_1():
try:
drv = gdal.GetDriverByName( 'HTTP' )
except:
drv = None
if drv is None:
return 'skip'
# RETODO: Bind to swig, change test
# Invalid header filename
gdal.ErrorReset()
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', '/i_dont/exist.py')
f = open_for_read('/vsigs/foo/bar')
if f is None:
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None)
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
data = gdal.VSIFReadL(1, 1, f)
last_err = gdal.GetLastErrorMsg()
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None)
gdal.VSIFCloseL(f)
if len(data) != 0:
gdaltest.post_reason('fail')
return 'fail'
if last_err.find('Cannot read') < 0:
gdaltest.post_reason('fail')
print(last_err)
return 'fail'
# Invalid content for header file
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', 'vsigs.py')
f = open_for_read('/vsigs/foo/bar')
if f is None:
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None)
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 1, f)
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None)
gdal.VSIFCloseL(f)
if len(data) != 0:
gdaltest.post_reason('fail')
return 'fail'
# Missing GS_SECRET_ACCESS_KEY
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find('GS_SECRET_ACCESS_KEY') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs_streaming/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find('GS_SECRET_ACCESS_KEY') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY')
# Missing GS_ACCESS_KEY_ID
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find('GS_ACCESS_KEY_ID') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID')
# ERROR 1: The User Id you provided does not exist in our records.
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar.baz')
if f is not None or gdal.VSIGetLastErrorMsg() == '':
if f is not None:
gdal.VSIFCloseL(f)
if gdal.GetConfigOption('APPVEYOR') is not None:
return 'success'
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs_streaming/foo/bar.baz')
if f is not None or gdal.VSIGetLastErrorMsg() == '':
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def test_vsigs_1():
if not gdaltest.built_against_curl():
pytest.skip()
# Invalid header filename
gdal.ErrorReset()
with gdaltest.config_option('GDAL_HTTP_HEADER_FILE', '/i_dont/exist.py'):
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
if f is not None:
gdal.VSIFCloseL(f)
pytest.fail()
last_err = gdal.GetLastErrorMsg()
assert 'Cannot read' in last_err
# Invalid content for header file
with gdaltest.config_option('GDAL_HTTP_HEADER_FILE', 'vsigs.py'):
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
f = open_for_read('/vsigs/foo/bar')
if f is not None:
gdal.VSIFCloseL(f)
pytest.fail()
# Missing GS_SECRET_ACCESS_KEY
gdal.ErrorReset()
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find('GS_SECRET_ACCESS_KEY') >= 0
gdal.ErrorReset()
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
with gdaltest.error_handler():
f = open_for_read('/vsigs_streaming/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find('GS_SECRET_ACCESS_KEY') >= 0
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY')
# Missing GS_ACCESS_KEY_ID
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find('GS_ACCESS_KEY_ID') >= 0
gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID')
# ERROR 1: The User Id you provided does not exist in our records.
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar.baz')
if f is not None or gdal.VSIGetLastErrorMsg() == '':
if f is not None:
gdal.VSIFCloseL(f)
if gdal.GetConfigOption('APPVEYOR') is not None:
return
pytest.fail(gdal.VSIGetLastErrorMsg())
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs_streaming/foo/bar.baz')
assert f is None and gdal.VSIGetLastErrorMsg() != ''
def vsis3_1():
try:
drv = gdal.GetDriverByName( 'HTTP' )
except:
drv = None
if drv is None:
return 'skip'
# RETODO: Bind to swig, change test
# Missing AWS_SECRET_ACCESS_KEY
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find('AWS_SECRET_ACCESS_KEY') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find('AWS_SECRET_ACCESS_KEY') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.SetConfigOption('AWS_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY')
# Missing AWS_ACCESS_KEY_ID
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find('AWS_ACCESS_KEY_ID') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.SetConfigOption('AWS_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID')
# ERROR 1: The AWS Access Key Id you provided does not exist in our records.
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3/foo/bar.baz')
if f is not None or gdal.VSIGetLastErrorMsg() == '':
if f is not None:
gdal.VSIFCloseL(f)
if gdal.GetConfigOption('APPVEYOR') is not None:
return 'success'
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/foo/bar.baz')
if f is not None or gdal.VSIGetLastErrorMsg() == '':
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def vsis3_extra_1():
try:
drv = gdal.GetDriverByName( 'HTTP' )
except:
drv = None
if drv is None:
return 'skip'
if gdal.GetConfigOption('AWS_SECRET_ACCESS_KEY') is None:
print('Missing AWS_SECRET_ACCESS_KEY for running gdaltest_list_extra')
return 'skip'
elif gdal.GetConfigOption('AWS_ACCESS_KEY_ID') is None:
print('Missing AWS_ACCESS_KEY_ID for running gdaltest_list_extra')
return 'skip'
elif gdal.GetConfigOption('S3_RESOURCE') is None:
print('Missing S3_RESOURCE for running gdaltest_list_extra')
return 'skip'
f = open_for_read('/vsis3/' + gdal.GetConfigOption('S3_RESOURCE'))
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Same with /vsis3_streaming/
f = open_for_read('/vsis3_streaming/' + gdal.GetConfigOption('S3_RESOURCE'))
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsis3/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if gdal.VSIGetLastErrorMsg() == '':
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsis3_streaming/' + gdal.GetConfigOption('S3_RESOURCE') + '/invalid_resource.baz')
if f is not None:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def vsis3_2():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('AWS_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY')
gdal.SetConfigOption('AWS_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID')
gdal.SetConfigOption('AWS_TIMESTAMP', '20150101T000000Z')
gdal.SetConfigOption('AWS_HTTPS', 'NO')
gdal.SetConfigOption('AWS_VIRTUAL_HOSTING', 'NO')
gdal.SetConfigOption('AWS_S3_ENDPOINT', '127.0.0.1:%d' % gdaltest.webserver_port)
f = open_for_read('/vsis3/s3_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
f = open_for_read('/vsis3_streaming/s3_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
# Test with temporary credentials
gdal.SetConfigOption('AWS_SESSION_TOKEN', 'AWS_SESSION_TOKEN')
f = open_for_read('/vsis3/s3_fake_bucket_with_session_token/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
gdal.SetConfigOption('AWS_SESSION_TOKEN', None)
#old_val = gdal.GetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN')
#gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR')
stat_res = gdal.VSIStatL('/vsis3/s3_fake_bucket/resource2.bin')
#gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', old_val)
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
stat_res = gdal.VSIStatL('/vsis3_streaming/s3_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
# Test region and endpoint 'redirects'
f = open_for_read('/vsis3/s3_fake_bucket/redirect')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
if 'TRAVIS_BRANCH' in os.environ and os.environ['TRAVIS_BRANCH'].find('trusty') >= 0:
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
print(data)
return 'fail'
# Test region and endpoint 'redirects'
f = open_for_read('/vsis3_streaming/s3_fake_bucket/redirect')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/non_xml_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('bla') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/invalid_xml_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<oops>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_code_in_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error/>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_endpoint_in_PermanentRedirect_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_message_in_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def vsiaz_extra_1():
if not gdaltest.built_against_curl():
return 'skip'
az_resource = gdal.GetConfigOption('/azure/blob/myaccount/az_RESOURCE')
if az_resource is None:
print('Missing AZ_RESOURCE for running gdaltest_list_extra')
return 'skip'
if az_resource.find('/') < 0:
path = '/vsiaz/' + az_resource
statres = gdal.VSIStatL(path)
if statres is None or not stat.S_ISDIR(statres.mode):
gdaltest.post_reason('fail')
print('%s is not a valid bucket' % path)
return 'fail'
readdir = gdal.ReadDir(path)
if readdir is None:
gdaltest.post_reason('fail')
print('ReadDir() should not return empty list')
return 'fail'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
if gdal.VSIStatL(subpath) is None:
gdaltest.post_reason('fail')
print('Stat(%s) should not return an error' % subpath)
return 'fail'
unique_id = 'vsiaz_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
if ret < 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) should not return an error' % subpath)
return 'fail'
readdir = gdal.ReadDir(path)
if unique_id not in readdir:
gdaltest.post_reason('fail')
print('ReadDir(%s) should contain %s' % (path, unique_id))
print(readdir)
return 'fail'
ret = gdal.Mkdir(subpath, 0)
if ret == 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) repeated should return an error' % subpath)
return 'fail'
ret = gdal.Rmdir(subpath)
if ret < 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) should not return an error' % subpath)
return 'fail'
readdir = gdal.ReadDir(path)
if unique_id in readdir:
gdaltest.post_reason('fail')
print('ReadDir(%s) should not contain %s' % (path, unique_id))
print(readdir)
return 'fail'
ret = gdal.Rmdir(subpath)
if ret == 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) repeated should return an error' % subpath)
return 'fail'
ret = gdal.Mkdir(subpath, 0)
if ret < 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) should not return an error' % subpath)
return 'fail'
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
if ret == 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) on non empty directory should return an error' %
subpath)
return 'fail'
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
if data != 'hello':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test.txt')
if ret < 0:
gdaltest.post_reason('fail')
print('Unlink(%s) should not return an error' %
(subpath + '/test.txt'))
return 'fail'
ret = gdal.Rmdir(subpath)
if ret < 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) should not return an error' % subpath)
return 'fail'
return 'success'
f = open_for_read('/vsiaz/' + az_resource)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Same with /vsiaz_streaming/
f = open_for_read('/vsiaz_streaming/' + az_resource)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsiaz/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if gdal.VSIGetLastErrorMsg() == '':
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsiaz_streaming/' + az_resource +
'/invalid_resource.baz')
if f is not None:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def vsiaz_real_server_errors():
if not gdaltest.built_against_curl():
return 'skip'
# Missing AZURE_STORAGE_ACCOUNT
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiaz/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find(
'AZURE_STORAGE_ACCOUNT') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiaz_streaming/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find(
'AZURE_STORAGE_ACCOUNT') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Invalid AZURE_STORAGE_CONNECTION_STRING
gdal.SetConfigOption('AZURE_STORAGE_CONNECTION_STRING', 'invalid')
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiaz/foo/bar')
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
gdal.SetConfigOption('AZURE_STORAGE_CONNECTION_STRING', '')
gdal.SetConfigOption('AZURE_STORAGE_ACCOUNT', 'AZURE_STORAGE_ACCOUNT')
# Missing AZURE_STORAGE_ACCESS_KEY
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiaz/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find(
'AZURE_STORAGE_ACCESS_KEY') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.SetConfigOption('AZURE_STORAGE_ACCESS_KEY',
'AZURE_STORAGE_ACCESS_KEY')
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiaz/foo/bar.baz')
if f is not None:
if f is not None:
gdal.VSIFCloseL(f)
if gdal.GetConfigOption('APPVEYOR') is not None:
return 'success'
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsiaz_streaming/foo/bar.baz')
if f is not None:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def test_vsiaz_extra_1():
if not gdaltest.built_against_curl():
pytest.skip()
az_resource = gdal.GetConfigOption('AZ_RESOURCE')
if az_resource is None:
pytest.skip('Missing AZ_RESOURCE')
if '/' not in az_resource:
path = '/vsiaz/' + az_resource
statres = gdal.VSIStatL(path)
assert statres is not None and stat.S_ISDIR(statres.mode), \
('%s is not a valid bucket' % path)
readdir = gdal.ReadDir(path)
assert readdir is not None, 'ReadDir() should not return empty list'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
assert gdal.VSIStatL(subpath) is not None, \
('Stat(%s) should not return an error' % subpath)
unique_id = 'vsiaz_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id in readdir, \
('ReadDir(%s) should contain %s' % (path, unique_id))
ret = gdal.Mkdir(subpath, 0)
assert ret != 0, ('Mkdir(%s) repeated should return an error' %
subpath)
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id not in readdir, \
('ReadDir(%s) should not contain %s' % (path, unique_id))
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
assert f is not None
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
assert ret != 0, \
('Rmdir(%s) on non empty directory should return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
assert f is not None
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
assert data == 'hello'
gdal.VSIFCloseL(f)
md = gdal.GetFileMetadata(subpath + '/test.txt', 'HEADERS')
assert 'x-ms-blob-type' in md
md = gdal.GetFileMetadata(subpath + '/test.txt', 'METADATA')
assert 'ETag' in md
assert 'x-ms-blob-type' not in md
md = gdal.GetFileMetadata(subpath + '/test.txt', 'TAGS')
assert md == {}
# Change properties
assert gdal.SetFileMetadata(subpath + '/test.txt',
{'x-ms-blob-content-type': 'foo'},
'PROPERTIES')
md = gdal.GetFileMetadata(subpath + '/test.txt', 'HEADERS')
assert md['Content-Type'] == 'foo'
# Change metadata
assert gdal.SetFileMetadata(subpath + '/test.txt',
{'x-ms-meta-FOO': 'BAR'}, 'METADATA')
md = gdal.GetFileMetadata(subpath + '/test.txt', 'METADATA')
assert md['x-ms-meta-FOO'] == 'BAR'
# Change tags
assert gdal.SetFileMetadata(subpath + '/test.txt', {'BAR': 'BAZ'},
'TAGS')
md = gdal.GetFileMetadata(subpath + '/test.txt', 'TAGS')
assert md['BAR'] == 'BAZ'
assert gdal.Rename(subpath + '/test.txt', subpath + '/test2.txt') == 0
f = gdal.VSIFOpenL(subpath + '/test2.txt', 'rb')
assert f is not None
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
assert data == 'hello'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test2.txt')
assert ret >= 0, \
('Unlink(%s) should not return an error' % (subpath + '/test2.txt'))
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
return
f = open_for_read('/vsiaz/' + az_resource)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
# Same with /vsiaz_streaming/
f = open_for_read('/vsiaz_streaming/' + az_resource)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
if False: # pylint: disable=using-constant-test
# we actually try to read at read() time and bSetError = false
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsiaz/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert gdal.VSIGetLastErrorMsg() != ''
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsiaz_streaming/' + az_resource +
'/invalid_resource.baz')
assert f is None, gdal.VSIGetLastErrorMsg()
# Test GetSignedURL()
signed_url = gdal.GetSignedURL('/vsiaz/' + az_resource)
f = open_for_read('/vsicurl_streaming/' + signed_url)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
def _check_error(self):
if gdal.VSIGetLastErrorNo() != 0:
raise SimpleVSIMemFileError(gdal.VSIGetLastErrorMsg())
def test_vsiswift_extra_1():
if not gdaltest.built_against_curl():
pytest.skip()
swift_resource = gdal.GetConfigOption('SWIFT_RESOURCE')
if swift_resource is None:
pytest.skip('Missing SWIFT_RESOURCE')
if '/' not in swift_resource:
path = '/vsiswift/' + swift_resource
statres = gdal.VSIStatL(path)
assert statres is not None and stat.S_ISDIR(statres.mode), \
('%s is not a valid bucket' % path)
readdir = gdal.ReadDir(path)
assert readdir is not None, 'ReadDir() should not return empty list'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
assert gdal.VSIStatL(subpath) is not None, \
('Stat(%s) should not return an error' % subpath)
unique_id = 'vsiswift_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id in readdir, \
('ReadDir(%s) should contain %s' % (path, unique_id))
ret = gdal.Mkdir(subpath, 0)
assert ret != 0, ('Mkdir(%s) repeated should return an error' %
subpath)
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id not in readdir, \
('ReadDir(%s) should not contain %s' % (path, unique_id))
ret = gdal.Rmdir(subpath)
assert ret != 0, ('Rmdir(%s) repeated should return an error' %
subpath)
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
assert f is not None
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
assert ret != 0, \
('Rmdir(%s) on non empty directory should return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
assert f is not None
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
assert data == 'hello'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test.txt')
assert ret >= 0, \
('Unlink(%s) should not return an error' % (subpath + '/test.txt'))
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
return
f = open_for_read('/vsiswift/' + swift_resource)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
# Same with /vsiswift_streaming/
f = open_for_read('/vsiswift_streaming/' + swift_resource)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsiswift_streaming/' + swift_resource +
'/invalid_resource.baz')
assert f is None, gdal.VSIGetLastErrorMsg()
def test_visoss_2():
if gdaltest.webserver_port == 0:
pytest.skip()
signed_url = gdal.GetSignedURL('/vsioss/oss_fake_bucket/resource',
['START_DATE=20180212T123456Z'])
assert (signed_url in (
'http://127.0.0.1:8080/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D',
'http://127.0.0.1:8081/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D'
))
handler = webserver.SequentialHandler()
handler.add('GET',
'/oss_fake_bucket/resource',
custom_method=get_oss_fake_bucket_resource_method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
handler.add('GET',
'/oss_fake_bucket/resource',
custom_method=get_oss_fake_bucket_resource_method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss_streaming/oss_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(400)
response = """<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>AccessDenied</Code>
<Message>The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.</Message>
<HostId>unused</HostId>
<Bucket>unuset</Bucket>
<Endpoint>localhost:%d</Endpoint>
</Error>""" % request.server.port
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
request.send_header('Content-type', 'application/xml')
request.send_header('Transfer-Encoding', 'chunked')
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method)
def method(request):
request.protocol_version = 'HTTP/1.1'
if request.headers['Host'].startswith('localhost'):
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
else:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method)
# Test region and endpoint 'redirects'
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_fake_bucket/redirect')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
pytest.fail(data)
# Test region and endpoint 'redirects'
handler.req_count = 0
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss_streaming/oss_fake_bucket/redirect')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
def method(request):
# /vsioss_streaming/ should have remembered the change of region and endpoint
if not request.headers['Host'].startswith('localhost'):
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.protocol_version = 'HTTP/1.1'
request.send_response(400)
response = 'bla'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
request.send_header('Content-type', 'application/xml')
request.send_header('Transfer-Encoding', 'chunked')
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET', '/oss_fake_bucket/non_xml_error', custom_method=method)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/non_xml_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('bla') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><oops>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/invalid_xml_error', 400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/invalid_xml_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<oops>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error/>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/no_code_in_error', 400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_code_in_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error/>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>AuthorizationHeaderMalformed</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET',
'/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error',
400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error'
)
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>PermanentRedirect</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error', 400,
{
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error'
)
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>bla</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/no_message_in_error', 400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_message_in_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
def test_visoss_extra_1():
if not gdaltest.built_against_curl():
pytest.skip()
# Either a bucket name or bucket/filename
OSS_RESOURCE = gdal.GetConfigOption('OSS_RESOURCE')
if gdal.GetConfigOption('OSS_SECRET_ACCESS_KEY') is None:
pytest.skip('Missing OSS_SECRET_ACCESS_KEY')
elif gdal.GetConfigOption('OSS_ACCESS_KEY_ID') is None:
pytest.skip('Missing OSS_ACCESS_KEY_ID')
elif OSS_RESOURCE is None:
pytest.skip('Missing OSS_RESOURCE')
if '/' not in OSS_RESOURCE:
path = '/vsioss/' + OSS_RESOURCE
statres = gdal.VSIStatL(path)
assert statres is not None and stat.S_ISDIR(statres.mode), \
('%s is not a valid bucket' % path)
readdir = gdal.ReadDir(path)
assert readdir is not None, 'ReadDir() should not return empty list'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
assert gdal.VSIStatL(subpath) is not None, \
('Stat(%s) should not return an error' % subpath)
unique_id = 'visoss_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id in readdir, \
('ReadDir(%s) should contain %s' % (path, unique_id))
ret = gdal.Mkdir(subpath, 0)
assert ret != 0, ('Mkdir(%s) repeated should return an error' %
subpath)
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id not in readdir, \
('ReadDir(%s) should not contain %s' % (path, unique_id))
ret = gdal.Rmdir(subpath)
assert ret != 0, ('Rmdir(%s) repeated should return an error' %
subpath)
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
assert f is not None
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
assert ret != 0, \
('Rmdir(%s) on non empty directory should return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
assert f is not None
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
assert data == 'hello'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test.txt')
assert ret >= 0, \
('Unlink(%s) should not return an error' % (subpath + '/test.txt'))
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
return
f = open_for_read('/vsioss/' + OSS_RESOURCE)
assert f is not None, ('cannot open %s' % ('/vsioss/' + OSS_RESOURCE))
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
# Same with /vsioss_streaming/
f = open_for_read('/vsioss_streaming/' + OSS_RESOURCE)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
if False: # pylint: disable=using-constant-test
# we actually try to read at read() time and bSetError = false:
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsioss/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert gdal.VSIGetLastErrorMsg() != ''
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsioss_streaming/' + OSS_RESOURCE +
'/invalid_resource.baz')
assert f is None, gdal.VSIGetLastErrorMsg()
# Test GetSignedURL()
signed_url = gdal.GetSignedURL('/vsioss/' + OSS_RESOURCE)
f = open_for_read('/vsicurl_streaming/' + signed_url)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
def validate(args):
if 'url' not in args:
return json.dumps({'status': 'failure', 'error': 'url missing'}), 400, \
{ "Content-Type": "application/json" }
remove_tmpfile = False
url = args.get('url')
if 'local_filename' in args:
ds = gdal.OpenEx(args['local_filename'], allowed_drivers=['GTiff'])
else:
use_vsicurl = args.get('use_vsicurl', 'true')
if use_vsicurl.lower() not in ('true', 'false'):
return json.dumps({
'status':
'failure',
'error':
'invalid value for use_vsicurl option. Expected true or false'
}), 400, {
"Content-Type": "application/json"
}
use_vsicurl = use_vsicurl.lower() == 'true'
gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR')
if use_vsicurl:
ds = gdal.OpenEx('/vsicurl/' + url, allowed_drivers=['GTiff'])
if ds is None:
f = gdal.VSIFOpenL('/vsicurl/' + url, 'rb')
if f is None:
return json.dumps({
'status': 'failure',
'error': 'Cannot download %s' % url
}), 400, {
"Content-Type": "application/json"
}
data = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(data) == 0:
error_msg = 'Cannot download %s' % url
gdal_error_msg = gdal.GetLastErrorMsg()
if gdal_error_msg == '':
gdal_error_msg = gdal.VSIGetLastErrorMsg()
if gdal_error_msg != '':
error_msg += ': ' + gdal_error_msg
return json.dumps({
'status': 'failure',
'error': error_msg
}), 400, {
"Content-Type": "application/json"
}
else:
try:
r = requests.get(url)
except Exception, e:
return json.dumps({
'status': 'failure',
'error': 'Cannot download %s' % url
}), 400, {
"Content-Type": "application/json"
}
remove_tmpfile = True
f = open(tmpfilename, 'wb')
f.write(r.content)
f.close()
ds = gdal.OpenEx(tmpfilename, allowed_drivers=['GTiff'])
def vsigs_1():
if not gdaltest.built_against_curl():
return 'skip'
# Invalid header filename
gdal.ErrorReset()
with gdaltest.config_option('GDAL_HTTP_HEADER_FILE', '/i_dont/exist.py'):
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
if f is not None:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
last_err = gdal.GetLastErrorMsg()
if last_err.find('Cannot read') < 0:
gdaltest.post_reason('fail')
print(last_err)
return 'fail'
# Invalid content for header file
with gdaltest.config_option('GDAL_HTTP_HEADER_FILE', 'vsigs.py'):
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
f = open_for_read('/vsigs/foo/bar')
if f is not None:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
# Missing GS_SECRET_ACCESS_KEY
gdal.ErrorReset()
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find(
'GS_SECRET_ACCESS_KEY') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
with gdaltest.error_handler():
f = open_for_read('/vsigs_streaming/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find(
'GS_SECRET_ACCESS_KEY') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY')
# Missing GS_ACCESS_KEY_ID
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find('GS_ACCESS_KEY_ID') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID')
# ERROR 1: The User Id you provided does not exist in our records.
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar.baz')
if f is not None or gdal.VSIGetLastErrorMsg() == '':
if f is not None:
gdal.VSIFCloseL(f)
if gdal.GetConfigOption('APPVEYOR') is not None:
return 'success'
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs_streaming/foo/bar.baz')
if f is not None or gdal.VSIGetLastErrorMsg() == '':
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
