def check_no_file_leaks():
num_files = len(gdaltest.get_opened_files())
yield
diff = len(gdaltest.get_opened_files()) - num_files
if diff != 0 and (gdaltest.is_travis_branch('ubuntu_1804') or gdaltest.is_travis_branch('ubuntu_1804_32bit') or gdaltest.is_travis_branch('fedora')):
print('Mysterious leak of file handle on some CI setups')
return
assert diff == 0, 'Leak of file handles: %d leaked' % diff
def vsigs_3():
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add('GET', '/gs_fake_bucket2?delimiter=/&prefix=a_dir/', 200,
{ 'Content-type': 'application/xml' },
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Contents>
<Key>a_dir/resource3.bin</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>123456</Size>
</Contents>
</ListBucketResult>
""")
handler.add('GET', '/gs_fake_bucket2?delimiter=/&marker=bla&prefix=a_dir/', 200,
{ 'Content-type': 'application/xml' },
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<Contents>
<Key>a_dir/resource4.bin</Key>
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
</Contents>
<CommonPrefixes>
<Prefix>a_dir/subdir/</Prefix>
</CommonPrefixes>
</ListBucketResult>
""")
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def runexternal_out_and_err(cmd, check_memleak=True):
from gdaltest import is_travis_branch
if not is_travis_branch('mingw'):
has_subprocess = False
try:
import subprocess
import shlex
if hasattr(subprocess, 'Popen') and hasattr(shlex, 'split'):
has_subprocess = True
except (ImportError, AttributeError):
pass
if has_subprocess:
return _runexternal_out_and_err_subprocess(cmd, check_memleak=check_memleak)
(ret_stdin, ret_stdout, ret_stderr) = os.popen3(cmd)
ret_stdin.close()
q_stdout = Queue()
t_stdout = Thread(target=read_in_thread, args=(ret_stdout, q_stdout))
q_stderr = Queue()
t_stderr = Thread(target=read_in_thread, args=(ret_stderr, q_stderr))
t_stdout.start()
t_stderr.start()
out_str = q_stdout.get()
err_str = q_stderr.get()
if check_memleak:
warn_if_memleak(cmd, out_str)
return (out_str, err_str)
def postgisraster_test_outdb():
if gdaltest.postgisrasterDriver is None:
return 'skip'
# For some reason fails with
# ERROR 1: PostGISRasterRasterBand::IRasterIO(): ERROR: rt_band_load_offline_data: Cannot open offline raster: /home/travis/build/rouault/gdal/autotest/gdrivers/data/small_world.tif
# See https://api.travis-ci.org/v3/job/428972866/log.txt
if gdaltest.is_travis_branch('ubuntu_1804'):
return 'skip'
ds = ogr.Open(gdaltest.postgisraster_connection_string_without_schema)
sql_lyr = ds.ExecuteSQL('SHOW postgis.enable_outdb_rasters')
has_guc = sql_lyr is not None
ds.ReleaseResultSet(sql_lyr)
ds = None
ds = gdal.Open(gdaltest.postgisraster_connection_string + "table='small_world_outdb_constraint' mode=2")
cs = [ds.GetRasterBand(i + 1).Checksum() for i in range(3)]
expected_cs = [30111, 32302, 40026] if has_guc else [0, 0, 0]
if cs != expected_cs:
gdaltest.post_reason('fail')
print(cs)
return 'fail'
return 'success'
def runexternal_out_and_err(cmd, check_memleak=True):
from gdaltest import is_travis_branch
if not is_travis_branch('mingw'):
has_subprocess = False
try:
import subprocess
import shlex
if hasattr(subprocess, 'Popen') and hasattr(shlex, 'split'):
has_subprocess = True
except ImportError:
pass
if has_subprocess:
return _runexternal_out_and_err_subprocess(cmd, check_memleak=check_memleak)
(ret_stdin, ret_stdout, ret_stderr) = os.popen3(cmd)
ret_stdin.close()
q_stdout = Queue()
t_stdout = Thread(target=read_in_thread, args=(ret_stdout, q_stdout))
q_stderr = Queue()
t_stderr = Thread(target=read_in_thread, args=(ret_stderr, q_stderr))
t_stdout.start()
t_stderr.start()
out_str = q_stdout.get()
err_str = q_stderr.get()
if check_memleak:
warn_if_memleak(cmd, out_str)
return (out_str, err_str)
def postgisraster_test_outdb():
if gdaltest.postgisrasterDriver is None:
return 'skip'
# For some reason fails with
# ERROR 1: PostGISRasterRasterBand::IRasterIO(): ERROR: rt_band_load_offline_data: Cannot open offline raster: /home/travis/build/rouault/gdal/autotest/gdrivers/data/small_world.tif
# See https://api.travis-ci.org/v3/job/428972866/log.txt
if gdaltest.is_travis_branch('ubuntu_1804'):
return 'skip'
ds = ogr.Open(gdaltest.postgisraster_connection_string_without_schema)
sql_lyr = ds.ExecuteSQL('SHOW postgis.enable_outdb_rasters')
has_guc = sql_lyr is not None
ds.ReleaseResultSet(sql_lyr)
ds = None
ds = gdal.Open(gdaltest.postgisraster_connection_string +
"table='small_world_outdb_constraint' mode=2")
cs = [ds.GetRasterBand(i + 1).Checksum() for i in range(3)]
expected_cs = [30111, 32302, 40026] if has_guc else [0, 0, 0]
if cs != expected_cs:
gdaltest.post_reason('fail')
print(cs)
return 'fail'
return 'success'
def vsigs_3():
if gdaltest.webserver_port == 0:
return 'skip'
f = open_for_read('/vsigs/gs_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def test_ogr_ogdi_1():
ogrtest.ogdi_ds = None
# Skip tests when -fsanitize is used because of memleaks in libogdi
if gdaltest.is_travis_branch('sanitize'):
ogrtest.ogdi_drv = None
pytest.skip('Skipping because of memory leaks in OGDI')
ogrtest.ogdi_drv = ogr.GetDriverByName('OGDI')
if ogrtest.ogdi_drv is None:
pytest.skip()
if not gdaltest.download_file(
'http://freefr.dl.sourceforge.net/project/ogdi/OGDI_Test_Suite/3.1/ogdits-3.1.0.zip',
'ogdits-3.1.0.zip'):
pytest.skip()
try:
os.stat('tmp/cache/ogdits-3.1')
except OSError:
try:
gdaltest.unzip('tmp/cache', 'tmp/cache/ogdits-3.1.0.zip')
try:
os.stat('tmp/cache/ogdits-3.1')
except OSError:
pytest.skip()
except OSError:
pytest.skip()
url_name = 'gltp:/vrf/' + os.getcwd(
) + '/tmp/cache/ogdits-3.1/data/vpf/vm2alv2/texash'
ds = ogr.Open(url_name)
ogrtest.ogdi_ds = ds
assert ds is not None, ('cannot open ' + url_name)
assert ds.GetLayerCount() == 57, 'did not get expected layer count'
layers = [('libref@libref(*)_line', ogr.wkbLineString, 15),
('libreft@libref(*)_text', ogr.wkbPoint, 4),
('markersp@bnd(*)_point', ogr.wkbPoint, 40),
('polbnda@bnd(*)_area', ogr.wkbPolygon, 6)]
for l in layers:
lyr = ds.GetLayerByName(l[0])
assert lyr.GetLayerDefn().GetGeomType() == l[1]
assert lyr.GetFeatureCount() == l[2]
# if l[1] != ogr.wkbNone:
# if lyr.GetSpatialRef().ExportToWkt().find('WGS 84') == -1:
# return 'fail'
lyr = ds.GetLayerByName('libref@libref(*)_line')
feat = lyr.GetNextFeature()
wkt = 'LINESTRING (-97.570159912109375 31.242000579833984,-97.569938659667969 31.242116928100586,-97.562828063964844 31.245765686035156,-97.558868408203125 31.247797012329102,-97.555778503417969 31.249361038208008,-97.55413818359375 31.250171661376953)'
ref_geom = ogr.CreateGeometryFromWkt(wkt)
assert ogrtest.check_feature_geometry(feat, ref_geom) == 0, \
feat.GetGeometryRef().ExportToWkt()
def eedai_GOOGLE_APPLICATION_CREDENTIALS():
if gdaltest.eedai_drv is None:
return 'skip'
if gdaltest.is_travis_branch('gcc52_stdcpp14_sanitize'):
return 'skip'
gdal.FileFromMemBuffer(
'/vsimem/my.json', """{
"private_key":"-----BEGIN PRIVATE KEY-----
MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk\n
VNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm\n
i2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ\n
DVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy\n
TabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I\n
2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd\n
DrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+\n
bpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg\n
Wz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3\n
4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5\n
Kvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj\n
j+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI\n
yzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq\n
gwE6fxOLyJDxuWRf\n
-----END PRIVATE KEY-----",
"client_email":"*****@*****.**"
}""")
gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/')
gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', '/vsimem/my.json')
gdal.SetConfigOption('EEDA_PRIVATE_KEY', None)
gdal.SetConfigOption('EEDA_CLIENT_EMAIL', None)
gdal.SetConfigOption('GO2A_AUD', '/vsimem/oauth2/v4/token')
gdal.SetConfigOption('GOA2_NOW', '123456')
gdal.FileFromMemBuffer(
'/vsimem/oauth2/v4/token&POSTFIELDS=grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAibXlAZW1haWwuY29tIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZWFydGhlbmdpbmUucmVhZG9ubHkiLCAiYXVkIjogIi92c2ltZW0vb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.1W564xcQESVsqZmBEIMzj4rr0RuGa4RiUPZp5H%2FNENN9V9oPSTdacw%2BMiu3pcFf9AJv8wj0ajUeRsgTmvSicAftER49xeCQYUrs6uV122FGVsxml26kMFacNsCgRad%2Fy7xCAhMPfRJsqxS2%2BB392ssBeEzTGCSI6W3AsJg64OfA%3D',
'{ "access_token": "my_token", "token_type": "Bearer", "expires_in": 3600 }'
)
ds = gdal.Open('EEDAI:image')
gdal.Unlink('/vsimem/my.json')
gdal.SetConfigOption('EEDA_URL', None)
gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', None)
gdal.SetConfigOption('EEDA_PRIVATE_KEY', None)
gdal.SetConfigOption('EEDA_CLIENT_EMAIL', None)
if gdal.GetLastErrorMsg().find('CPLRSASHA256Sign() not implemented') >= 0:
return 'skip'
if ds is None:
return 'fail'
return 'success'
def test_ogr_ogdi_1():
ogrtest.ogdi_ds = None
# Skip tests when -fsanitize is used because of memleaks in libogdi
if gdaltest.is_travis_branch('sanitize'):
ogrtest.ogdi_drv = None
pytest.skip('Skipping because of memory leaks in OGDI')
ogrtest.ogdi_drv = ogr.GetDriverByName('OGDI')
if ogrtest.ogdi_drv is None:
pytest.skip()
if not gdaltest.download_file('http://freefr.dl.sourceforge.net/project/ogdi/OGDI_Test_Suite/3.1/ogdits-3.1.0.zip', 'ogdits-3.1.0.zip'):
pytest.skip()
try:
os.stat('tmp/cache/ogdits-3.1')
except OSError:
try:
gdaltest.unzip('tmp/cache', 'tmp/cache/ogdits-3.1.0.zip')
try:
os.stat('tmp/cache/ogdits-3.1')
except OSError:
pytest.skip()
except OSError:
pytest.skip()
url_name = 'gltp:/vrf/' + os.getcwd() + '/tmp/cache/ogdits-3.1/data/vpf/vm2alv2/texash'
ds = ogr.Open(url_name)
ogrtest.ogdi_ds = ds
assert ds is not None, ('cannot open ' + url_name)
assert ds.GetLayerCount() == 57, 'did not get expected layer count'
layers = [('libref@libref(*)_line', ogr.wkbLineString, 15),
('libreft@libref(*)_text', ogr.wkbPoint, 4),
('markersp@bnd(*)_point', ogr.wkbPoint, 40),
('polbnda@bnd(*)_area', ogr.wkbPolygon, 6)]
for l in layers:
lyr = ds.GetLayerByName(l[0])
assert lyr.GetLayerDefn().GetGeomType() == l[1]
assert lyr.GetFeatureCount() == l[2]
# if l[1] != ogr.wkbNone:
# if lyr.GetSpatialRef().ExportToWkt().find('WGS 84') == -1:
# return 'fail'
lyr = ds.GetLayerByName('libref@libref(*)_line')
feat = lyr.GetNextFeature()
wkt = 'LINESTRING (-97.570159912109375 31.242000579833984,-97.569938659667969 31.242116928100586,-97.562828063964844 31.245765686035156,-97.558868408203125 31.247797012329102,-97.555778503417969 31.249361038208008,-97.55413818359375 31.250171661376953)'
ref_geom = ogr.CreateGeometryFromWkt(wkt)
assert ogrtest.check_feature_geometry(feat, ref_geom) == 0, \
feat.GetGeometryRef().ExportToWkt()
def test_vrt_shared_no_proxy_pool():
before = gdaltest.get_opened_files()
vrt_text = """<VRTDataset rasterXSize="50" rasterYSize="50">
<VRTRasterBand dataType="Byte" band="1">
<ColorInterp>Red</ColorInterp>
<SimpleSource>
<SourceFilename>data/rgbsmall.tif</SourceFilename>
<SourceBand>1</SourceBand>
</SimpleSource>
</VRTRasterBand>
<VRTRasterBand dataType="Byte" band="2">
<ColorInterp>Green</ColorInterp>
<SimpleSource>
<SourceFilename>data/rgbsmall.tif</SourceFilename>
<SourceBand>2</SourceBand>
</SimpleSource>
</VRTRasterBand>
<VRTRasterBand dataType="Byte" band="3">
<ColorInterp>Blue</ColorInterp>
<SimpleSource>
<SourceFilename>data/rgbsmall.tif</SourceFilename>
<SourceBand>3</SourceBand>
</SimpleSource>
</VRTRasterBand>
</VRTDataset>"""
ds = gdal.Open(vrt_text)
assert ds
assert ds.GetRasterBand(1).Checksum() == 21212
assert ds.GetRasterBand(2).Checksum() == 21053
assert ds.GetRasterBand(3).Checksum() == 21349
ds = None
after = gdaltest.get_opened_files()
if len(before) != len(after) and (
gdaltest.is_travis_branch('trusty_clang')
or gdaltest.is_travis_branch('trusty_32bit')
or gdaltest.is_travis_branch('ubuntu_1604')):
pytest.xfail('Mysterious failure')
assert len(before) == len(after)
def test_gdalinfo_13():
if test_cli_utilities.get_gdalinfo_path() is None:
pytest.skip()
ret = gdaltest.runexternal(test_cli_utilities.get_gdalinfo_path() + ' --license', check_memleak=False)
ret = ret.replace('\r\n', '\n')
if not ret.startswith(gdal.VersionInfo('LICENSE')):
print(gdal.VersionInfo('LICENSE'))
if gdaltest.is_travis_branch('mingw'):
return 'expected_fail'
pytest.fail(ret)
def test_ignfheightasciigrid_description_multiword_and_lf():
if gdaltest.is_travis_branch('mingw_w64'):
pytest.skip()
filename = '/vsimem/ignfheightasciigrid_invalid'
ok_content = b'2 3 49 50 1 1 1 0 1 0 -0. MULTI WORD\xC3\xA9\xC3\xA8\n1 2 3 4'
gdal.FileFromMemBuffer(filename, ok_content)
ds = gdal.OpenEx(filename)
desc = ds.GetMetadataItem('DESCRIPTION')
assert desc == 'MULTI WORDee'
def test_gdalbuildvrt_16():
if test_cli_utilities.get_gdalbuildvrt_path() is None:
pytest.skip()
(out, err) = gdaltest.runexternal_out_and_err(test_cli_utilities.get_gdalbuildvrt_path() + ' /non_existing_dir/non_existing_subdir/out.vrt ../gcore/data/byte.tif')
if not gdaltest.is_travis_branch('mingw'):
assert 'ERROR ret code = 1' in err, out
else:
# We don't get the error code on Travis mingw
assert 'ERROR' in err, out
def check_no_file_leaks():
num_files = len(gdaltest.get_opened_files())
yield
diff = len(gdaltest.get_opened_files()) - num_files
if diff != 0 and gdaltest.is_travis_branch('trusty_clang'):
print('Mysterious leak of file handle on trusty_clang')
return
assert diff == 0, 'Leak of file handles: %d leaked' % diff
def eedai_GOOGLE_APPLICATION_CREDENTIALS():
if gdaltest.eedai_drv is None:
return 'skip'
if gdaltest.is_travis_branch('gcc52_stdcpp14_sanitize'):
return 'skip'
gdal.FileFromMemBuffer('/vsimem/my.json', """{
"private_key":"-----BEGIN PRIVATE KEY-----
MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk\n
VNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm\n
i2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ\n
DVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy\n
TabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I\n
2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd\n
DrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+\n
bpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg\n
Wz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3\n
4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5\n
Kvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj\n
j+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI\n
yzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq\n
gwE6fxOLyJDxuWRf\n
-----END PRIVATE KEY-----",
"client_email":"*****@*****.**"
}""")
gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/')
gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', '/vsimem/my.json')
gdal.SetConfigOption('EEDA_PRIVATE_KEY', None)
gdal.SetConfigOption('EEDA_CLIENT_EMAIL', None)
gdal.SetConfigOption('GO2A_AUD', '/vsimem/oauth2/v4/token')
gdal.SetConfigOption('GOA2_NOW', '123456')
gdal.FileFromMemBuffer('/vsimem/oauth2/v4/token&POSTFIELDS=grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAibXlAZW1haWwuY29tIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZWFydGhlbmdpbmUucmVhZG9ubHkiLCAiYXVkIjogIi92c2ltZW0vb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.1W564xcQESVsqZmBEIMzj4rr0RuGa4RiUPZp5H%2FNENN9V9oPSTdacw%2BMiu3pcFf9AJv8wj0ajUeRsgTmvSicAftER49xeCQYUrs6uV122FGVsxml26kMFacNsCgRad%2Fy7xCAhMPfRJsqxS2%2BB392ssBeEzTGCSI6W3AsJg64OfA%3D',
'{ "access_token": "my_token", "token_type": "Bearer", "expires_in": 3600 }')
ds = gdal.Open('EEDAI:image')
gdal.Unlink('/vsimem/my.json')
gdal.SetConfigOption('EEDA_URL', None)
gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', None)
gdal.SetConfigOption('EEDA_PRIVATE_KEY', None)
gdal.SetConfigOption('EEDA_CLIENT_EMAIL', None)
if gdal.GetLastErrorMsg().find('CPLRSASHA256Sign() not implemented') >= 0:
return 'skip'
if ds is None:
return 'fail'
return 'success'
def do_GET(self):
if do_log:
f = open('/tmp/log.txt', 'a')
f.write('GET %s\n' % self.path)
f.close()
split = urllib.parse.urlparse(self.path)
query = urllib.parse.parse_qs(split.query)
query2 = {}
for key in query:
query2[key.lower()] = query[key]
server = query2['server'][0]
version = query2['version'][0]
request = query2['request'][0]
test = ''
if 'test' in query2:
test = query2['test'][0]
if gdaltest.is_travis_branch('s390x') or gdaltest.is_travis_branch(
'graviton2') or gdaltest.is_travis_branch('ubuntu_2004'):
# cannot strictly compare URL due to subtle difference of roundings
# in BOUNDINGBOX computations.
pass
else:
key = server + '-' + version
if key in urls and test in urls[key]:
_, got = self.path.split('SERVICE=WCS')
got = re.sub(r'\&test=.*', '', got)
_, have = urls[key][test].split('SERVICE=WCS')
have += '&server=' + server
if got == have:
ok = 'ok'
else:
ok = "not ok\ngot: " + got + "\nhave: " + have
global wcs_6_ok
wcs_6_ok = False
print('test ' + server + ' ' + test + ' WCS ' + version + ' ' +
ok)
self.Respond(request, server, version, test)
def test_gdalinfo_13():
if test_cli_utilities.get_gdalinfo_path() is None:
pytest.skip()
ret = gdaltest.runexternal(test_cli_utilities.get_gdalinfo_path() +
' --license',
check_memleak=False)
ret = ret.replace('\r\n', '\n')
if not ret.startswith(gdal.VersionInfo('LICENSE')):
print(gdal.VersionInfo('LICENSE'))
if gdaltest.is_travis_branch('mingw'):
return 'expected_fail'
pytest.fail(ret)
def test_postgisraster_test_outdb_client_side_if_possible():
if gdaltest.postgisrasterDriver is None:
pytest.skip()
# For some reason fails with
# ERROR 1: PostGISRasterRasterBand::IRasterIO(): ERROR: rt_band_load_offline_data: Cannot open offline raster: /home/travis/build/rouault/gdal/autotest/gdrivers/data/small_world.tif
# See https://api.travis-ci.org/v3/job/484385907/log.txt
if gdaltest.is_travis_branch('ubuntu_1804'):
pytest.skip()
ds = gdal.Open(gdaltest.postgisraster_connection_string + "table='small_world_outdb_constraint' mode=2 outdb_resolution=client_side_if_possible")
cs = [ds.GetRasterBand(i + 1).Checksum() for i in range(3)]
assert cs == [30111, 32302, 40026]
def runexternal(cmd,
strin=None,
check_memleak=True,
display_live_on_parent_stdout=False,
encoding=None):
from gdaltest import is_travis_branch
if not is_travis_branch('mingw'):
has_subprocess = False
try:
import subprocess
import shlex
subprocess.Popen
shlex.split
has_subprocess = True
except:
pass
if has_subprocess:
return _runexternal_subprocess(
cmd,
strin=strin,
check_memleak=check_memleak,
display_live_on_parent_stdout=display_live_on_parent_stdout,
encoding=encoding)
if strin is None:
ret_stdout = os.popen(cmd)
else:
(ret_stdin, ret_stdout) = os.popen2(cmd)
ret_stdin.write(strin)
ret_stdin.close()
if display_live_on_parent_stdout:
out_str = ''
while True:
c = ret_stdout.read(1)
if c == '':
break
out_str = out_str + c
sys.stdout.write(c)
ret_stdout.close()
else:
out_str = ret_stdout.read()
ret_stdout.close()
if check_memleak:
warn_if_memleak(cmd, out_str)
if encoding is not None:
out_str = out_str.decode(encoding)
return out_str
def test_postgisraster_test_outdb_client_side_if_possible():
if gdaltest.postgisrasterDriver is None:
pytest.skip()
# For some reason fails with
# ERROR 1: PostGISRasterRasterBand::IRasterIO(): ERROR: rt_band_load_offline_data: Cannot open offline raster: /home/travis/build/rouault/gdal/autotest/gdrivers/data/small_world.tif
# See https://api.travis-ci.org/v3/job/484385907/log.txt
if gdaltest.is_travis_branch('ubuntu_1804'):
pytest.skip()
ds = gdal.Open(gdaltest.postgisraster_connection_string + "table='small_world_outdb_constraint' mode=2 outdb_resolution=client_side_if_possible")
cs = [ds.GetRasterBand(i + 1).Checksum() for i in range(3)]
assert cs == [30111, 32302, 40026]
def test_gdalinfo_13():
if test_cli_utilities.get_gdalinfo_path() is None:
return 'skip'
ret = gdaltest.runexternal(test_cli_utilities.get_gdalinfo_path() + ' --license', check_memleak=False)
ret = ret.replace('\r\n', '\n')
if ret.find(gdal.VersionInfo('LICENSE')) != 0:
print(ret)
print(gdal.VersionInfo('LICENSE'))
if gdaltest.is_travis_branch('mingw'):
return 'expected_fail'
return 'fail'
return 'success'
def test_gdalsrsinfo_6():
if test_cli_utilities.get_gdalsrsinfo_path() is None:
pytest.skip()
if gdaltest.is_travis_branch('mingw'):
pytest.skip()
ret = gdaltest.runexternal(test_cli_utilities.get_gdalsrsinfo_path() +
' --single-line -o wkt_simple ../gcore/data/byte.tif')
ret = ret.replace('\r\n', '\n')
val = """PROJCS["NAD27 / UTM zone 11N",GEOGCS["NAD27",DATUM["North_American_Datum_1927",SPHEROID["Clarke 1866",6378206.4,294.978698213898]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",-117],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1]]"""
assert ret.strip() == val
def test_gdalsrsinfo_6():
if test_cli_utilities.get_gdalsrsinfo_path() is None:
pytest.skip()
if gdaltest.is_travis_branch('mingw'):
pytest.skip()
ret = gdaltest.runexternal(test_cli_utilities.get_gdalsrsinfo_path() +
' --single-line -o wkt_simple ../gcore/data/byte.tif')
ret = ret.replace('\r\n', '\n')
val = """PROJCS["NAD27 / UTM zone 11N",GEOGCS["NAD27",DATUM["North_American_Datum_1927",SPHEROID["Clarke 1866",6378206.4,294.978698213898]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",-117],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1]]"""
assert ret.strip() == val
def test_gdalinfo_13():
if test_cli_utilities.get_gdalinfo_path() is None:
return 'skip'
ret = gdaltest.runexternal(test_cli_utilities.get_gdalinfo_path() + ' --license', check_memleak = False )
ret = ret.replace('\r\n', '\n')
if ret.find(gdal.VersionInfo('LICENSE')) != 0:
print(ret)
print(gdal.VersionInfo('LICENSE'))
if gdaltest.is_travis_branch('mingw'):
return 'expected_fail'
return 'fail'
return 'success'
def test_ogrinfo_nogeomtype():
if test_cli_utilities.get_ogrinfo_path() is None:
pytest.skip()
(ret, err) = gdaltest.runexternal_out_and_err(test_cli_utilities.get_ogrinfo_path() + ' -nogeomtype ../ogr/data/poly.shp')
assert (err is None or err == ''), 'got error/warning'
expected_ret = """INFO: Open of `../ogr/data/poly.shp'
using driver `ESRI Shapefile' successful.
1: poly
"""
expected_lines = expected_ret.splitlines()
lines = ret.splitlines()
for i, exp_line in enumerate(expected_lines):
if exp_line != lines[i]:
if gdaltest.is_travis_branch('mingw'):
return 'expected_fail'
pytest.fail(ret)
def runexternal(cmd, strin = None, check_memleak = True, display_live_on_parent_stdout = False, encoding = None):
from gdaltest import is_travis_branch
if not is_travis_branch('mingw'):
has_subprocess = False
try:
import subprocess
import shlex
subprocess.Popen
shlex.split
has_subprocess = True
except:
pass
if has_subprocess:
return _runexternal_subprocess(cmd,
strin=strin,
check_memleak=check_memleak,
display_live_on_parent_stdout=display_live_on_parent_stdout,
encoding=encoding)
if strin is None:
ret_stdout = os.popen(cmd)
else:
(ret_stdin, ret_stdout) = os.popen2(cmd)
ret_stdin.write(strin)
ret_stdin.close()
if display_live_on_parent_stdout:
out_str = ''
while True:
c = ret_stdout.read(1)
if c == '':
break
out_str = out_str + c
sys.stdout.write(c)
ret_stdout.close()
else:
out_str = ret_stdout.read()
ret_stdout.close()
if check_memleak:
warn_if_memleak(cmd, out_str)
if encoding is not None:
out_str = out_str.decode(encoding)
return out_str
def test_pixfun_mod_r():
filename = 'data/vrt/pixfun_mod_r.vrt'
ds = gdal.OpenShared(filename, gdal.GA_ReadOnly)
assert ds is not None, ('Unable to open "%s" dataset.' % filename)
data = ds.GetRasterBand(1).ReadAsArray()
reffilename = 'data/int32.tif'
refds = gdal.Open(reffilename)
assert refds is not None, ('Unable to open "%s" dataset.' % reffilename)
refdata = refds.GetRasterBand(1).ReadAsArray()
res = numpy.alltrue(data == numpy.abs(refdata))
if gdaltest.is_travis_branch('sanitize') and not res:
print(data)
print(numpy.abs(refdata))
pytest.xfail()
assert res
def test_gdalbuildvrt_16():
if test_cli_utilities.get_gdalbuildvrt_path() is None:
return 'skip'
(out, err) = gdaltest.runexternal_out_and_err(test_cli_utilities.get_gdalbuildvrt_path() + ' /non_existing_dir/non_existing_subdir/out.vrt ../gcore/data/byte.tif')
if not gdaltest.is_travis_branch('mingw'):
if err.find('ERROR ret code = 1') < 0:
gdaltest.post_reason('fail')
print(out)
print(err)
return 'fail'
else:
# We don't get the error code on Travis mingw
if err.find('ERROR') < 0:
gdaltest.post_reason('fail')
print(out)
print(err)
return 'fail'
return 'success'
def test_gdalbuildvrt_16():
if test_cli_utilities.get_gdalbuildvrt_path() is None:
return 'skip'
(out, err) = gdaltest.runexternal_out_and_err(test_cli_utilities.get_gdalbuildvrt_path() + ' /non_existing_dir/non_existing_subdir/out.vrt ../gcore/data/byte.tif')
if not gdaltest.is_travis_branch('mingw'):
if err.find('ERROR ret code = 1') < 0:
gdaltest.post_reason('fail')
print(out)
print(err)
return 'fail'
else:
# We don't get the error code on Travis mingw
if err.find('ERROR') < 0:
gdaltest.post_reason('fail')
print(out)
print(err)
return 'fail'
return 'success'
def vsicurl_test_redirect():
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
f = gdal.VSIFOpenL(
'/vsicurl/http://localhost:%d/test_redirect/test.bin' %
gdaltest.webserver_port, 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFSeekL(f, 0, 2)
if gdal.VSIFTellL(f) != 1000000:
gdaltest.post_reason('fail')
print(gdal.VSIFTellL(f))
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFSeekL(f, 0, 0)
content = gdal.VSIFReadL(1, 16383, f).decode('ascii')
if len(content) != 16383 or content[0] != 'x':
gdaltest.post_reason('fail')
print(content)
gdal.VSIFCloseL(f)
return 'fail'
content = gdal.VSIFReadL(1, 2, f).decode('ascii')
if content != 'xy':
gdaltest.post_reason('fail')
print(content)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
return 'success'
def vsicurl_test_redirect():
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
f = gdal.VSIFOpenL('/vsicurl/http://localhost:%d/test_redirect/test.bin' % gdaltest.webserver_port, 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFSeekL(f,0,2)
if gdal.VSIFTellL(f) != 1000000:
gdaltest.post_reason('fail')
print(gdal.VSIFTellL(f))
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFSeekL(f,0,0)
content = gdal.VSIFReadL(1, 16383, f).decode('ascii')
if len(content) != 16383 or content[0] != 'x':
gdaltest.post_reason('fail')
print(content)
gdal.VSIFCloseL(f)
return 'fail'
content = gdal.VSIFReadL(1, 2, f).decode('ascii')
if content != 'xy':
gdaltest.post_reason('fail')
print(content)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
return 'success'
def test_visoss_2():
if gdaltest.webserver_port == 0:
pytest.skip()
signed_url = gdal.GetSignedURL('/vsioss/oss_fake_bucket/resource',
['START_DATE=20180212T123456Z'])
assert (signed_url in (
'http://127.0.0.1:8080/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D',
'http://127.0.0.1:8081/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D'
))
handler = webserver.SequentialHandler()
handler.add('GET',
'/oss_fake_bucket/resource',
custom_method=get_oss_fake_bucket_resource_method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
handler.add('GET',
'/oss_fake_bucket/resource',
custom_method=get_oss_fake_bucket_resource_method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss_streaming/oss_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(400)
response = """<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>AccessDenied</Code>
<Message>The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.</Message>
<HostId>unused</HostId>
<Bucket>unuset</Bucket>
<Endpoint>localhost:%d</Endpoint>
</Error>""" % request.server.port
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
request.send_header('Content-type', 'application/xml')
request.send_header('Transfer-Encoding', 'chunked')
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method)
def method(request):
request.protocol_version = 'HTTP/1.1'
if request.headers['Host'].startswith('localhost'):
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
else:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method)
# Test region and endpoint 'redirects'
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_fake_bucket/redirect')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
pytest.fail(data)
# Test region and endpoint 'redirects'
handler.req_count = 0
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss_streaming/oss_fake_bucket/redirect')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
def method(request):
# /vsioss_streaming/ should have remembered the change of region and endpoint
if not request.headers['Host'].startswith('localhost'):
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.protocol_version = 'HTTP/1.1'
request.send_response(400)
response = 'bla'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
request.send_header('Content-type', 'application/xml')
request.send_header('Transfer-Encoding', 'chunked')
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET', '/oss_fake_bucket/non_xml_error', custom_method=method)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/non_xml_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('bla') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><oops>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/invalid_xml_error', 400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/invalid_xml_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<oops>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error/>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/no_code_in_error', 400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_code_in_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error/>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>AuthorizationHeaderMalformed</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET',
'/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error',
400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error'
)
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>PermanentRedirect</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error', 400,
{
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error'
)
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>bla</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add(
'GET', '/oss_fake_bucket/no_message_in_error', 400, {
'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'
}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read(
'/vsioss_streaming/oss_fake_bucket/no_message_in_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
def test_vsigs_readdir():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add('GET', '/gs_fake_bucket2/?delimiter=%2F&prefix=a_dir%2F', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Contents>
<Key>a_dir/resource3.bin</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>123456</Size>
</Contents>
</ListBucketResult>
""")
handler.add('GET', '/gs_fake_bucket2/?delimiter=%2F&marker=bla&prefix=a_dir%2F', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<Contents>
<Key>a_dir/resource4.bin</Key>
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
</Contents>
<CommonPrefixes>
<Prefix>a_dir/subdir/</Prefix>
</CommonPrefixes>
</ListBucketResult>
""")
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
pytest.fail()
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir')
assert dir_contents == ['resource3.bin', 'resource4.bin', 'subdir']
assert gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').size == 123456
assert gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').mtime == 1
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir/resource3.bin')
assert dir_contents is None
# List buckets
handler = webserver.SequentialHandler()
handler.add('GET', '/', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult>
<Buckets>
<Bucket>
<Name>mybucket</Name>
</Bucket>
</Buckets>
</ListAllMyBucketsResult>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsigs/')
assert dir_contents == ['mybucket']
def test_ogrinfo_24():
if test_cli_utilities.get_ogrinfo_path() is None:
pytest.skip()
f = open('tmp/test_ogrinfo_24.vrt', 'wt')
f.write("""<OGRVRTDataSource>
<Metadata>
<MDI key="foo">bar</MDI>
</Metadata>
<Metadata domain="other_domain">
<MDI key="baz">foo</MDI>
</Metadata>
<OGRVRTLayer name="poly">
<Metadata>
<MDI key="bar">baz</MDI>
</Metadata>
<SrcDataSource relativeToVRT="1" shared="1">../../ogr/data/poly.shp</SrcDataSource>
<SrcLayer>poly</SrcLayer>
</OGRVRTLayer>
</OGRVRTDataSource>""")
f.close()
ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() + ' -ro -al tmp/test_ogrinfo_24.vrt -so', check_memleak=False)
expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt'
using driver `OGR_VRT' successful.
Metadata:
foo=bar
Layer name: poly
Metadata:
bar=baz
Geometry: Polygon
Feature Count: 10
Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000)
Layer SRS WKT:
PROJCS["OSGB 1936 / British National Grid",
GEOGCS["OSGB 1936",
DATUM["OSGB_1936",
SPHEROID["Airy 1830",6377563.396,299.3249646,
AUTHORITY["EPSG","7001"]],
AUTHORITY["EPSG","6277"]],
PRIMEM["Greenwich",0,
AUTHORITY["EPSG","8901"]],
UNIT["degree",0.0174532925199433,
AUTHORITY["EPSG","9122"]],
AUTHORITY["EPSG","4277"]],
PROJECTION["Transverse_Mercator"],
PARAMETER["latitude_of_origin",49],
PARAMETER["central_meridian",-2],
PARAMETER["scale_factor",0.9996012717],
PARAMETER["false_easting",400000],
PARAMETER["false_northing",-100000],
UNIT["metre",1,
AUTHORITY["EPSG","9001"]],
AXIS["Easting",EAST],
AXIS["Northing",NORTH],
AUTHORITY["EPSG","27700"]]
"""
expected_lines = expected_ret.splitlines()
lines = ret.splitlines()
for i, exp_line in enumerate(expected_lines):
if exp_line != lines[i]:
if gdaltest.is_travis_branch('mingw'):
return 'expected_fail'
pytest.fail(ret)
ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() + ' -ro -al tmp/test_ogrinfo_24.vrt -so -mdd all', check_memleak=False)
expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt'
using driver `OGR_VRT' successful.
Metadata:
foo=bar
Metadata (other_domain):
baz=foo
Layer name: poly
Metadata:
bar=baz
Geometry: Polygon
Feature Count: 10
Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000)
Layer SRS WKT:
PROJCS["OSGB 1936 / British National Grid",
GEOGCS["OSGB 1936",
DATUM["OSGB_1936",
SPHEROID["Airy 1830",6377563.396,299.3249646,
AUTHORITY["EPSG","7001"]],
AUTHORITY["EPSG","6277"]],
PRIMEM["Greenwich",0,
AUTHORITY["EPSG","8901"]],
UNIT["degree",0.0174532925199433,
AUTHORITY["EPSG","9122"]],
AUTHORITY["EPSG","4277"]],
PROJECTION["Transverse_Mercator"],
PARAMETER["latitude_of_origin",49],
PARAMETER["central_meridian",-2],
PARAMETER["scale_factor",0.9996012717],
PARAMETER["false_easting",400000],
PARAMETER["false_northing",-100000],
UNIT["metre",1,
AUTHORITY["EPSG","9001"]],
AXIS["Easting",EAST],
AXIS["Northing",NORTH],
AUTHORITY["EPSG","27700"]]
Data axis to CRS axis mapping: 1,2
AREA: Real (12.3)
EAS_ID: Integer64 (11.0)
PRFEDEA: String (16.0)
"""
expected_lines = expected_ret.splitlines()
lines = ret.splitlines()
for i, exp_line in enumerate(expected_lines):
if exp_line != lines[i]:
if gdaltest.is_travis_branch('mingw'):
return 'expected_fail'
pytest.fail(ret)
ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() + ' -ro -al tmp/test_ogrinfo_24.vrt -so -nomd', check_memleak=False)
expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt'
using driver `OGR_VRT' successful.
Layer name: poly
Geometry: Polygon
Feature Count: 10
Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000)
Layer SRS WKT:
PROJCS["OSGB 1936 / British National Grid",
GEOGCS["OSGB 1936",
DATUM["OSGB_1936",
SPHEROID["Airy 1830",6377563.396,299.3249646,
AUTHORITY["EPSG","7001"]],
AUTHORITY["EPSG","6277"]],
PRIMEM["Greenwich",0,
AUTHORITY["EPSG","8901"]],
UNIT["degree",0.0174532925199433,
AUTHORITY["EPSG","9122"]],
AUTHORITY["EPSG","4277"]],
PROJECTION["Transverse_Mercator"],
PARAMETER["latitude_of_origin",49],
PARAMETER["central_meridian",-2],
PARAMETER["scale_factor",0.9996012717],
PARAMETER["false_easting",400000],
PARAMETER["false_northing",-100000],
UNIT["metre",1,
AUTHORITY["EPSG","9001"]],
AXIS["Easting",EAST],
AXIS["Northing",NORTH],
AUTHORITY["EPSG","27700"]]
Data axis to CRS axis mapping: 1,2
AREA: Real (12.3)
EAS_ID: Integer64 (11.0)
PRFEDEA: String (16.0)
"""
expected_lines = expected_ret.splitlines()
lines = ret.splitlines()
for i, exp_line in enumerate(expected_lines):
if exp_line != lines[i]:
if gdaltest.is_travis_branch('mingw'):
return 'expected_fail'
pytest.fail(ret)
os.unlink('tmp/test_ogrinfo_24.vrt')
def test_visoss_2():
if gdaltest.webserver_port == 0:
pytest.skip()
signed_url = gdal.GetSignedURL('/vsioss/oss_fake_bucket/resource',
['START_DATE=20180212T123456Z'])
assert (signed_url in ('http://127.0.0.1:8080/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D',
'http://127.0.0.1:8081/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D'))
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_fake_bucket/resource', custom_method=get_oss_fake_bucket_resource_method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_fake_bucket/resource', custom_method=get_oss_fake_bucket_resource_method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss_streaming/oss_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(400)
response = """<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>AccessDenied</Code>
<Message>The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.</Message>
<HostId>unused</HostId>
<Bucket>unuset</Bucket>
<Endpoint>localhost:%d</Endpoint>
</Error>""" % request.server.port
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
request.send_header('Content-type', 'application/xml')
request.send_header('Transfer-Encoding', 'chunked')
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method)
def method(request):
request.protocol_version = 'HTTP/1.1'
if request.headers['Host'].startswith('localhost'):
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
else:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method)
# Test region and endpoint 'redirects'
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_fake_bucket/redirect')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
pytest.fail(data)
# Test region and endpoint 'redirects'
handler.req_count = 0
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss_streaming/oss_fake_bucket/redirect')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
def method(request):
# /vsioss_streaming/ should have remembered the change of region and endpoint
if not request.headers['Host'].startswith('localhost'):
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.protocol_version = 'HTTP/1.1'
request.send_response(400)
response = 'bla'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
request.send_header('Content-type', 'application/xml')
request.send_header('Transfer-Encoding', 'chunked')
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET', '/oss_fake_bucket/non_xml_error', custom_method=method)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read('/vsioss_streaming/oss_fake_bucket/non_xml_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('bla') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><oops>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add('GET', '/oss_fake_bucket/invalid_xml_error', 400,
{'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read('/vsioss_streaming/oss_fake_bucket/invalid_xml_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<oops>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error/>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add('GET', '/oss_fake_bucket/no_code_in_error', 400,
{'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read('/vsioss_streaming/oss_fake_bucket/no_code_in_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error/>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>AuthorizationHeaderMalformed</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add('GET', '/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error', 400,
{'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read('/vsioss_streaming/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>PermanentRedirect</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add('GET', '/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error', 400,
{'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read('/vsioss_streaming/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
handler = webserver.SequentialHandler()
response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>bla</Code></Error>'
response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response)
handler.add('GET', '/oss_fake_bucket/no_message_in_error', 400,
{'Content-type': 'application/xml',
'Transfer-Encoding': 'chunked',
'Connection': 'close'}, response)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
f = open_for_read('/vsioss_streaming/oss_fake_bucket/no_message_in_error')
assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
def vsiaz_fake_readdir():
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add('GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%2F&restype=container', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Blobs>
<Blob>
<Name>a_dir/resource3.bin</Name>
<Properties>
<Last-Modified>01 Jan 1970 00:00:01</Last-Modified>
<Content-Length>123456</Content-Length>
</Properties>
</Blob>
</Blobs>
</EnumerationResults>
""")
handler.add('GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla&prefix=a_dir%2F&restype=container', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<Blobs>
<Blob>
<Name>a_dir/resource4.bin</Name>
<Properties>
<Last-Modified>16 Oct 2016 12:34:56</Last-Modified>
<Content-Length>456789</Content-Length>
</Properties>
</Blob>
<BlobPrefix>
<Name>a_dir/subdir/</Name>
</BlobPrefix>
</Blobs>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
print(gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size)
return 'fail'
if gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
if dir_contents is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add('GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=error_test%2F&restype=container', 500)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/error_test/')
if dir_contents is not None:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# List containers (empty result)
handler = webserver.SequentialHandler()
handler.add('GET', '/azure/blob/myaccount/?comp=list', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults ServiceEndpoint="https://myaccount.blob.core.windows.net">
<Containers/>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
if dir_contents != ['.']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
gdal.VSICurlClearCache()
# List containers
handler = webserver.SequentialHandler()
handler.add('GET', '/azure/blob/myaccount/?comp=list', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer1</Name>
</Container>
</Containers>
<NextMarker>bla</NextMarker>
</EnumerationResults>
""")
handler.add('GET', '/azure/blob/myaccount/?comp=list&marker=bla', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer2</Name>
</Container>
</Containers>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
if dir_contents != ['mycontainer1', 'mycontainer2']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
return 'success'
def vsiaz_fake_readdir():
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%2F&restype=container',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Blobs>
<Blob>
<Name>a_dir/resource3.bin</Name>
<Properties>
<Last-Modified>01 Jan 1970 00:00:01</Last-Modified>
<Content-Length>123456</Content-Length>
</Properties>
</Blob>
</Blobs>
</EnumerationResults>
""")
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla&prefix=a_dir%2F&restype=container',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<Blobs>
<Blob>
<Name>a_dir/resource4.bin</Name>
<Properties>
<Last-Modified>16 Oct 2016 12:34:56</Last-Modified>
<Content-Length>456789</Content-Length>
</Properties>
</Blob>
<BlobPrefix>
<Name>a_dir/subdir/</Name>
</BlobPrefix>
</Blobs>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL(
'/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
print(gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size)
return 'fail'
if gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
if dir_contents is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=error_test%2F&restype=container',
500)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/error_test/')
if dir_contents is not None:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# List containers (empty result)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/azure/blob/myaccount/?comp=list', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults ServiceEndpoint="https://myaccount.blob.core.windows.net">
<Containers/>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
if dir_contents != ['.']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
gdal.VSICurlClearCache()
# List containers
handler = webserver.SequentialHandler()
handler.add(
'GET', '/azure/blob/myaccount/?comp=list', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer1</Name>
</Container>
</Containers>
<NextMarker>bla</NextMarker>
</EnumerationResults>
""")
handler.add(
'GET', '/azure/blob/myaccount/?comp=list&marker=bla', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer2</Name>
</Container>
</Containers>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
if dir_contents != ['mycontainer1', 'mycontainer2']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
return 'success'
def test_ogrinfo_24():
if test_cli_utilities.get_ogrinfo_path() is None:
pytest.skip()
f = open('tmp/test_ogrinfo_24.vrt', 'wt')
f.write("""<OGRVRTDataSource>
<Metadata>
<MDI key="foo">bar</MDI>
</Metadata>
<Metadata domain="other_domain">
<MDI key="baz">foo</MDI>
</Metadata>
<OGRVRTLayer name="poly">
<Metadata>
<MDI key="bar">baz</MDI>
</Metadata>
<SrcDataSource relativeToVRT="1" shared="1">../../ogr/data/poly.shp</SrcDataSource>
<SrcLayer>poly</SrcLayer>
</OGRVRTLayer>
</OGRVRTDataSource>""")
f.close()
ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() +
' -ro -al tmp/test_ogrinfo_24.vrt -so',
check_memleak=False)
expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt'
using driver `OGR_VRT' successful.
Metadata:
foo=bar
Layer name: poly
Metadata:
bar=baz
Geometry: Polygon
Feature Count: 10
Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000)
Layer SRS WKT:
PROJCS["OSGB 1936 / British National Grid",
GEOGCS["OSGB 1936",
DATUM["OSGB_1936",
SPHEROID["Airy 1830",6377563.396,299.3249646,
AUTHORITY["EPSG","7001"]],
TOWGS84[446.448,-125.157,542.06,0.15,0.247,0.842,-20.489],
AUTHORITY["EPSG","6277"]],
PRIMEM["Greenwich",0,
AUTHORITY["EPSG","8901"]],
UNIT["degree",0.0174532925199433,
AUTHORITY["EPSG","9122"]],
AUTHORITY["EPSG","4277"]],
PROJECTION["Transverse_Mercator"],
PARAMETER["latitude_of_origin",49],
PARAMETER["central_meridian",-2],
PARAMETER["scale_factor",0.9996012717],
PARAMETER["false_easting",400000],
PARAMETER["false_northing",-100000],
UNIT["metre",1,
AUTHORITY["EPSG","9001"]],
AXIS["Easting",EAST],
AXIS["Northing",NORTH],
AUTHORITY["EPSG","27700"]]
"""
expected_lines = expected_ret.splitlines()
lines = ret.splitlines()
for i, exp_line in enumerate(expected_lines):
if exp_line != lines[i]:
if gdaltest.is_travis_branch('mingw'):
return 'expected_fail'
pytest.fail(ret)
ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() +
' -ro -al tmp/test_ogrinfo_24.vrt -so -mdd all',
check_memleak=False)
expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt'
using driver `OGR_VRT' successful.
Metadata:
foo=bar
Metadata (other_domain):
baz=foo
Layer name: poly
Metadata:
bar=baz
Geometry: Polygon
Feature Count: 10
Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000)
Layer SRS WKT:
PROJCS["OSGB 1936 / British National Grid",
GEOGCS["OSGB 1936",
DATUM["OSGB_1936",
SPHEROID["Airy 1830",6377563.396,299.3249646,
AUTHORITY["EPSG","7001"]],
TOWGS84[446.448,-125.157,542.06,0.15,0.247,0.842,-20.489],
AUTHORITY["EPSG","6277"]],
PRIMEM["Greenwich",0,
AUTHORITY["EPSG","8901"]],
UNIT["degree",0.0174532925199433,
AUTHORITY["EPSG","9122"]],
AUTHORITY["EPSG","4277"]],
PROJECTION["Transverse_Mercator"],
PARAMETER["latitude_of_origin",49],
PARAMETER["central_meridian",-2],
PARAMETER["scale_factor",0.9996012717],
PARAMETER["false_easting",400000],
PARAMETER["false_northing",-100000],
UNIT["metre",1,
AUTHORITY["EPSG","9001"]],
AXIS["Easting",EAST],
AXIS["Northing",NORTH],
AUTHORITY["EPSG","27700"]]
AREA: Real (12.3)
EAS_ID: Integer64 (11.0)
PRFEDEA: String (16.0)
"""
expected_lines = expected_ret.splitlines()
lines = ret.splitlines()
for i, exp_line in enumerate(expected_lines):
if exp_line != lines[i]:
if gdaltest.is_travis_branch('mingw'):
return 'expected_fail'
pytest.fail(ret)
ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() +
' -ro -al tmp/test_ogrinfo_24.vrt -so -nomd',
check_memleak=False)
expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt'
using driver `OGR_VRT' successful.
Layer name: poly
Geometry: Polygon
Feature Count: 10
Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000)
Layer SRS WKT:
PROJCS["OSGB 1936 / British National Grid",
GEOGCS["OSGB 1936",
DATUM["OSGB_1936",
SPHEROID["Airy 1830",6377563.396,299.3249646,
AUTHORITY["EPSG","7001"]],
TOWGS84[446.448,-125.157,542.06,0.15,0.247,0.842,-20.489],
AUTHORITY["EPSG","6277"]],
PRIMEM["Greenwich",0,
AUTHORITY["EPSG","8901"]],
UNIT["degree",0.0174532925199433,
AUTHORITY["EPSG","9122"]],
AUTHORITY["EPSG","4277"]],
PROJECTION["Transverse_Mercator"],
PARAMETER["latitude_of_origin",49],
PARAMETER["central_meridian",-2],
PARAMETER["scale_factor",0.9996012717],
PARAMETER["false_easting",400000],
PARAMETER["false_northing",-100000],
UNIT["metre",1,
AUTHORITY["EPSG","9001"]],
AXIS["Easting",EAST],
AXIS["Northing",NORTH],
AUTHORITY["EPSG","27700"]]
AREA: Real (12.3)
EAS_ID: Integer64 (11.0)
PRFEDEA: String (16.0)
"""
expected_lines = expected_ret.splitlines()
lines = ret.splitlines()
for i, exp_line in enumerate(expected_lines):
if exp_line != lines[i]:
if gdaltest.is_travis_branch('mingw'):
return 'expected_fail'
pytest.fail(ret)
os.unlink('tmp/test_ogrinfo_24.vrt')
def test_visoss_3():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'application/xml')
response = """<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Contents>
<Key>a_dir/resource3.bin</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>123456</Size>
</Contents>
</ListBucketResult>
"""
request.send_header('Content-Length', len(response))
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET',
'/oss_fake_bucket2/?delimiter=%2F&prefix=a_dir%2F',
custom_method=method)
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'application/xml')
response = """<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<Contents>
<Key>a_dir/resource4.bin</Key>
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
</Contents>
<CommonPrefixes>
<Prefix>a_dir/subdir/</Prefix>
</CommonPrefixes>
</ListBucketResult>
"""
request.send_header('Content-Length', len(response))
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET',
'/oss_fake_bucket2/?delimiter=%2F&marker=bla&prefix=a_dir%2F',
custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
pytest.fail()
gdal.VSIFCloseL(f)
with webserver.install_http_handler(webserver.SequentialHandler()):
dir_contents = gdal.ReadDir('/vsioss/oss_fake_bucket2/a_dir')
assert dir_contents == ['resource3.bin', 'resource4.bin', 'subdir']
assert gdal.VSIStatL(
'/vsioss/oss_fake_bucket2/a_dir/resource3.bin').size == 123456
assert gdal.VSIStatL(
'/vsioss/oss_fake_bucket2/a_dir/resource3.bin').mtime == 1
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsioss/oss_fake_bucket2/a_dir/resource3.bin')
assert dir_contents is None
# Test CPL_VSIL_CURL_NON_CACHED
for config_option_value in [
'/vsioss/oss_non_cached/test.txt', '/vsioss/oss_non_cached',
'/vsioss/oss_non_cached:/vsioss/unrelated',
'/vsioss/unrelated:/vsioss/oss_non_cached',
'/vsioss/unrelated:/vsioss/oss_non_cached:/vsioss/unrelated'
]:
with gdaltest.config_option('CPL_VSIL_CURL_NON_CACHED',
config_option_value):
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo', config_option_value
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'bar2')
with webserver.install_http_handler(handler):
size = gdal.VSIStatL('/vsioss/oss_non_cached/test.txt').size
assert size == 4, config_option_value
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
size = gdal.VSIStatL('/vsioss/oss_non_cached/test.txt').size
if size != 3:
print(config_option_value)
pytest.fail(data)
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'bar2')
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'bar2', config_option_value
# Retry without option
for config_option_value in [None, '/vsioss/oss_non_cached/bar.txt']:
with gdaltest.config_option('CPL_VSIL_CURL_NON_CACHED',
config_option_value):
handler = webserver.SequentialHandler()
if config_option_value is None:
handler.add(
'GET', '/oss_non_cached/?delimiter=%2F', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>/</Prefix>
<Contents>
<Key>/test.txt</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>40</Size>
</Contents>
<Contents>
<Key>/test2.txt</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>40</Size>
</Contents>
</ListBucketResult>
""")
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo', config_option_value
handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
# We should still get foo because of caching
assert data == 'foo', config_option_value
# List buckets (empty result)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult>
<Buckets>
</Buckets>
</ListAllMyBucketsResult>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsioss/')
assert dir_contents == ['.']
gdal.VSICurlClearCache()
# List buckets
handler = webserver.SequentialHandler()
handler.add(
'GET', '/', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult>
<Buckets>
<Bucket>
<Name>mybucket</Name>
</Bucket>
</Buckets>
</ListAllMyBucketsResult>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsioss/')
assert dir_contents == ['mybucket']
def test_visoss_3():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'application/xml')
response = """<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Contents>
<Key>a_dir/resource3.bin</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>123456</Size>
</Contents>
</ListBucketResult>
"""
request.send_header('Content-Length', len(response))
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET', '/oss_fake_bucket2/?delimiter=%2F&prefix=a_dir%2F', custom_method=method)
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'application/xml')
response = """<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<Contents>
<Key>a_dir/resource4.bin</Key>
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
</Contents>
<CommonPrefixes>
<Prefix>a_dir/subdir/</Prefix>
</CommonPrefixes>
</ListBucketResult>
"""
request.send_header('Content-Length', len(response))
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('GET', '/oss_fake_bucket2/?delimiter=%2F&marker=bla&prefix=a_dir%2F', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
pytest.fail()
gdal.VSIFCloseL(f)
with webserver.install_http_handler(webserver.SequentialHandler()):
dir_contents = gdal.ReadDir('/vsioss/oss_fake_bucket2/a_dir')
assert dir_contents == ['resource3.bin', 'resource4.bin', 'subdir']
assert gdal.VSIStatL('/vsioss/oss_fake_bucket2/a_dir/resource3.bin').size == 123456
assert gdal.VSIStatL('/vsioss/oss_fake_bucket2/a_dir/resource3.bin').mtime == 1
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsioss/oss_fake_bucket2/a_dir/resource3.bin')
assert dir_contents is None
# Test CPL_VSIL_CURL_NON_CACHED
for config_option_value in ['/vsioss/oss_non_cached/test.txt',
'/vsioss/oss_non_cached',
'/vsioss/oss_non_cached:/vsioss/unrelated',
'/vsioss/unrelated:/vsioss/oss_non_cached',
'/vsioss/unrelated:/vsioss/oss_non_cached:/vsioss/unrelated']:
with gdaltest.config_option('CPL_VSIL_CURL_NON_CACHED', config_option_value):
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo', config_option_value
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'bar2')
with webserver.install_http_handler(handler):
size = gdal.VSIStatL('/vsioss/oss_non_cached/test.txt').size
assert size == 4, config_option_value
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
size = gdal.VSIStatL('/vsioss/oss_non_cached/test.txt').size
if size != 3:
print(config_option_value)
pytest.fail(data)
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'bar2')
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'bar2', config_option_value
# Retry without option
for config_option_value in [None,
'/vsioss/oss_non_cached/bar.txt']:
with gdaltest.config_option('CPL_VSIL_CURL_NON_CACHED', config_option_value):
handler = webserver.SequentialHandler()
if config_option_value is None:
handler.add('GET', '/oss_non_cached/?delimiter=%2F', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>/</Prefix>
<Contents>
<Key>/test.txt</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>40</Size>
</Contents>
<Contents>
<Key>/test2.txt</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>40</Size>
</Contents>
</ListBucketResult>
""")
handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo')
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo', config_option_value
handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
f = open_for_read('/vsioss/oss_non_cached/test.txt')
assert f is not None, config_option_value
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
# We should still get foo because of caching
assert data == 'foo', config_option_value
# List buckets (empty result)
handler = webserver.SequentialHandler()
handler.add('GET', '/', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult>
<Buckets>
</Buckets>
</ListAllMyBucketsResult>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsioss/')
assert dir_contents == ['.']
gdal.VSICurlClearCache()
# List buckets
handler = webserver.SequentialHandler()
handler.add('GET', '/', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult>
<Buckets>
<Bucket>
<Name>mybucket</Name>
</Bucket>
</Buckets>
</ListAllMyBucketsResult>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsioss/')
assert dir_contents == ['mybucket']
def vsis3_3():
if gdaltest.webserver_port == 0:
return 'skip'
f = open_for_read('/vsis3/s3_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsis3/s3_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL(
'/vsis3/s3_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsis3/s3_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# Test CPL_VSIL_CURL_NON_CACHED
for config_option_value in [
'/vsis3/s3_non_cached/test.txt', '/vsis3/s3_non_cached',
'/vsis3/s3_non_cached:/vsis3/unrelated',
'/vsis3/unrelated:/vsis3/s3_non_cached',
'/vsis3/unrelated:/vsis3/s3_non_cached:/vsis3/unrelated'
]:
gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', config_option_value)
# So that the server knows we want it to serve initial content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_1' %
gdaltest.webserver_port)
f = open_for_read('/vsis3/s3_non_cached/test.txt')
if f is None:
gdaltest.post_reason('fail')
print(config_option_value)
return 'fail'
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
# So that the server knows we want it to serve other content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_2' %
gdaltest.webserver_port)
size = gdal.VSIStatL('/vsis3/s3_non_cached/test.txt').size
if size != 4:
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
# So that the server knows we want it to serve initial content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_1' %
gdaltest.webserver_port)
size = gdal.VSIStatL('/vsis3/s3_non_cached/test.txt').size
if size != 3:
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
# So that the server knows we want it to serve other content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_2' %
gdaltest.webserver_port)
f = open_for_read('/vsis3/s3_non_cached/test.txt')
if f is None:
gdaltest.post_reason('fail')
print(config_option_value)
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'bar2':
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', None)
# Retry without option
for config_option_value in [None, '/vsis3/s3_non_cached/bar.txt']:
gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', config_option_value)
# So that the server knows we want it to serve initial content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_1' %
gdaltest.webserver_port)
f = open_for_read('/vsis3/s3_non_cached/test.txt')
if f is None:
gdaltest.post_reason('fail')
print(config_option_value)
return 'fail'
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
# So that the server knows we want it to serve other content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_2' %
gdaltest.webserver_port)
f = open_for_read('/vsis3/s3_non_cached/test.txt')
if f is None:
gdaltest.post_reason('fail')
print(config_option_value)
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
# We should still get foo because of caching
if data != 'foo':
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', None)
return 'success'
def vsis3_2():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('AWS_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY')
gdal.SetConfigOption('AWS_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID')
gdal.SetConfigOption('AWS_TIMESTAMP', '20150101T000000Z')
gdal.SetConfigOption('AWS_HTTPS', 'NO')
gdal.SetConfigOption('AWS_VIRTUAL_HOSTING', 'NO')
gdal.SetConfigOption('AWS_S3_ENDPOINT',
'127.0.0.1:%d' % gdaltest.webserver_port)
f = open_for_read('/vsis3/s3_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
f = open_for_read('/vsis3_streaming/s3_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
# Test with temporary credentials
gdal.SetConfigOption('AWS_SESSION_TOKEN', 'AWS_SESSION_TOKEN')
f = open_for_read('/vsis3/s3_fake_bucket_with_session_token/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
gdal.SetConfigOption('AWS_SESSION_TOKEN', None)
#old_val = gdal.GetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN')
#gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR')
stat_res = gdal.VSIStatL('/vsis3/s3_fake_bucket/resource2.bin')
#gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', old_val)
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
stat_res = gdal.VSIStatL('/vsis3_streaming/s3_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
# Test region and endpoint 'redirects'
f = open_for_read('/vsis3/s3_fake_bucket/redirect')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
print(data)
return 'fail'
# Test region and endpoint 'redirects'
f = open_for_read('/vsis3_streaming/s3_fake_bucket/redirect')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/non_xml_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('bla') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/invalid_xml_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<oops>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_code_in_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error/>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read(
'/vsis3_streaming/s3_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error'
)
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read(
'/vsis3_streaming/s3_fake_bucket/no_endpoint_in_PermanentRedirect_error'
)
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read(
'/vsis3_streaming/s3_fake_bucket/no_message_in_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Test with requester pays
gdal.SetConfigOption('AWS_REQUEST_PAYER', 'requester')
f = open_for_read('/vsis3/s3_fake_bucket_with_requester_pays/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
gdal.SetConfigOption('AWS_REQUEST_PAYER', None)
if data != 'foo':
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsis3_2():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('AWS_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY')
gdal.SetConfigOption('AWS_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID')
gdal.SetConfigOption('AWS_TIMESTAMP', '20150101T000000Z')
gdal.SetConfigOption('AWS_HTTPS', 'NO')
gdal.SetConfigOption('AWS_VIRTUAL_HOSTING', 'NO')
gdal.SetConfigOption('AWS_S3_ENDPOINT', '127.0.0.1:%d' % gdaltest.webserver_port)
f = open_for_read('/vsis3/s3_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
f = open_for_read('/vsis3_streaming/s3_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
# Test with temporary credentials
gdal.SetConfigOption('AWS_SESSION_TOKEN', 'AWS_SESSION_TOKEN')
f = open_for_read('/vsis3/s3_fake_bucket_with_session_token/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
gdal.SetConfigOption('AWS_SESSION_TOKEN', None)
#old_val = gdal.GetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN')
#gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR')
stat_res = gdal.VSIStatL('/vsis3/s3_fake_bucket/resource2.bin')
#gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', old_val)
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
stat_res = gdal.VSIStatL('/vsis3_streaming/s3_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
# Test region and endpoint 'redirects'
f = open_for_read('/vsis3/s3_fake_bucket/redirect')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
print(data)
return 'fail'
# Test region and endpoint 'redirects'
f = open_for_read('/vsis3_streaming/s3_fake_bucket/redirect')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/non_xml_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('bla') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/invalid_xml_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<oops>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_code_in_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error/>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_endpoint_in_PermanentRedirect_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_message_in_error')
if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Test with requester pays
gdal.SetConfigOption('AWS_REQUEST_PAYER', 'requester')
f = open_for_read('/vsis3/s3_fake_bucket_with_requester_pays/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
gdal.SetConfigOption('AWS_REQUEST_PAYER', None)
if data != 'foo':
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsicurl_test_redirect():
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_redirect/', 404)
# Simulate a big time difference between server and local machine
current_time = 1500
def method(request):
response = 'HTTP/1.1 302\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n'
response += 'Location: %s\r\n' % ('http://localhost:%d/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (gdaltest.webserver_port, current_time + 30))
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add('HEAD', '/test_redirect/test.bin', custom_method=method)
handler.add('HEAD', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), 403,
{'Server': 'foo'}, '')
def method(request):
if 'Range' in request.headers:
if request.headers['Range'] == 'bytes=0-16383':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 0-16383/1000000')
request.send_header('Content-Length', 16384)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
elif request.headers['Range'] == 'bytes=16384-49151':
# Test expiration of the signed URL
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
else:
request.send_response(404)
request.send_header('Content-Length', 0)
request.end_headers()
else:
# After a failed attempt on a HEAD, the client should go there
response = 'HTTP/1.1 200\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n'
response += 'Content-type: text/plain\r\n'
response += 'Content-Length: 1000000\r\n'
response += 'Connection: close\r\n'
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add('GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsicurl/http://localhost:%d/test_redirect/test.bin' % gdaltest.webserver_port, 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFSeekL(f, 0, 2)
if gdal.VSIFTellL(f) != 1000000:
gdaltest.post_reason('fail')
print(gdal.VSIFTellL(f))
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFSeekL(f, 0, 0)
handler = webserver.SequentialHandler()
handler.add('GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method)
handler.add('GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method)
current_time = int(time.time())
def method(request):
# We should go there after expiration of the first signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(302)
# Return a new signed URL
request.send_header('Location', 'http://localhost:%d/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d' % (request.server.port, current_time + 30))
request.send_header('Content-Length', 16384)
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
handler.add('GET', '/test_redirect/test.bin', custom_method=method)
def method(request):
# Second signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 16384-16384/1000000')
request.send_header('Content-Length', 1)
request.end_headers()
request.wfile.write('y'.encode('ascii'))
handler.add('GET', '/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method)
with webserver.install_http_handler(handler):
content = gdal.VSIFReadL(1, 16383, f).decode('ascii')
if len(content) != 16383 or content[0] != 'x':
gdaltest.post_reason('fail')
print(content)
gdal.VSIFCloseL(f)
return 'fail'
content = gdal.VSIFReadL(1, 2, f).decode('ascii')
if content != 'xy':
gdaltest.post_reason('fail')
print(content)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
return 'success'
def test_vsicurl_test_redirect():
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_redirect/', 404)
# Simulate a big time difference between server and local machine
current_time = 1500
def method(request):
response = 'HTTP/1.1 302\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n'
response += 'Location: %s\r\n' % (
'http://localhost:%d/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (gdaltest.webserver_port, current_time + 30))
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add('HEAD', '/test_redirect/test.bin', custom_method=method)
handler.add(
'HEAD',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30), 403, {'Server': 'foo'}, '')
def method(request):
if 'Range' in request.headers:
if request.headers['Range'] == 'bytes=0-16383':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 0-16383/1000000')
request.send_header('Content-Length', 16384)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
elif request.headers['Range'] == 'bytes=16384-49151':
# Test expiration of the signed URL
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
else:
request.send_response(404)
request.send_header('Content-Length', 0)
request.end_headers()
else:
# After a failed attempt on a HEAD, the client should go there
response = 'HTTP/1.1 200\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime(
"%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(current_time)) + '\r\n'
response += 'Content-type: text/plain\r\n'
response += 'Content-Length: 1000000\r\n'
response += 'Connection: close\r\n'
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL(
'/vsicurl/http://localhost:%d/test_redirect/test.bin' %
gdaltest.webserver_port, 'rb')
assert f is not None
gdal.VSIFSeekL(f, 0, 2)
if gdal.VSIFTellL(f) != 1000000:
gdal.VSIFCloseL(f)
pytest.fail(gdal.VSIFTellL(f))
gdal.VSIFSeekL(f, 0, 0)
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
current_time = int(time.time())
def method(request):
# We should go there after expiration of the first signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(302)
# Return a new signed URL
request.send_header(
'Location',
'http://localhost:%d/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d'
% (request.server.port, current_time + 30))
request.send_header('Content-Length', 16384)
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
handler.add('GET', '/test_redirect/test.bin', custom_method=method)
def method(request):
# Second signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 16384-16384/1000000')
request.send_header('Content-Length', 1)
request.end_headers()
request.wfile.write('y'.encode('ascii'))
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
with webserver.install_http_handler(handler):
content = gdal.VSIFReadL(1, 16383, f).decode('ascii')
if len(content) != 16383 or content[0] != 'x':
gdal.VSIFCloseL(f)
pytest.fail(content)
content = gdal.VSIFReadL(1, 2, f).decode('ascii')
if content != 'xy':
gdal.VSIFCloseL(f)
pytest.fail(content)
gdal.VSIFCloseL(f)
