def check_no_file_leaks(): num_files = len(gdaltest.get_opened_files()) yield diff = len(gdaltest.get_opened_files()) - num_files if diff != 0 and (gdaltest.is_travis_branch('ubuntu_1804') or gdaltest.is_travis_branch('ubuntu_1804_32bit') or gdaltest.is_travis_branch('fedora')): print('Mysterious leak of file handle on some CI setups') return assert diff == 0, 'Leak of file handles: %d leaked' % diff
def vsigs_3(): if gdaltest.webserver_port == 0: return 'skip' handler = webserver.SequentialHandler() handler.add('GET', '/gs_fake_bucket2?delimiter=/&prefix=a_dir/', 200, { 'Content-type': 'application/xml' }, """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>a_dir/</Prefix> <NextMarker>bla</NextMarker> <Contents> <Key>a_dir/resource3.bin</Key> <LastModified>1970-01-01T00:00:01.000Z</LastModified> <Size>123456</Size> </Contents> </ListBucketResult> """) handler.add('GET', '/gs_fake_bucket2?delimiter=/&marker=bla&prefix=a_dir/', 200, { 'Content-type': 'application/xml' }, """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>a_dir/</Prefix> <Contents> <Key>a_dir/resource4.bin</Key> <LastModified>2015-10-16T12:34:56.000Z</LastModified> <Size>456789</Size> </Contents> <CommonPrefixes> <Prefix>a_dir/subdir/</Prefix> </CommonPrefixes> </ListBucketResult> """) with webserver.install_http_handler(handler): f = open_for_read('/vsigs/gs_fake_bucket2/a_dir/resource3.bin') if f is None: if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir') if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' if gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').size != 123456: gdaltest.post_reason('fail') return 'fail' if gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').mtime != 1: gdaltest.post_reason('fail') return 'fail' return 'success'
def runexternal_out_and_err(cmd, check_memleak=True): from gdaltest import is_travis_branch if not is_travis_branch('mingw'): has_subprocess = False try: import subprocess import shlex if hasattr(subprocess, 'Popen') and hasattr(shlex, 'split'): has_subprocess = True except (ImportError, AttributeError): pass if has_subprocess: return _runexternal_out_and_err_subprocess(cmd, check_memleak=check_memleak) (ret_stdin, ret_stdout, ret_stderr) = os.popen3(cmd) ret_stdin.close() q_stdout = Queue() t_stdout = Thread(target=read_in_thread, args=(ret_stdout, q_stdout)) q_stderr = Queue() t_stderr = Thread(target=read_in_thread, args=(ret_stderr, q_stderr)) t_stdout.start() t_stderr.start() out_str = q_stdout.get() err_str = q_stderr.get() if check_memleak: warn_if_memleak(cmd, out_str) return (out_str, err_str)
def postgisraster_test_outdb(): if gdaltest.postgisrasterDriver is None: return 'skip' # For some reason fails with # ERROR 1: PostGISRasterRasterBand::IRasterIO(): ERROR: rt_band_load_offline_data: Cannot open offline raster: /home/travis/build/rouault/gdal/autotest/gdrivers/data/small_world.tif # See https://api.travis-ci.org/v3/job/428972866/log.txt if gdaltest.is_travis_branch('ubuntu_1804'): return 'skip' ds = ogr.Open(gdaltest.postgisraster_connection_string_without_schema) sql_lyr = ds.ExecuteSQL('SHOW postgis.enable_outdb_rasters') has_guc = sql_lyr is not None ds.ReleaseResultSet(sql_lyr) ds = None ds = gdal.Open(gdaltest.postgisraster_connection_string + "table='small_world_outdb_constraint' mode=2") cs = [ds.GetRasterBand(i + 1).Checksum() for i in range(3)] expected_cs = [30111, 32302, 40026] if has_guc else [0, 0, 0] if cs != expected_cs: gdaltest.post_reason('fail') print(cs) return 'fail' return 'success'
def runexternal_out_and_err(cmd, check_memleak=True): from gdaltest import is_travis_branch if not is_travis_branch('mingw'): has_subprocess = False try: import subprocess import shlex if hasattr(subprocess, 'Popen') and hasattr(shlex, 'split'): has_subprocess = True except ImportError: pass if has_subprocess: return _runexternal_out_and_err_subprocess(cmd, check_memleak=check_memleak) (ret_stdin, ret_stdout, ret_stderr) = os.popen3(cmd) ret_stdin.close() q_stdout = Queue() t_stdout = Thread(target=read_in_thread, args=(ret_stdout, q_stdout)) q_stderr = Queue() t_stderr = Thread(target=read_in_thread, args=(ret_stderr, q_stderr)) t_stdout.start() t_stderr.start() out_str = q_stdout.get() err_str = q_stderr.get() if check_memleak: warn_if_memleak(cmd, out_str) return (out_str, err_str)
def vsigs_3(): if gdaltest.webserver_port == 0: return 'skip' f = open_for_read('/vsigs/gs_fake_bucket2/a_dir/resource3.bin') if f is None: if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir') if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' if gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').size != 123456: gdaltest.post_reason('fail') return 'fail' if gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').mtime != 1: gdaltest.post_reason('fail') return 'fail' return 'success'
def test_ogr_ogdi_1(): ogrtest.ogdi_ds = None # Skip tests when -fsanitize is used because of memleaks in libogdi if gdaltest.is_travis_branch('sanitize'): ogrtest.ogdi_drv = None pytest.skip('Skipping because of memory leaks in OGDI') ogrtest.ogdi_drv = ogr.GetDriverByName('OGDI') if ogrtest.ogdi_drv is None: pytest.skip() if not gdaltest.download_file( 'http://freefr.dl.sourceforge.net/project/ogdi/OGDI_Test_Suite/3.1/ogdits-3.1.0.zip', 'ogdits-3.1.0.zip'): pytest.skip() try: os.stat('tmp/cache/ogdits-3.1') except OSError: try: gdaltest.unzip('tmp/cache', 'tmp/cache/ogdits-3.1.0.zip') try: os.stat('tmp/cache/ogdits-3.1') except OSError: pytest.skip() except OSError: pytest.skip() url_name = 'gltp:/vrf/' + os.getcwd( ) + '/tmp/cache/ogdits-3.1/data/vpf/vm2alv2/texash' ds = ogr.Open(url_name) ogrtest.ogdi_ds = ds assert ds is not None, ('cannot open ' + url_name) assert ds.GetLayerCount() == 57, 'did not get expected layer count' layers = [('libref@libref(*)_line', ogr.wkbLineString, 15), ('libreft@libref(*)_text', ogr.wkbPoint, 4), ('markersp@bnd(*)_point', ogr.wkbPoint, 40), ('polbnda@bnd(*)_area', ogr.wkbPolygon, 6)] for l in layers: lyr = ds.GetLayerByName(l[0]) assert lyr.GetLayerDefn().GetGeomType() == l[1] assert lyr.GetFeatureCount() == l[2] # if l[1] != ogr.wkbNone: # if lyr.GetSpatialRef().ExportToWkt().find('WGS 84') == -1: # return 'fail' lyr = ds.GetLayerByName('libref@libref(*)_line') feat = lyr.GetNextFeature() wkt = 'LINESTRING (-97.570159912109375 31.242000579833984,-97.569938659667969 31.242116928100586,-97.562828063964844 31.245765686035156,-97.558868408203125 31.247797012329102,-97.555778503417969 31.249361038208008,-97.55413818359375 31.250171661376953)' ref_geom = ogr.CreateGeometryFromWkt(wkt) assert ogrtest.check_feature_geometry(feat, ref_geom) == 0, \ feat.GetGeometryRef().ExportToWkt()
def eedai_GOOGLE_APPLICATION_CREDENTIALS(): if gdaltest.eedai_drv is None: return 'skip' if gdaltest.is_travis_branch('gcc52_stdcpp14_sanitize'): return 'skip' gdal.FileFromMemBuffer( '/vsimem/my.json', """{ "private_key":"-----BEGIN PRIVATE KEY----- MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk\n VNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm\n i2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ\n DVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy\n TabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I\n 2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd\n DrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+\n bpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg\n Wz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3\n 4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5\n Kvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj\n j+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI\n yzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq\n gwE6fxOLyJDxuWRf\n -----END PRIVATE KEY-----", "client_email":"*****@*****.**" }""") gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/') gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', '/vsimem/my.json') gdal.SetConfigOption('EEDA_PRIVATE_KEY', None) gdal.SetConfigOption('EEDA_CLIENT_EMAIL', None) gdal.SetConfigOption('GO2A_AUD', '/vsimem/oauth2/v4/token') gdal.SetConfigOption('GOA2_NOW', '123456') gdal.FileFromMemBuffer( '/vsimem/oauth2/v4/token&POSTFIELDS=grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAibXlAZW1haWwuY29tIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZWFydGhlbmdpbmUucmVhZG9ubHkiLCAiYXVkIjogIi92c2ltZW0vb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.1W564xcQESVsqZmBEIMzj4rr0RuGa4RiUPZp5H%2FNENN9V9oPSTdacw%2BMiu3pcFf9AJv8wj0ajUeRsgTmvSicAftER49xeCQYUrs6uV122FGVsxml26kMFacNsCgRad%2Fy7xCAhMPfRJsqxS2%2BB392ssBeEzTGCSI6W3AsJg64OfA%3D', '{ "access_token": "my_token", "token_type": "Bearer", "expires_in": 3600 }' ) ds = gdal.Open('EEDAI:image') gdal.Unlink('/vsimem/my.json') gdal.SetConfigOption('EEDA_URL', None) gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', None) gdal.SetConfigOption('EEDA_PRIVATE_KEY', None) gdal.SetConfigOption('EEDA_CLIENT_EMAIL', None) if gdal.GetLastErrorMsg().find('CPLRSASHA256Sign() not implemented') >= 0: return 'skip' if ds is None: return 'fail' return 'success'
def test_ogr_ogdi_1(): ogrtest.ogdi_ds = None # Skip tests when -fsanitize is used because of memleaks in libogdi if gdaltest.is_travis_branch('sanitize'): ogrtest.ogdi_drv = None pytest.skip('Skipping because of memory leaks in OGDI') ogrtest.ogdi_drv = ogr.GetDriverByName('OGDI') if ogrtest.ogdi_drv is None: pytest.skip() if not gdaltest.download_file('http://freefr.dl.sourceforge.net/project/ogdi/OGDI_Test_Suite/3.1/ogdits-3.1.0.zip', 'ogdits-3.1.0.zip'): pytest.skip() try: os.stat('tmp/cache/ogdits-3.1') except OSError: try: gdaltest.unzip('tmp/cache', 'tmp/cache/ogdits-3.1.0.zip') try: os.stat('tmp/cache/ogdits-3.1') except OSError: pytest.skip() except OSError: pytest.skip() url_name = 'gltp:/vrf/' + os.getcwd() + '/tmp/cache/ogdits-3.1/data/vpf/vm2alv2/texash' ds = ogr.Open(url_name) ogrtest.ogdi_ds = ds assert ds is not None, ('cannot open ' + url_name) assert ds.GetLayerCount() == 57, 'did not get expected layer count' layers = [('libref@libref(*)_line', ogr.wkbLineString, 15), ('libreft@libref(*)_text', ogr.wkbPoint, 4), ('markersp@bnd(*)_point', ogr.wkbPoint, 40), ('polbnda@bnd(*)_area', ogr.wkbPolygon, 6)] for l in layers: lyr = ds.GetLayerByName(l[0]) assert lyr.GetLayerDefn().GetGeomType() == l[1] assert lyr.GetFeatureCount() == l[2] # if l[1] != ogr.wkbNone: # if lyr.GetSpatialRef().ExportToWkt().find('WGS 84') == -1: # return 'fail' lyr = ds.GetLayerByName('libref@libref(*)_line') feat = lyr.GetNextFeature() wkt = 'LINESTRING (-97.570159912109375 31.242000579833984,-97.569938659667969 31.242116928100586,-97.562828063964844 31.245765686035156,-97.558868408203125 31.247797012329102,-97.555778503417969 31.249361038208008,-97.55413818359375 31.250171661376953)' ref_geom = ogr.CreateGeometryFromWkt(wkt) assert ogrtest.check_feature_geometry(feat, ref_geom) == 0, \ feat.GetGeometryRef().ExportToWkt()
def test_vrt_shared_no_proxy_pool(): before = gdaltest.get_opened_files() vrt_text = """<VRTDataset rasterXSize="50" rasterYSize="50"> <VRTRasterBand dataType="Byte" band="1"> <ColorInterp>Red</ColorInterp> <SimpleSource> <SourceFilename>data/rgbsmall.tif</SourceFilename> <SourceBand>1</SourceBand> </SimpleSource> </VRTRasterBand> <VRTRasterBand dataType="Byte" band="2"> <ColorInterp>Green</ColorInterp> <SimpleSource> <SourceFilename>data/rgbsmall.tif</SourceFilename> <SourceBand>2</SourceBand> </SimpleSource> </VRTRasterBand> <VRTRasterBand dataType="Byte" band="3"> <ColorInterp>Blue</ColorInterp> <SimpleSource> <SourceFilename>data/rgbsmall.tif</SourceFilename> <SourceBand>3</SourceBand> </SimpleSource> </VRTRasterBand> </VRTDataset>""" ds = gdal.Open(vrt_text) assert ds assert ds.GetRasterBand(1).Checksum() == 21212 assert ds.GetRasterBand(2).Checksum() == 21053 assert ds.GetRasterBand(3).Checksum() == 21349 ds = None after = gdaltest.get_opened_files() if len(before) != len(after) and ( gdaltest.is_travis_branch('trusty_clang') or gdaltest.is_travis_branch('trusty_32bit') or gdaltest.is_travis_branch('ubuntu_1604')): pytest.xfail('Mysterious failure') assert len(before) == len(after)
def test_gdalinfo_13(): if test_cli_utilities.get_gdalinfo_path() is None: pytest.skip() ret = gdaltest.runexternal(test_cli_utilities.get_gdalinfo_path() + ' --license', check_memleak=False) ret = ret.replace('\r\n', '\n') if not ret.startswith(gdal.VersionInfo('LICENSE')): print(gdal.VersionInfo('LICENSE')) if gdaltest.is_travis_branch('mingw'): return 'expected_fail' pytest.fail(ret)
def test_ignfheightasciigrid_description_multiword_and_lf(): if gdaltest.is_travis_branch('mingw_w64'): pytest.skip() filename = '/vsimem/ignfheightasciigrid_invalid' ok_content = b'2 3 49 50 1 1 1 0 1 0 -0. MULTI WORD\xC3\xA9\xC3\xA8\n1 2 3 4' gdal.FileFromMemBuffer(filename, ok_content) ds = gdal.OpenEx(filename) desc = ds.GetMetadataItem('DESCRIPTION') assert desc == 'MULTI WORDee'
def test_gdalbuildvrt_16(): if test_cli_utilities.get_gdalbuildvrt_path() is None: pytest.skip() (out, err) = gdaltest.runexternal_out_and_err(test_cli_utilities.get_gdalbuildvrt_path() + ' /non_existing_dir/non_existing_subdir/out.vrt ../gcore/data/byte.tif') if not gdaltest.is_travis_branch('mingw'): assert 'ERROR ret code = 1' in err, out else: # We don't get the error code on Travis mingw assert 'ERROR' in err, out
def check_no_file_leaks(): num_files = len(gdaltest.get_opened_files()) yield diff = len(gdaltest.get_opened_files()) - num_files if diff != 0 and gdaltest.is_travis_branch('trusty_clang'): print('Mysterious leak of file handle on trusty_clang') return assert diff == 0, 'Leak of file handles: %d leaked' % diff
def eedai_GOOGLE_APPLICATION_CREDENTIALS(): if gdaltest.eedai_drv is None: return 'skip' if gdaltest.is_travis_branch('gcc52_stdcpp14_sanitize'): return 'skip' gdal.FileFromMemBuffer('/vsimem/my.json', """{ "private_key":"-----BEGIN PRIVATE KEY----- MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk\n VNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm\n i2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ\n DVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy\n TabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I\n 2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd\n DrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+\n bpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg\n Wz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3\n 4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5\n Kvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj\n j+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI\n yzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq\n gwE6fxOLyJDxuWRf\n -----END PRIVATE KEY-----", "client_email":"*****@*****.**" }""") gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/') gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', '/vsimem/my.json') gdal.SetConfigOption('EEDA_PRIVATE_KEY', None) gdal.SetConfigOption('EEDA_CLIENT_EMAIL', None) gdal.SetConfigOption('GO2A_AUD', '/vsimem/oauth2/v4/token') gdal.SetConfigOption('GOA2_NOW', '123456') gdal.FileFromMemBuffer('/vsimem/oauth2/v4/token&POSTFIELDS=grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAibXlAZW1haWwuY29tIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZWFydGhlbmdpbmUucmVhZG9ubHkiLCAiYXVkIjogIi92c2ltZW0vb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.1W564xcQESVsqZmBEIMzj4rr0RuGa4RiUPZp5H%2FNENN9V9oPSTdacw%2BMiu3pcFf9AJv8wj0ajUeRsgTmvSicAftER49xeCQYUrs6uV122FGVsxml26kMFacNsCgRad%2Fy7xCAhMPfRJsqxS2%2BB392ssBeEzTGCSI6W3AsJg64OfA%3D', '{ "access_token": "my_token", "token_type": "Bearer", "expires_in": 3600 }') ds = gdal.Open('EEDAI:image') gdal.Unlink('/vsimem/my.json') gdal.SetConfigOption('EEDA_URL', None) gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', None) gdal.SetConfigOption('EEDA_PRIVATE_KEY', None) gdal.SetConfigOption('EEDA_CLIENT_EMAIL', None) if gdal.GetLastErrorMsg().find('CPLRSASHA256Sign() not implemented') >= 0: return 'skip' if ds is None: return 'fail' return 'success'
def do_GET(self): if do_log: f = open('/tmp/log.txt', 'a') f.write('GET %s\n' % self.path) f.close() split = urllib.parse.urlparse(self.path) query = urllib.parse.parse_qs(split.query) query2 = {} for key in query: query2[key.lower()] = query[key] server = query2['server'][0] version = query2['version'][0] request = query2['request'][0] test = '' if 'test' in query2: test = query2['test'][0] if gdaltest.is_travis_branch('s390x') or gdaltest.is_travis_branch( 'graviton2') or gdaltest.is_travis_branch('ubuntu_2004'): # cannot strictly compare URL due to subtle difference of roundings # in BOUNDINGBOX computations. pass else: key = server + '-' + version if key in urls and test in urls[key]: _, got = self.path.split('SERVICE=WCS') got = re.sub(r'\&test=.*', '', got) _, have = urls[key][test].split('SERVICE=WCS') have += '&server=' + server if got == have: ok = 'ok' else: ok = "not ok\ngot: " + got + "\nhave: " + have global wcs_6_ok wcs_6_ok = False print('test ' + server + ' ' + test + ' WCS ' + version + ' ' + ok) self.Respond(request, server, version, test)
def test_postgisraster_test_outdb_client_side_if_possible(): if gdaltest.postgisrasterDriver is None: pytest.skip() # For some reason fails with # ERROR 1: PostGISRasterRasterBand::IRasterIO(): ERROR: rt_band_load_offline_data: Cannot open offline raster: /home/travis/build/rouault/gdal/autotest/gdrivers/data/small_world.tif # See https://api.travis-ci.org/v3/job/484385907/log.txt if gdaltest.is_travis_branch('ubuntu_1804'): pytest.skip() ds = gdal.Open(gdaltest.postgisraster_connection_string + "table='small_world_outdb_constraint' mode=2 outdb_resolution=client_side_if_possible") cs = [ds.GetRasterBand(i + 1).Checksum() for i in range(3)] assert cs == [30111, 32302, 40026]
def runexternal(cmd, strin=None, check_memleak=True, display_live_on_parent_stdout=False, encoding=None): from gdaltest import is_travis_branch if not is_travis_branch('mingw'): has_subprocess = False try: import subprocess import shlex subprocess.Popen shlex.split has_subprocess = True except: pass if has_subprocess: return _runexternal_subprocess( cmd, strin=strin, check_memleak=check_memleak, display_live_on_parent_stdout=display_live_on_parent_stdout, encoding=encoding) if strin is None: ret_stdout = os.popen(cmd) else: (ret_stdin, ret_stdout) = os.popen2(cmd) ret_stdin.write(strin) ret_stdin.close() if display_live_on_parent_stdout: out_str = '' while True: c = ret_stdout.read(1) if c == '': break out_str = out_str + c sys.stdout.write(c) ret_stdout.close() else: out_str = ret_stdout.read() ret_stdout.close() if check_memleak: warn_if_memleak(cmd, out_str) if encoding is not None: out_str = out_str.decode(encoding) return out_str
def test_gdalinfo_13(): if test_cli_utilities.get_gdalinfo_path() is None: return 'skip' ret = gdaltest.runexternal(test_cli_utilities.get_gdalinfo_path() + ' --license', check_memleak=False) ret = ret.replace('\r\n', '\n') if ret.find(gdal.VersionInfo('LICENSE')) != 0: print(ret) print(gdal.VersionInfo('LICENSE')) if gdaltest.is_travis_branch('mingw'): return 'expected_fail' return 'fail' return 'success'
def test_gdalsrsinfo_6(): if test_cli_utilities.get_gdalsrsinfo_path() is None: pytest.skip() if gdaltest.is_travis_branch('mingw'): pytest.skip() ret = gdaltest.runexternal(test_cli_utilities.get_gdalsrsinfo_path() + ' --single-line -o wkt_simple ../gcore/data/byte.tif') ret = ret.replace('\r\n', '\n') val = """PROJCS["NAD27 / UTM zone 11N",GEOGCS["NAD27",DATUM["North_American_Datum_1927",SPHEROID["Clarke 1866",6378206.4,294.978698213898]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",-117],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1]]""" assert ret.strip() == val
def test_gdalinfo_13(): if test_cli_utilities.get_gdalinfo_path() is None: return 'skip' ret = gdaltest.runexternal(test_cli_utilities.get_gdalinfo_path() + ' --license', check_memleak = False ) ret = ret.replace('\r\n', '\n') if ret.find(gdal.VersionInfo('LICENSE')) != 0: print(ret) print(gdal.VersionInfo('LICENSE')) if gdaltest.is_travis_branch('mingw'): return 'expected_fail' return 'fail' return 'success'
def test_ogrinfo_nogeomtype(): if test_cli_utilities.get_ogrinfo_path() is None: pytest.skip() (ret, err) = gdaltest.runexternal_out_and_err(test_cli_utilities.get_ogrinfo_path() + ' -nogeomtype ../ogr/data/poly.shp') assert (err is None or err == ''), 'got error/warning' expected_ret = """INFO: Open of `../ogr/data/poly.shp' using driver `ESRI Shapefile' successful. 1: poly """ expected_lines = expected_ret.splitlines() lines = ret.splitlines() for i, exp_line in enumerate(expected_lines): if exp_line != lines[i]: if gdaltest.is_travis_branch('mingw'): return 'expected_fail' pytest.fail(ret)
def runexternal(cmd, strin = None, check_memleak = True, display_live_on_parent_stdout = False, encoding = None): from gdaltest import is_travis_branch if not is_travis_branch('mingw'): has_subprocess = False try: import subprocess import shlex subprocess.Popen shlex.split has_subprocess = True except: pass if has_subprocess: return _runexternal_subprocess(cmd, strin=strin, check_memleak=check_memleak, display_live_on_parent_stdout=display_live_on_parent_stdout, encoding=encoding) if strin is None: ret_stdout = os.popen(cmd) else: (ret_stdin, ret_stdout) = os.popen2(cmd) ret_stdin.write(strin) ret_stdin.close() if display_live_on_parent_stdout: out_str = '' while True: c = ret_stdout.read(1) if c == '': break out_str = out_str + c sys.stdout.write(c) ret_stdout.close() else: out_str = ret_stdout.read() ret_stdout.close() if check_memleak: warn_if_memleak(cmd, out_str) if encoding is not None: out_str = out_str.decode(encoding) return out_str
def test_pixfun_mod_r(): filename = 'data/vrt/pixfun_mod_r.vrt' ds = gdal.OpenShared(filename, gdal.GA_ReadOnly) assert ds is not None, ('Unable to open "%s" dataset.' % filename) data = ds.GetRasterBand(1).ReadAsArray() reffilename = 'data/int32.tif' refds = gdal.Open(reffilename) assert refds is not None, ('Unable to open "%s" dataset.' % reffilename) refdata = refds.GetRasterBand(1).ReadAsArray() res = numpy.alltrue(data == numpy.abs(refdata)) if gdaltest.is_travis_branch('sanitize') and not res: print(data) print(numpy.abs(refdata)) pytest.xfail() assert res
def test_gdalbuildvrt_16(): if test_cli_utilities.get_gdalbuildvrt_path() is None: return 'skip' (out, err) = gdaltest.runexternal_out_and_err(test_cli_utilities.get_gdalbuildvrt_path() + ' /non_existing_dir/non_existing_subdir/out.vrt ../gcore/data/byte.tif') if not gdaltest.is_travis_branch('mingw'): if err.find('ERROR ret code = 1') < 0: gdaltest.post_reason('fail') print(out) print(err) return 'fail' else: # We don't get the error code on Travis mingw if err.find('ERROR') < 0: gdaltest.post_reason('fail') print(out) print(err) return 'fail' return 'success'
def vsicurl_test_redirect(): if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' if gdaltest.webserver_port == 0: return 'skip' f = gdal.VSIFOpenL( '/vsicurl/http://localhost:%d/test_redirect/test.bin' % gdaltest.webserver_port, 'rb') if f is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFSeekL(f, 0, 2) if gdal.VSIFTellL(f) != 1000000: gdaltest.post_reason('fail') print(gdal.VSIFTellL(f)) gdal.VSIFCloseL(f) return 'fail' gdal.VSIFSeekL(f, 0, 0) content = gdal.VSIFReadL(1, 16383, f).decode('ascii') if len(content) != 16383 or content[0] != 'x': gdaltest.post_reason('fail') print(content) gdal.VSIFCloseL(f) return 'fail' content = gdal.VSIFReadL(1, 2, f).decode('ascii') if content != 'xy': gdaltest.post_reason('fail') print(content) gdal.VSIFCloseL(f) return 'fail' gdal.VSIFCloseL(f) return 'success'
def vsicurl_test_redirect(): if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' if gdaltest.webserver_port == 0: return 'skip' f = gdal.VSIFOpenL('/vsicurl/http://localhost:%d/test_redirect/test.bin' % gdaltest.webserver_port, 'rb') if f is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFSeekL(f,0,2) if gdal.VSIFTellL(f) != 1000000: gdaltest.post_reason('fail') print(gdal.VSIFTellL(f)) gdal.VSIFCloseL(f) return 'fail' gdal.VSIFSeekL(f,0,0) content = gdal.VSIFReadL(1, 16383, f).decode('ascii') if len(content) != 16383 or content[0] != 'x': gdaltest.post_reason('fail') print(content) gdal.VSIFCloseL(f) return 'fail' content = gdal.VSIFReadL(1, 2, f).decode('ascii') if content != 'xy': gdaltest.post_reason('fail') print(content) gdal.VSIFCloseL(f) return 'fail' gdal.VSIFCloseL(f) return 'success'
def test_visoss_2(): if gdaltest.webserver_port == 0: pytest.skip() signed_url = gdal.GetSignedURL('/vsioss/oss_fake_bucket/resource', ['START_DATE=20180212T123456Z']) assert (signed_url in ( 'http://127.0.0.1:8080/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D', 'http://127.0.0.1:8081/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D' )) handler = webserver.SequentialHandler() handler.add('GET', '/oss_fake_bucket/resource', custom_method=get_oss_fake_bucket_resource_method) with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_fake_bucket/resource') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo' handler = webserver.SequentialHandler() handler.add('GET', '/oss_fake_bucket/resource', custom_method=get_oss_fake_bucket_resource_method) with webserver.install_http_handler(handler): f = open_for_read('/vsioss_streaming/oss_fake_bucket/resource') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo' handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(400) response = """<?xml version="1.0" encoding="UTF-8"?> <Error> <Code>AccessDenied</Code> <Message>The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.</Message> <HostId>unused</HostId> <Bucket>unuset</Bucket> <Endpoint>localhost:%d</Endpoint> </Error>""" % request.server.port response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) request.send_header('Content-type', 'application/xml') request.send_header('Transfer-Encoding', 'chunked') request.send_header('Connection', 'close') request.end_headers() request.wfile.write(response.encode('ascii')) handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method) def method(request): request.protocol_version = 'HTTP/1.1' if request.headers['Host'].startswith('localhost'): request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.send_header('Connection', 'close') request.end_headers() request.wfile.write("""foo""".encode('ascii')) else: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method) # Test region and endpoint 'redirects' with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_fake_bucket/redirect') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': if gdaltest.is_travis_branch('trusty'): pytest.skip('Skipped on trusty branch, but should be investigated') pytest.fail(data) # Test region and endpoint 'redirects' handler.req_count = 0 with webserver.install_http_handler(handler): f = open_for_read('/vsioss_streaming/oss_fake_bucket/redirect') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo' handler = webserver.SequentialHandler() def method(request): # /vsioss_streaming/ should have remembered the change of region and endpoint if not request.headers['Host'].startswith('localhost'): sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.protocol_version = 'HTTP/1.1' request.send_response(400) response = 'bla' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) request.send_header('Content-type', 'application/xml') request.send_header('Transfer-Encoding', 'chunked') request.send_header('Connection', 'close') request.end_headers() request.wfile.write(response.encode('ascii')) handler.add('GET', '/oss_fake_bucket/non_xml_error', custom_method=method) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read( '/vsioss_streaming/oss_fake_bucket/non_xml_error') assert f is None and gdal.VSIGetLastErrorMsg().find('bla') >= 0 handler = webserver.SequentialHandler() response = '<?xml version="1.0" encoding="UTF-8"?><oops>' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) handler.add( 'GET', '/oss_fake_bucket/invalid_xml_error', 400, { 'Content-type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Connection': 'close' }, response) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read( '/vsioss_streaming/oss_fake_bucket/invalid_xml_error') assert f is None and gdal.VSIGetLastErrorMsg().find('<oops>') >= 0 handler = webserver.SequentialHandler() response = '<?xml version="1.0" encoding="UTF-8"?><Error/>' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) handler.add( 'GET', '/oss_fake_bucket/no_code_in_error', 400, { 'Content-type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Connection': 'close' }, response) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read( '/vsioss_streaming/oss_fake_bucket/no_code_in_error') assert f is None and gdal.VSIGetLastErrorMsg().find('<Error/>') >= 0 handler = webserver.SequentialHandler() response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>AuthorizationHeaderMalformed</Code></Error>' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) handler.add( 'GET', '/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error', 400, { 'Content-type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Connection': 'close' }, response) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read( '/vsioss_streaming/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error' ) assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0 handler = webserver.SequentialHandler() response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>PermanentRedirect</Code></Error>' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) handler.add( 'GET', '/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error', 400, { 'Content-type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Connection': 'close' }, response) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read( '/vsioss_streaming/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error' ) assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0 handler = webserver.SequentialHandler() response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>bla</Code></Error>' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) handler.add( 'GET', '/oss_fake_bucket/no_message_in_error', 400, { 'Content-type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Connection': 'close' }, response) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read( '/vsioss_streaming/oss_fake_bucket/no_message_in_error') assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
def test_vsigs_readdir(): if gdaltest.webserver_port == 0: pytest.skip() handler = webserver.SequentialHandler() handler.add('GET', '/gs_fake_bucket2/?delimiter=%2F&prefix=a_dir%2F', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>a_dir/</Prefix> <NextMarker>bla</NextMarker> <Contents> <Key>a_dir/resource3.bin</Key> <LastModified>1970-01-01T00:00:01.000Z</LastModified> <Size>123456</Size> </Contents> </ListBucketResult> """) handler.add('GET', '/gs_fake_bucket2/?delimiter=%2F&marker=bla&prefix=a_dir%2F', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>a_dir/</Prefix> <Contents> <Key>a_dir/resource4.bin</Key> <LastModified>2015-10-16T12:34:56.000Z</LastModified> <Size>456789</Size> </Contents> <CommonPrefixes> <Prefix>a_dir/subdir/</Prefix> </CommonPrefixes> </ListBucketResult> """) with webserver.install_http_handler(handler): f = open_for_read('/vsigs/gs_fake_bucket2/a_dir/resource3.bin') if f is None: if gdaltest.is_travis_branch('trusty'): pytest.skip('Skipped on trusty branch, but should be investigated') pytest.fail() gdal.VSIFCloseL(f) dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir') assert dir_contents == ['resource3.bin', 'resource4.bin', 'subdir'] assert gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').size == 123456 assert gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').mtime == 1 # ReadDir on something known to be a file shouldn't cause network access dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir/resource3.bin') assert dir_contents is None # List buckets handler = webserver.SequentialHandler() handler.add('GET', '/', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListAllMyBucketsResult> <Buckets> <Bucket> <Name>mybucket</Name> </Bucket> </Buckets> </ListAllMyBucketsResult> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsigs/') assert dir_contents == ['mybucket']
def test_ogrinfo_24(): if test_cli_utilities.get_ogrinfo_path() is None: pytest.skip() f = open('tmp/test_ogrinfo_24.vrt', 'wt') f.write("""<OGRVRTDataSource> <Metadata> <MDI key="foo">bar</MDI> </Metadata> <Metadata domain="other_domain"> <MDI key="baz">foo</MDI> </Metadata> <OGRVRTLayer name="poly"> <Metadata> <MDI key="bar">baz</MDI> </Metadata> <SrcDataSource relativeToVRT="1" shared="1">../../ogr/data/poly.shp</SrcDataSource> <SrcLayer>poly</SrcLayer> </OGRVRTLayer> </OGRVRTDataSource>""") f.close() ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() + ' -ro -al tmp/test_ogrinfo_24.vrt -so', check_memleak=False) expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt' using driver `OGR_VRT' successful. Metadata: foo=bar Layer name: poly Metadata: bar=baz Geometry: Polygon Feature Count: 10 Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000) Layer SRS WKT: PROJCS["OSGB 1936 / British National Grid", GEOGCS["OSGB 1936", DATUM["OSGB_1936", SPHEROID["Airy 1830",6377563.396,299.3249646, AUTHORITY["EPSG","7001"]], AUTHORITY["EPSG","6277"]], PRIMEM["Greenwich",0, AUTHORITY["EPSG","8901"]], UNIT["degree",0.0174532925199433, AUTHORITY["EPSG","9122"]], AUTHORITY["EPSG","4277"]], PROJECTION["Transverse_Mercator"], PARAMETER["latitude_of_origin",49], PARAMETER["central_meridian",-2], PARAMETER["scale_factor",0.9996012717], PARAMETER["false_easting",400000], PARAMETER["false_northing",-100000], UNIT["metre",1, AUTHORITY["EPSG","9001"]], AXIS["Easting",EAST], AXIS["Northing",NORTH], AUTHORITY["EPSG","27700"]] """ expected_lines = expected_ret.splitlines() lines = ret.splitlines() for i, exp_line in enumerate(expected_lines): if exp_line != lines[i]: if gdaltest.is_travis_branch('mingw'): return 'expected_fail' pytest.fail(ret) ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() + ' -ro -al tmp/test_ogrinfo_24.vrt -so -mdd all', check_memleak=False) expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt' using driver `OGR_VRT' successful. Metadata: foo=bar Metadata (other_domain): baz=foo Layer name: poly Metadata: bar=baz Geometry: Polygon Feature Count: 10 Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000) Layer SRS WKT: PROJCS["OSGB 1936 / British National Grid", GEOGCS["OSGB 1936", DATUM["OSGB_1936", SPHEROID["Airy 1830",6377563.396,299.3249646, AUTHORITY["EPSG","7001"]], AUTHORITY["EPSG","6277"]], PRIMEM["Greenwich",0, AUTHORITY["EPSG","8901"]], UNIT["degree",0.0174532925199433, AUTHORITY["EPSG","9122"]], AUTHORITY["EPSG","4277"]], PROJECTION["Transverse_Mercator"], PARAMETER["latitude_of_origin",49], PARAMETER["central_meridian",-2], PARAMETER["scale_factor",0.9996012717], PARAMETER["false_easting",400000], PARAMETER["false_northing",-100000], UNIT["metre",1, AUTHORITY["EPSG","9001"]], AXIS["Easting",EAST], AXIS["Northing",NORTH], AUTHORITY["EPSG","27700"]] Data axis to CRS axis mapping: 1,2 AREA: Real (12.3) EAS_ID: Integer64 (11.0) PRFEDEA: String (16.0) """ expected_lines = expected_ret.splitlines() lines = ret.splitlines() for i, exp_line in enumerate(expected_lines): if exp_line != lines[i]: if gdaltest.is_travis_branch('mingw'): return 'expected_fail' pytest.fail(ret) ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() + ' -ro -al tmp/test_ogrinfo_24.vrt -so -nomd', check_memleak=False) expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt' using driver `OGR_VRT' successful. Layer name: poly Geometry: Polygon Feature Count: 10 Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000) Layer SRS WKT: PROJCS["OSGB 1936 / British National Grid", GEOGCS["OSGB 1936", DATUM["OSGB_1936", SPHEROID["Airy 1830",6377563.396,299.3249646, AUTHORITY["EPSG","7001"]], AUTHORITY["EPSG","6277"]], PRIMEM["Greenwich",0, AUTHORITY["EPSG","8901"]], UNIT["degree",0.0174532925199433, AUTHORITY["EPSG","9122"]], AUTHORITY["EPSG","4277"]], PROJECTION["Transverse_Mercator"], PARAMETER["latitude_of_origin",49], PARAMETER["central_meridian",-2], PARAMETER["scale_factor",0.9996012717], PARAMETER["false_easting",400000], PARAMETER["false_northing",-100000], UNIT["metre",1, AUTHORITY["EPSG","9001"]], AXIS["Easting",EAST], AXIS["Northing",NORTH], AUTHORITY["EPSG","27700"]] Data axis to CRS axis mapping: 1,2 AREA: Real (12.3) EAS_ID: Integer64 (11.0) PRFEDEA: String (16.0) """ expected_lines = expected_ret.splitlines() lines = ret.splitlines() for i, exp_line in enumerate(expected_lines): if exp_line != lines[i]: if gdaltest.is_travis_branch('mingw'): return 'expected_fail' pytest.fail(ret) os.unlink('tmp/test_ogrinfo_24.vrt')
def test_visoss_2(): if gdaltest.webserver_port == 0: pytest.skip() signed_url = gdal.GetSignedURL('/vsioss/oss_fake_bucket/resource', ['START_DATE=20180212T123456Z']) assert (signed_url in ('http://127.0.0.1:8080/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D', 'http://127.0.0.1:8081/oss_fake_bucket/resource?Expires=1518442496&OSSAccessKeyId=OSS_ACCESS_KEY_ID&Signature=bpFqur6tQMNN7Xe7UHVFFrugmgs%3D')) handler = webserver.SequentialHandler() handler.add('GET', '/oss_fake_bucket/resource', custom_method=get_oss_fake_bucket_resource_method) with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_fake_bucket/resource') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo' handler = webserver.SequentialHandler() handler.add('GET', '/oss_fake_bucket/resource', custom_method=get_oss_fake_bucket_resource_method) with webserver.install_http_handler(handler): f = open_for_read('/vsioss_streaming/oss_fake_bucket/resource') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo' handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(400) response = """<?xml version="1.0" encoding="UTF-8"?> <Error> <Code>AccessDenied</Code> <Message>The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.</Message> <HostId>unused</HostId> <Bucket>unuset</Bucket> <Endpoint>localhost:%d</Endpoint> </Error>""" % request.server.port response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) request.send_header('Content-type', 'application/xml') request.send_header('Transfer-Encoding', 'chunked') request.send_header('Connection', 'close') request.end_headers() request.wfile.write(response.encode('ascii')) handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method) def method(request): request.protocol_version = 'HTTP/1.1' if request.headers['Host'].startswith('localhost'): request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.send_header('Connection', 'close') request.end_headers() request.wfile.write("""foo""".encode('ascii')) else: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) handler.add('GET', '/oss_fake_bucket/redirect', custom_method=method) # Test region and endpoint 'redirects' with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_fake_bucket/redirect') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': if gdaltest.is_travis_branch('trusty'): pytest.skip('Skipped on trusty branch, but should be investigated') pytest.fail(data) # Test region and endpoint 'redirects' handler.req_count = 0 with webserver.install_http_handler(handler): f = open_for_read('/vsioss_streaming/oss_fake_bucket/redirect') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo' handler = webserver.SequentialHandler() def method(request): # /vsioss_streaming/ should have remembered the change of region and endpoint if not request.headers['Host'].startswith('localhost'): sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.protocol_version = 'HTTP/1.1' request.send_response(400) response = 'bla' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) request.send_header('Content-type', 'application/xml') request.send_header('Transfer-Encoding', 'chunked') request.send_header('Connection', 'close') request.end_headers() request.wfile.write(response.encode('ascii')) handler.add('GET', '/oss_fake_bucket/non_xml_error', custom_method=method) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read('/vsioss_streaming/oss_fake_bucket/non_xml_error') assert f is None and gdal.VSIGetLastErrorMsg().find('bla') >= 0 handler = webserver.SequentialHandler() response = '<?xml version="1.0" encoding="UTF-8"?><oops>' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) handler.add('GET', '/oss_fake_bucket/invalid_xml_error', 400, {'Content-type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Connection': 'close'}, response) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read('/vsioss_streaming/oss_fake_bucket/invalid_xml_error') assert f is None and gdal.VSIGetLastErrorMsg().find('<oops>') >= 0 handler = webserver.SequentialHandler() response = '<?xml version="1.0" encoding="UTF-8"?><Error/>' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) handler.add('GET', '/oss_fake_bucket/no_code_in_error', 400, {'Content-type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Connection': 'close'}, response) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read('/vsioss_streaming/oss_fake_bucket/no_code_in_error') assert f is None and gdal.VSIGetLastErrorMsg().find('<Error/>') >= 0 handler = webserver.SequentialHandler() response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>AuthorizationHeaderMalformed</Code></Error>' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) handler.add('GET', '/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error', 400, {'Content-type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Connection': 'close'}, response) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read('/vsioss_streaming/oss_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error') assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0 handler = webserver.SequentialHandler() response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>PermanentRedirect</Code></Error>' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) handler.add('GET', '/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error', 400, {'Content-type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Connection': 'close'}, response) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read('/vsioss_streaming/oss_fake_bucket/no_endpoint_in_PermanentRedirect_error') assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0 handler = webserver.SequentialHandler() response = '<?xml version="1.0" encoding="UTF-8"?><Error><Code>bla</Code></Error>' response = '%x\r\n%s\r\n0\r\n\r\n' % (len(response), response) handler.add('GET', '/oss_fake_bucket/no_message_in_error', 400, {'Content-type': 'application/xml', 'Transfer-Encoding': 'chunked', 'Connection': 'close'}, response) gdal.ErrorReset() with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = open_for_read('/vsioss_streaming/oss_fake_bucket/no_message_in_error') assert f is None and gdal.VSIGetLastErrorMsg().find('<Error>') >= 0
def vsiaz_fake_readdir(): if gdaltest.webserver_port == 0: return 'skip' handler = webserver.SequentialHandler() handler.add('GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%2F&restype=container', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Prefix>a_dir/</Prefix> <NextMarker>bla</NextMarker> <Blobs> <Blob> <Name>a_dir/resource3.bin</Name> <Properties> <Last-Modified>01 Jan 1970 00:00:01</Last-Modified> <Content-Length>123456</Content-Length> </Properties> </Blob> </Blobs> </EnumerationResults> """) handler.add('GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla&prefix=a_dir%2F&restype=container', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Prefix>a_dir/</Prefix> <Blobs> <Blob> <Name>a_dir/resource4.bin</Name> <Properties> <Last-Modified>16 Oct 2016 12:34:56</Last-Modified> <Content-Length>456789</Content-Length> </Properties> </Blob> <BlobPrefix> <Name>a_dir/subdir/</Name> </BlobPrefix> </Blobs> </EnumerationResults> """) with webserver.install_http_handler(handler): f = open_for_read('/vsiaz/az_fake_bucket2/a_dir/resource3.bin') if f is None: if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir') if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' if gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size != 123456: gdaltest.post_reason('fail') print(gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size) return 'fail' if gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').mtime != 1: gdaltest.post_reason('fail') return 'fail' # ReadDir on something known to be a file shouldn't cause network access dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir/resource3.bin') if dir_contents is not None: gdaltest.post_reason('fail') return 'fail' # Test error on ReadDir() handler = webserver.SequentialHandler() handler.add('GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=error_test%2F&restype=container', 500) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/error_test/') if dir_contents is not None: gdaltest.post_reason('fail') print(dir_contents) return 'fail' # List containers (empty result) handler = webserver.SequentialHandler() handler.add('GET', '/azure/blob/myaccount/?comp=list', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults ServiceEndpoint="https://myaccount.blob.core.windows.net"> <Containers/> </EnumerationResults> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiaz/') if dir_contents != ['.']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' gdal.VSICurlClearCache() # List containers handler = webserver.SequentialHandler() handler.add('GET', '/azure/blob/myaccount/?comp=list', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Containers> <Container> <Name>mycontainer1</Name> </Container> </Containers> <NextMarker>bla</NextMarker> </EnumerationResults> """) handler.add('GET', '/azure/blob/myaccount/?comp=list&marker=bla', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Containers> <Container> <Name>mycontainer2</Name> </Container> </Containers> </EnumerationResults> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiaz/') if dir_contents != ['mycontainer1', 'mycontainer2']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' return 'success'
def vsiaz_fake_readdir(): if gdaltest.webserver_port == 0: return 'skip' handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%2F&restype=container', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Prefix>a_dir/</Prefix> <NextMarker>bla</NextMarker> <Blobs> <Blob> <Name>a_dir/resource3.bin</Name> <Properties> <Last-Modified>01 Jan 1970 00:00:01</Last-Modified> <Content-Length>123456</Content-Length> </Properties> </Blob> </Blobs> </EnumerationResults> """) handler.add( 'GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla&prefix=a_dir%2F&restype=container', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Prefix>a_dir/</Prefix> <Blobs> <Blob> <Name>a_dir/resource4.bin</Name> <Properties> <Last-Modified>16 Oct 2016 12:34:56</Last-Modified> <Content-Length>456789</Content-Length> </Properties> </Blob> <BlobPrefix> <Name>a_dir/subdir/</Name> </BlobPrefix> </Blobs> </EnumerationResults> """) with webserver.install_http_handler(handler): f = open_for_read('/vsiaz/az_fake_bucket2/a_dir/resource3.bin') if f is None: if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir') if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' if gdal.VSIStatL( '/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size != 123456: gdaltest.post_reason('fail') print(gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size) return 'fail' if gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').mtime != 1: gdaltest.post_reason('fail') return 'fail' # ReadDir on something known to be a file shouldn't cause network access dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir/resource3.bin') if dir_contents is not None: gdaltest.post_reason('fail') return 'fail' # Test error on ReadDir() handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=error_test%2F&restype=container', 500) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/error_test/') if dir_contents is not None: gdaltest.post_reason('fail') print(dir_contents) return 'fail' # List containers (empty result) handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/?comp=list', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults ServiceEndpoint="https://myaccount.blob.core.windows.net"> <Containers/> </EnumerationResults> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiaz/') if dir_contents != ['.']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' gdal.VSICurlClearCache() # List containers handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/?comp=list', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Containers> <Container> <Name>mycontainer1</Name> </Container> </Containers> <NextMarker>bla</NextMarker> </EnumerationResults> """) handler.add( 'GET', '/azure/blob/myaccount/?comp=list&marker=bla', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Containers> <Container> <Name>mycontainer2</Name> </Container> </Containers> </EnumerationResults> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiaz/') if dir_contents != ['mycontainer1', 'mycontainer2']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' return 'success'
def test_ogrinfo_24(): if test_cli_utilities.get_ogrinfo_path() is None: pytest.skip() f = open('tmp/test_ogrinfo_24.vrt', 'wt') f.write("""<OGRVRTDataSource> <Metadata> <MDI key="foo">bar</MDI> </Metadata> <Metadata domain="other_domain"> <MDI key="baz">foo</MDI> </Metadata> <OGRVRTLayer name="poly"> <Metadata> <MDI key="bar">baz</MDI> </Metadata> <SrcDataSource relativeToVRT="1" shared="1">../../ogr/data/poly.shp</SrcDataSource> <SrcLayer>poly</SrcLayer> </OGRVRTLayer> </OGRVRTDataSource>""") f.close() ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() + ' -ro -al tmp/test_ogrinfo_24.vrt -so', check_memleak=False) expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt' using driver `OGR_VRT' successful. Metadata: foo=bar Layer name: poly Metadata: bar=baz Geometry: Polygon Feature Count: 10 Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000) Layer SRS WKT: PROJCS["OSGB 1936 / British National Grid", GEOGCS["OSGB 1936", DATUM["OSGB_1936", SPHEROID["Airy 1830",6377563.396,299.3249646, AUTHORITY["EPSG","7001"]], TOWGS84[446.448,-125.157,542.06,0.15,0.247,0.842,-20.489], AUTHORITY["EPSG","6277"]], PRIMEM["Greenwich",0, AUTHORITY["EPSG","8901"]], UNIT["degree",0.0174532925199433, AUTHORITY["EPSG","9122"]], AUTHORITY["EPSG","4277"]], PROJECTION["Transverse_Mercator"], PARAMETER["latitude_of_origin",49], PARAMETER["central_meridian",-2], PARAMETER["scale_factor",0.9996012717], PARAMETER["false_easting",400000], PARAMETER["false_northing",-100000], UNIT["metre",1, AUTHORITY["EPSG","9001"]], AXIS["Easting",EAST], AXIS["Northing",NORTH], AUTHORITY["EPSG","27700"]] """ expected_lines = expected_ret.splitlines() lines = ret.splitlines() for i, exp_line in enumerate(expected_lines): if exp_line != lines[i]: if gdaltest.is_travis_branch('mingw'): return 'expected_fail' pytest.fail(ret) ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() + ' -ro -al tmp/test_ogrinfo_24.vrt -so -mdd all', check_memleak=False) expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt' using driver `OGR_VRT' successful. Metadata: foo=bar Metadata (other_domain): baz=foo Layer name: poly Metadata: bar=baz Geometry: Polygon Feature Count: 10 Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000) Layer SRS WKT: PROJCS["OSGB 1936 / British National Grid", GEOGCS["OSGB 1936", DATUM["OSGB_1936", SPHEROID["Airy 1830",6377563.396,299.3249646, AUTHORITY["EPSG","7001"]], TOWGS84[446.448,-125.157,542.06,0.15,0.247,0.842,-20.489], AUTHORITY["EPSG","6277"]], PRIMEM["Greenwich",0, AUTHORITY["EPSG","8901"]], UNIT["degree",0.0174532925199433, AUTHORITY["EPSG","9122"]], AUTHORITY["EPSG","4277"]], PROJECTION["Transverse_Mercator"], PARAMETER["latitude_of_origin",49], PARAMETER["central_meridian",-2], PARAMETER["scale_factor",0.9996012717], PARAMETER["false_easting",400000], PARAMETER["false_northing",-100000], UNIT["metre",1, AUTHORITY["EPSG","9001"]], AXIS["Easting",EAST], AXIS["Northing",NORTH], AUTHORITY["EPSG","27700"]] AREA: Real (12.3) EAS_ID: Integer64 (11.0) PRFEDEA: String (16.0) """ expected_lines = expected_ret.splitlines() lines = ret.splitlines() for i, exp_line in enumerate(expected_lines): if exp_line != lines[i]: if gdaltest.is_travis_branch('mingw'): return 'expected_fail' pytest.fail(ret) ret = gdaltest.runexternal(test_cli_utilities.get_ogrinfo_path() + ' -ro -al tmp/test_ogrinfo_24.vrt -so -nomd', check_memleak=False) expected_ret = """INFO: Open of `tmp/test_ogrinfo_24.vrt' using driver `OGR_VRT' successful. Layer name: poly Geometry: Polygon Feature Count: 10 Extent: (478315.531250, 4762880.500000) - (481645.312500, 4765610.500000) Layer SRS WKT: PROJCS["OSGB 1936 / British National Grid", GEOGCS["OSGB 1936", DATUM["OSGB_1936", SPHEROID["Airy 1830",6377563.396,299.3249646, AUTHORITY["EPSG","7001"]], TOWGS84[446.448,-125.157,542.06,0.15,0.247,0.842,-20.489], AUTHORITY["EPSG","6277"]], PRIMEM["Greenwich",0, AUTHORITY["EPSG","8901"]], UNIT["degree",0.0174532925199433, AUTHORITY["EPSG","9122"]], AUTHORITY["EPSG","4277"]], PROJECTION["Transverse_Mercator"], PARAMETER["latitude_of_origin",49], PARAMETER["central_meridian",-2], PARAMETER["scale_factor",0.9996012717], PARAMETER["false_easting",400000], PARAMETER["false_northing",-100000], UNIT["metre",1, AUTHORITY["EPSG","9001"]], AXIS["Easting",EAST], AXIS["Northing",NORTH], AUTHORITY["EPSG","27700"]] AREA: Real (12.3) EAS_ID: Integer64 (11.0) PRFEDEA: String (16.0) """ expected_lines = expected_ret.splitlines() lines = ret.splitlines() for i, exp_line in enumerate(expected_lines): if exp_line != lines[i]: if gdaltest.is_travis_branch('mingw'): return 'expected_fail' pytest.fail(ret) os.unlink('tmp/test_ogrinfo_24.vrt')
def test_visoss_3(): if gdaltest.webserver_port == 0: pytest.skip() handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(200) request.send_header('Content-type', 'application/xml') response = """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>a_dir/</Prefix> <NextMarker>bla</NextMarker> <Contents> <Key>a_dir/resource3.bin</Key> <LastModified>1970-01-01T00:00:01.000Z</LastModified> <Size>123456</Size> </Contents> </ListBucketResult> """ request.send_header('Content-Length', len(response)) request.end_headers() request.wfile.write(response.encode('ascii')) handler.add('GET', '/oss_fake_bucket2/?delimiter=%2F&prefix=a_dir%2F', custom_method=method) def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(200) request.send_header('Content-type', 'application/xml') response = """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>a_dir/</Prefix> <Contents> <Key>a_dir/resource4.bin</Key> <LastModified>2015-10-16T12:34:56.000Z</LastModified> <Size>456789</Size> </Contents> <CommonPrefixes> <Prefix>a_dir/subdir/</Prefix> </CommonPrefixes> </ListBucketResult> """ request.send_header('Content-Length', len(response)) request.end_headers() request.wfile.write(response.encode('ascii')) handler.add('GET', '/oss_fake_bucket2/?delimiter=%2F&marker=bla&prefix=a_dir%2F', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_fake_bucket2/a_dir/resource3.bin') if f is None: if gdaltest.is_travis_branch('trusty'): pytest.skip('Skipped on trusty branch, but should be investigated') pytest.fail() gdal.VSIFCloseL(f) with webserver.install_http_handler(webserver.SequentialHandler()): dir_contents = gdal.ReadDir('/vsioss/oss_fake_bucket2/a_dir') assert dir_contents == ['resource3.bin', 'resource4.bin', 'subdir'] assert gdal.VSIStatL( '/vsioss/oss_fake_bucket2/a_dir/resource3.bin').size == 123456 assert gdal.VSIStatL( '/vsioss/oss_fake_bucket2/a_dir/resource3.bin').mtime == 1 # ReadDir on something known to be a file shouldn't cause network access dir_contents = gdal.ReadDir('/vsioss/oss_fake_bucket2/a_dir/resource3.bin') assert dir_contents is None # Test CPL_VSIL_CURL_NON_CACHED for config_option_value in [ '/vsioss/oss_non_cached/test.txt', '/vsioss/oss_non_cached', '/vsioss/oss_non_cached:/vsioss/unrelated', '/vsioss/unrelated:/vsioss/oss_non_cached', '/vsioss/unrelated:/vsioss/oss_non_cached:/vsioss/unrelated' ]: with gdaltest.config_option('CPL_VSIL_CURL_NON_CACHED', config_option_value): handler = webserver.SequentialHandler() handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo') with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_non_cached/test.txt') assert f is not None, config_option_value data = gdal.VSIFReadL(1, 3, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo', config_option_value handler = webserver.SequentialHandler() handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'bar2') with webserver.install_http_handler(handler): size = gdal.VSIStatL('/vsioss/oss_non_cached/test.txt').size assert size == 4, config_option_value handler = webserver.SequentialHandler() handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo') with webserver.install_http_handler(handler): size = gdal.VSIStatL('/vsioss/oss_non_cached/test.txt').size if size != 3: print(config_option_value) pytest.fail(data) handler = webserver.SequentialHandler() handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'bar2') with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_non_cached/test.txt') assert f is not None, config_option_value data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'bar2', config_option_value # Retry without option for config_option_value in [None, '/vsioss/oss_non_cached/bar.txt']: with gdaltest.config_option('CPL_VSIL_CURL_NON_CACHED', config_option_value): handler = webserver.SequentialHandler() if config_option_value is None: handler.add( 'GET', '/oss_non_cached/?delimiter=%2F', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>/</Prefix> <Contents> <Key>/test.txt</Key> <LastModified>1970-01-01T00:00:01.000Z</LastModified> <Size>40</Size> </Contents> <Contents> <Key>/test2.txt</Key> <LastModified>1970-01-01T00:00:01.000Z</LastModified> <Size>40</Size> </Contents> </ListBucketResult> """) handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo') with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_non_cached/test.txt') assert f is not None, config_option_value data = gdal.VSIFReadL(1, 3, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo', config_option_value handler = webserver.SequentialHandler() with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_non_cached/test.txt') assert f is not None, config_option_value data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) # We should still get foo because of caching assert data == 'foo', config_option_value # List buckets (empty result) handler = webserver.SequentialHandler() handler.add( 'GET', '/', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListAllMyBucketsResult> <Buckets> </Buckets> </ListAllMyBucketsResult> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsioss/') assert dir_contents == ['.'] gdal.VSICurlClearCache() # List buckets handler = webserver.SequentialHandler() handler.add( 'GET', '/', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListAllMyBucketsResult> <Buckets> <Bucket> <Name>mybucket</Name> </Bucket> </Buckets> </ListAllMyBucketsResult> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsioss/') assert dir_contents == ['mybucket']
def test_visoss_3(): if gdaltest.webserver_port == 0: pytest.skip() handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(200) request.send_header('Content-type', 'application/xml') response = """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>a_dir/</Prefix> <NextMarker>bla</NextMarker> <Contents> <Key>a_dir/resource3.bin</Key> <LastModified>1970-01-01T00:00:01.000Z</LastModified> <Size>123456</Size> </Contents> </ListBucketResult> """ request.send_header('Content-Length', len(response)) request.end_headers() request.wfile.write(response.encode('ascii')) handler.add('GET', '/oss_fake_bucket2/?delimiter=%2F&prefix=a_dir%2F', custom_method=method) def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(200) request.send_header('Content-type', 'application/xml') response = """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>a_dir/</Prefix> <Contents> <Key>a_dir/resource4.bin</Key> <LastModified>2015-10-16T12:34:56.000Z</LastModified> <Size>456789</Size> </Contents> <CommonPrefixes> <Prefix>a_dir/subdir/</Prefix> </CommonPrefixes> </ListBucketResult> """ request.send_header('Content-Length', len(response)) request.end_headers() request.wfile.write(response.encode('ascii')) handler.add('GET', '/oss_fake_bucket2/?delimiter=%2F&marker=bla&prefix=a_dir%2F', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_fake_bucket2/a_dir/resource3.bin') if f is None: if gdaltest.is_travis_branch('trusty'): pytest.skip('Skipped on trusty branch, but should be investigated') pytest.fail() gdal.VSIFCloseL(f) with webserver.install_http_handler(webserver.SequentialHandler()): dir_contents = gdal.ReadDir('/vsioss/oss_fake_bucket2/a_dir') assert dir_contents == ['resource3.bin', 'resource4.bin', 'subdir'] assert gdal.VSIStatL('/vsioss/oss_fake_bucket2/a_dir/resource3.bin').size == 123456 assert gdal.VSIStatL('/vsioss/oss_fake_bucket2/a_dir/resource3.bin').mtime == 1 # ReadDir on something known to be a file shouldn't cause network access dir_contents = gdal.ReadDir('/vsioss/oss_fake_bucket2/a_dir/resource3.bin') assert dir_contents is None # Test CPL_VSIL_CURL_NON_CACHED for config_option_value in ['/vsioss/oss_non_cached/test.txt', '/vsioss/oss_non_cached', '/vsioss/oss_non_cached:/vsioss/unrelated', '/vsioss/unrelated:/vsioss/oss_non_cached', '/vsioss/unrelated:/vsioss/oss_non_cached:/vsioss/unrelated']: with gdaltest.config_option('CPL_VSIL_CURL_NON_CACHED', config_option_value): handler = webserver.SequentialHandler() handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo') with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_non_cached/test.txt') assert f is not None, config_option_value data = gdal.VSIFReadL(1, 3, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo', config_option_value handler = webserver.SequentialHandler() handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'bar2') with webserver.install_http_handler(handler): size = gdal.VSIStatL('/vsioss/oss_non_cached/test.txt').size assert size == 4, config_option_value handler = webserver.SequentialHandler() handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo') with webserver.install_http_handler(handler): size = gdal.VSIStatL('/vsioss/oss_non_cached/test.txt').size if size != 3: print(config_option_value) pytest.fail(data) handler = webserver.SequentialHandler() handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'bar2') with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_non_cached/test.txt') assert f is not None, config_option_value data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'bar2', config_option_value # Retry without option for config_option_value in [None, '/vsioss/oss_non_cached/bar.txt']: with gdaltest.config_option('CPL_VSIL_CURL_NON_CACHED', config_option_value): handler = webserver.SequentialHandler() if config_option_value is None: handler.add('GET', '/oss_non_cached/?delimiter=%2F', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>/</Prefix> <Contents> <Key>/test.txt</Key> <LastModified>1970-01-01T00:00:01.000Z</LastModified> <Size>40</Size> </Contents> <Contents> <Key>/test2.txt</Key> <LastModified>1970-01-01T00:00:01.000Z</LastModified> <Size>40</Size> </Contents> </ListBucketResult> """) handler.add('GET', '/oss_non_cached/test.txt', 200, {}, 'foo') with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_non_cached/test.txt') assert f is not None, config_option_value data = gdal.VSIFReadL(1, 3, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo', config_option_value handler = webserver.SequentialHandler() with webserver.install_http_handler(handler): f = open_for_read('/vsioss/oss_non_cached/test.txt') assert f is not None, config_option_value data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) # We should still get foo because of caching assert data == 'foo', config_option_value # List buckets (empty result) handler = webserver.SequentialHandler() handler.add('GET', '/', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListAllMyBucketsResult> <Buckets> </Buckets> </ListAllMyBucketsResult> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsioss/') assert dir_contents == ['.'] gdal.VSICurlClearCache() # List buckets handler = webserver.SequentialHandler() handler.add('GET', '/', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListAllMyBucketsResult> <Buckets> <Bucket> <Name>mybucket</Name> </Bucket> </Buckets> </ListAllMyBucketsResult> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsioss/') assert dir_contents == ['mybucket']
def vsis3_3(): if gdaltest.webserver_port == 0: return 'skip' f = open_for_read('/vsis3/s3_fake_bucket2/a_dir/resource3.bin') if f is None: if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) dir_contents = gdal.ReadDir('/vsis3/s3_fake_bucket2/a_dir') if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' if gdal.VSIStatL( '/vsis3/s3_fake_bucket2/a_dir/resource3.bin').size != 123456: gdaltest.post_reason('fail') return 'fail' if gdal.VSIStatL('/vsis3/s3_fake_bucket2/a_dir/resource3.bin').mtime != 1: gdaltest.post_reason('fail') return 'fail' # Test CPL_VSIL_CURL_NON_CACHED for config_option_value in [ '/vsis3/s3_non_cached/test.txt', '/vsis3/s3_non_cached', '/vsis3/s3_non_cached:/vsis3/unrelated', '/vsis3/unrelated:/vsis3/s3_non_cached', '/vsis3/unrelated:/vsis3/s3_non_cached:/vsis3/unrelated' ]: gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', config_option_value) # So that the server knows we want it to serve initial content gdaltest.gdalurlopen( 'http://127.0.0.1:%d/s3_non_cached_test_use_content_1' % gdaltest.webserver_port) f = open_for_read('/vsis3/s3_non_cached/test.txt') if f is None: gdaltest.post_reason('fail') print(config_option_value) return 'fail' data = gdal.VSIFReadL(1, 3, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(config_option_value) print(data) return 'fail' # So that the server knows we want it to serve other content gdaltest.gdalurlopen( 'http://127.0.0.1:%d/s3_non_cached_test_use_content_2' % gdaltest.webserver_port) size = gdal.VSIStatL('/vsis3/s3_non_cached/test.txt').size if size != 4: gdaltest.post_reason('fail') print(config_option_value) print(data) return 'fail' # So that the server knows we want it to serve initial content gdaltest.gdalurlopen( 'http://127.0.0.1:%d/s3_non_cached_test_use_content_1' % gdaltest.webserver_port) size = gdal.VSIStatL('/vsis3/s3_non_cached/test.txt').size if size != 3: gdaltest.post_reason('fail') print(config_option_value) print(data) return 'fail' # So that the server knows we want it to serve other content gdaltest.gdalurlopen( 'http://127.0.0.1:%d/s3_non_cached_test_use_content_2' % gdaltest.webserver_port) f = open_for_read('/vsis3/s3_non_cached/test.txt') if f is None: gdaltest.post_reason('fail') print(config_option_value) return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'bar2': gdaltest.post_reason('fail') print(config_option_value) print(data) return 'fail' gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', None) # Retry without option for config_option_value in [None, '/vsis3/s3_non_cached/bar.txt']: gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', config_option_value) # So that the server knows we want it to serve initial content gdaltest.gdalurlopen( 'http://127.0.0.1:%d/s3_non_cached_test_use_content_1' % gdaltest.webserver_port) f = open_for_read('/vsis3/s3_non_cached/test.txt') if f is None: gdaltest.post_reason('fail') print(config_option_value) return 'fail' data = gdal.VSIFReadL(1, 3, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(config_option_value) print(data) return 'fail' # So that the server knows we want it to serve other content gdaltest.gdalurlopen( 'http://127.0.0.1:%d/s3_non_cached_test_use_content_2' % gdaltest.webserver_port) f = open_for_read('/vsis3/s3_non_cached/test.txt') if f is None: gdaltest.post_reason('fail') print(config_option_value) return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) # We should still get foo because of caching if data != 'foo': gdaltest.post_reason('fail') print(config_option_value) print(data) return 'fail' gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', None) return 'success'
def vsis3_2(): if gdaltest.webserver_port == 0: return 'skip' gdal.SetConfigOption('AWS_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY') gdal.SetConfigOption('AWS_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID') gdal.SetConfigOption('AWS_TIMESTAMP', '20150101T000000Z') gdal.SetConfigOption('AWS_HTTPS', 'NO') gdal.SetConfigOption('AWS_VIRTUAL_HOSTING', 'NO') gdal.SetConfigOption('AWS_S3_ENDPOINT', '127.0.0.1:%d' % gdaltest.webserver_port) f = open_for_read('/vsis3/s3_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' f = open_for_read('/vsis3_streaming/s3_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' # Test with temporary credentials gdal.SetConfigOption('AWS_SESSION_TOKEN', 'AWS_SESSION_TOKEN') f = open_for_read('/vsis3/s3_fake_bucket_with_session_token/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) gdal.SetConfigOption('AWS_SESSION_TOKEN', None) #old_val = gdal.GetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN') #gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR') stat_res = gdal.VSIStatL('/vsis3/s3_fake_bucket/resource2.bin') #gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', old_val) if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' stat_res = gdal.VSIStatL('/vsis3_streaming/s3_fake_bucket/resource2.bin') if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' # Test region and endpoint 'redirects' f = open_for_read('/vsis3/s3_fake_bucket/redirect') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' gdaltest.post_reason('fail') print(data) return 'fail' # Test region and endpoint 'redirects' f = open_for_read('/vsis3_streaming/s3_fake_bucket/redirect') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsis3_streaming/s3_fake_bucket/non_xml_error') if f is not None or gdal.VSIGetLastErrorMsg().find('bla') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsis3_streaming/s3_fake_bucket/invalid_xml_error') if f is not None or gdal.VSIGetLastErrorMsg().find('<oops>') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_code_in_error') if f is not None or gdal.VSIGetLastErrorMsg().find('<Error/>') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read( '/vsis3_streaming/s3_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error' ) if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read( '/vsis3_streaming/s3_fake_bucket/no_endpoint_in_PermanentRedirect_error' ) if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read( '/vsis3_streaming/s3_fake_bucket/no_message_in_error') if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' # Test with requester pays gdal.SetConfigOption('AWS_REQUEST_PAYER', 'requester') f = open_for_read('/vsis3/s3_fake_bucket_with_requester_pays/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 3, f).decode('ascii') gdal.VSIFCloseL(f) gdal.SetConfigOption('AWS_REQUEST_PAYER', None) if data != 'foo': gdaltest.post_reason('fail') return 'fail' return 'success'
def vsis3_2(): if gdaltest.webserver_port == 0: return 'skip' gdal.SetConfigOption('AWS_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY') gdal.SetConfigOption('AWS_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID') gdal.SetConfigOption('AWS_TIMESTAMP', '20150101T000000Z') gdal.SetConfigOption('AWS_HTTPS', 'NO') gdal.SetConfigOption('AWS_VIRTUAL_HOSTING', 'NO') gdal.SetConfigOption('AWS_S3_ENDPOINT', '127.0.0.1:%d' % gdaltest.webserver_port) f = open_for_read('/vsis3/s3_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' f = open_for_read('/vsis3_streaming/s3_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' # Test with temporary credentials gdal.SetConfigOption('AWS_SESSION_TOKEN', 'AWS_SESSION_TOKEN') f = open_for_read('/vsis3/s3_fake_bucket_with_session_token/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) gdal.SetConfigOption('AWS_SESSION_TOKEN', None) #old_val = gdal.GetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN') #gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR') stat_res = gdal.VSIStatL('/vsis3/s3_fake_bucket/resource2.bin') #gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', old_val) if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' stat_res = gdal.VSIStatL('/vsis3_streaming/s3_fake_bucket/resource2.bin') if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' # Test region and endpoint 'redirects' f = open_for_read('/vsis3/s3_fake_bucket/redirect') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' gdaltest.post_reason('fail') print(data) return 'fail' # Test region and endpoint 'redirects' f = open_for_read('/vsis3_streaming/s3_fake_bucket/redirect') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsis3_streaming/s3_fake_bucket/non_xml_error') if f is not None or gdal.VSIGetLastErrorMsg().find('bla') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsis3_streaming/s3_fake_bucket/invalid_xml_error') if f is not None or gdal.VSIGetLastErrorMsg().find('<oops>') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_code_in_error') if f is not None or gdal.VSIGetLastErrorMsg().find('<Error/>') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_region_in_AuthorizationHeaderMalformed_error') if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_endpoint_in_PermanentRedirect_error') if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsis3_streaming/s3_fake_bucket/no_message_in_error') if f is not None or gdal.VSIGetLastErrorMsg().find('<Error>') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' # Test with requester pays gdal.SetConfigOption('AWS_REQUEST_PAYER', 'requester') f = open_for_read('/vsis3/s3_fake_bucket_with_requester_pays/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 3, f).decode('ascii') gdal.VSIFCloseL(f) gdal.SetConfigOption('AWS_REQUEST_PAYER', None) if data != 'foo': gdaltest.post_reason('fail') return 'fail' return 'success'
def vsicurl_test_redirect(): if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' if gdaltest.webserver_port == 0: return 'skip' gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add('GET', '/test_redirect/', 404) # Simulate a big time difference between server and local machine current_time = 1500 def method(request): response = 'HTTP/1.1 302\r\n' response += 'Server: foo\r\n' response += 'Date: ' + time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n' response += 'Location: %s\r\n' % ('http://localhost:%d/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (gdaltest.webserver_port, current_time + 30)) response += '\r\n' request.wfile.write(response.encode('ascii')) handler.add('HEAD', '/test_redirect/test.bin', custom_method=method) handler.add('HEAD', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), 403, {'Server': 'foo'}, '') def method(request): if 'Range' in request.headers: if request.headers['Range'] == 'bytes=0-16383': request.protocol_version = 'HTTP/1.1' request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Range', 'bytes 0-16383/1000000') request.send_header('Content-Length', 16384) request.send_header('Connection', 'close') request.end_headers() request.wfile.write(('x' * 16384).encode('ascii')) elif request.headers['Range'] == 'bytes=16384-49151': # Test expiration of the signed URL request.protocol_version = 'HTTP/1.1' request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() else: request.send_response(404) request.send_header('Content-Length', 0) request.end_headers() else: # After a failed attempt on a HEAD, the client should go there response = 'HTTP/1.1 200\r\n' response += 'Server: foo\r\n' response += 'Date: ' + time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n' response += 'Content-type: text/plain\r\n' response += 'Content-Length: 1000000\r\n' response += 'Connection: close\r\n' response += '\r\n' request.wfile.write(response.encode('ascii')) handler.add('GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) with webserver.install_http_handler(handler): f = gdal.VSIFOpenL('/vsicurl/http://localhost:%d/test_redirect/test.bin' % gdaltest.webserver_port, 'rb') if f is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFSeekL(f, 0, 2) if gdal.VSIFTellL(f) != 1000000: gdaltest.post_reason('fail') print(gdal.VSIFTellL(f)) gdal.VSIFCloseL(f) return 'fail' gdal.VSIFSeekL(f, 0, 0) handler = webserver.SequentialHandler() handler.add('GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) handler.add('GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) current_time = int(time.time()) def method(request): # We should go there after expiration of the first signed URL if 'Range' in request.headers and \ request.headers['Range'] == 'bytes=16384-49151': request.protocol_version = 'HTTP/1.1' request.send_response(302) # Return a new signed URL request.send_header('Location', 'http://localhost:%d/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d' % (request.server.port, current_time + 30)) request.send_header('Content-Length', 16384) request.end_headers() request.wfile.write(('x' * 16384).encode('ascii')) handler.add('GET', '/test_redirect/test.bin', custom_method=method) def method(request): # Second signed URL if 'Range' in request.headers and \ request.headers['Range'] == 'bytes=16384-49151': request.protocol_version = 'HTTP/1.1' request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Range', 'bytes 16384-16384/1000000') request.send_header('Content-Length', 1) request.end_headers() request.wfile.write('y'.encode('ascii')) handler.add('GET', '/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) with webserver.install_http_handler(handler): content = gdal.VSIFReadL(1, 16383, f).decode('ascii') if len(content) != 16383 or content[0] != 'x': gdaltest.post_reason('fail') print(content) gdal.VSIFCloseL(f) return 'fail' content = gdal.VSIFReadL(1, 2, f).decode('ascii') if content != 'xy': gdaltest.post_reason('fail') print(content) gdal.VSIFCloseL(f) return 'fail' gdal.VSIFCloseL(f) return 'success'
def test_vsicurl_test_redirect(): if gdaltest.is_travis_branch('trusty'): pytest.skip('Skipped on trusty branch, but should be investigated') if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add('GET', '/test_redirect/', 404) # Simulate a big time difference between server and local machine current_time = 1500 def method(request): response = 'HTTP/1.1 302\r\n' response += 'Server: foo\r\n' response += 'Date: ' + time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n' response += 'Location: %s\r\n' % ( 'http://localhost:%d/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (gdaltest.webserver_port, current_time + 30)) response += '\r\n' request.wfile.write(response.encode('ascii')) handler.add('HEAD', '/test_redirect/test.bin', custom_method=method) handler.add( 'HEAD', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), 403, {'Server': 'foo'}, '') def method(request): if 'Range' in request.headers: if request.headers['Range'] == 'bytes=0-16383': request.protocol_version = 'HTTP/1.1' request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Range', 'bytes 0-16383/1000000') request.send_header('Content-Length', 16384) request.send_header('Connection', 'close') request.end_headers() request.wfile.write(('x' * 16384).encode('ascii')) elif request.headers['Range'] == 'bytes=16384-49151': # Test expiration of the signed URL request.protocol_version = 'HTTP/1.1' request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() else: request.send_response(404) request.send_header('Content-Length', 0) request.end_headers() else: # After a failed attempt on a HEAD, the client should go there response = 'HTTP/1.1 200\r\n' response += 'Server: foo\r\n' response += 'Date: ' + time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n' response += 'Content-type: text/plain\r\n' response += 'Content-Length: 1000000\r\n' response += 'Connection: close\r\n' response += '\r\n' request.wfile.write(response.encode('ascii')) handler.add( 'GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) with webserver.install_http_handler(handler): f = gdal.VSIFOpenL( '/vsicurl/http://localhost:%d/test_redirect/test.bin' % gdaltest.webserver_port, 'rb') assert f is not None gdal.VSIFSeekL(f, 0, 2) if gdal.VSIFTellL(f) != 1000000: gdal.VSIFCloseL(f) pytest.fail(gdal.VSIFTellL(f)) gdal.VSIFSeekL(f, 0, 0) handler = webserver.SequentialHandler() handler.add( 'GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) handler.add( 'GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) current_time = int(time.time()) def method(request): # We should go there after expiration of the first signed URL if 'Range' in request.headers and \ request.headers['Range'] == 'bytes=16384-49151': request.protocol_version = 'HTTP/1.1' request.send_response(302) # Return a new signed URL request.send_header( 'Location', 'http://localhost:%d/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d' % (request.server.port, current_time + 30)) request.send_header('Content-Length', 16384) request.end_headers() request.wfile.write(('x' * 16384).encode('ascii')) handler.add('GET', '/test_redirect/test.bin', custom_method=method) def method(request): # Second signed URL if 'Range' in request.headers and \ request.headers['Range'] == 'bytes=16384-49151': request.protocol_version = 'HTTP/1.1' request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Range', 'bytes 16384-16384/1000000') request.send_header('Content-Length', 1) request.end_headers() request.wfile.write('y'.encode('ascii')) handler.add( 'GET', '/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) with webserver.install_http_handler(handler): content = gdal.VSIFReadL(1, 16383, f).decode('ascii') if len(content) != 16383 or content[0] != 'x': gdal.VSIFCloseL(f) pytest.fail(content) content = gdal.VSIFReadL(1, 2, f).decode('ascii') if content != 'xy': gdal.VSIFCloseL(f) pytest.fail(content) gdal.VSIFCloseL(f)