def test_vsiaz_real_server_errors(): if not gdaltest.built_against_curl(): pytest.skip() # Missing AZURE_STORAGE_ACCOUNT gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsiaz/foo/bar') assert f is None and gdal.VSIGetLastErrorMsg().find( 'AZURE_STORAGE_ACCOUNT') >= 0 gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsiaz_streaming/foo/bar') assert f is None and gdal.VSIGetLastErrorMsg().find( 'AZURE_STORAGE_ACCOUNT') >= 0 # Invalid AZURE_STORAGE_CONNECTION_STRING with gdaltest.config_option('AZURE_STORAGE_CONNECTION_STRING', 'invalid'): gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsiaz/foo/bar') assert f is None # Missing AZURE_STORAGE_ACCESS_KEY gdal.ErrorReset() with gdaltest.config_options({ 'AZURE_STORAGE_ACCOUNT': 'AZURE_STORAGE_ACCOUNT', 'CPL_AZURE_VM_API_ROOT_URL': 'disabled' }): with gdaltest.error_handler(): f = open_for_read('/vsiaz/foo/bar') assert f is None and gdal.VSIGetLastErrorMsg().find( 'AZURE_STORAGE_ACCESS_KEY') >= 0 # AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_ACCESS_KEY but invalid gdal.ErrorReset() with gdaltest.config_options({ 'AZURE_STORAGE_ACCOUNT': 'AZURE_STORAGE_ACCOUNT', 'AZURE_STORAGE_ACCESS_KEY': 'AZURE_STORAGE_ACCESS_KEY' }): with gdaltest.error_handler(): f = open_for_read('/vsiaz/foo/bar.baz') if f is not None: if f is not None: gdal.VSIFCloseL(f) if gdal.GetConfigOption('APPVEYOR') is not None: return pytest.fail(gdal.VSIGetLastErrorMsg()) gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsiaz_streaming/foo/bar.baz') assert f is None, gdal.VSIGetLastErrorMsg()
def test_vsizip_multi_thread(): with gdaltest.config_options({ 'GDAL_NUM_THREADS': 'ALL_CPUS', 'CPL_VSIL_DEFLATE_CHUNK_SIZE': '32K' }): fmain = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip', 'wb') f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'wb') for i in range(100000): gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(fmain) f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'rb') data = gdal.VSIFReadL(100000, 5, f).decode('ascii') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/vsizip_multi_thread.zip') if data != 'hello' * 100000: for i in range(10000): if data[i * 5:i * 5 + 5] != 'hello': print(i * 5, data[i * 5:i * 5 + 5], data[i * 5 - 5:i * 5 + 5 - 5]) break pytest.fail()
def test_vsiaz_write_appendblob_retry(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() with gdaltest.config_options({ 'GDAL_HTTP_MAX_RETRY': '2', 'GDAL_HTTP_RETRY_DELAY': '0.01', 'VSIAZ_CHUNK_SIZE_BYTES': '10' }): f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 502) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201) handler.add( 'PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 502) handler.add( 'PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 201) handler.add( 'PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 502) handler.add( 'PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 201) with webserver.install_http_handler(handler): assert gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) == 16 gdal.VSIFCloseL(f)
def test_vsizip_create_zip64(): if not gdaltest.run_slow_tests(): pytest.skip() niters = 1000 s = 'hello' * 1000 * 1000 zip_name = '/vsimem/vsizip_create_zip64.zip' with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): fmain = gdal.VSIFOpenL('/vsizip/' + zip_name, 'wb') f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test', 'wb') for i in range(niters): gdal.VSIFWriteL(s, 1, len(s), f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(fmain) size = gdal.VSIStatL(zip_name).size assert size <= 0xFFFFFFFF size = gdal.VSIStatL('/vsizip/' + zip_name + '/test').size assert size == len(s) * niters f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test', 'rb') data = gdal.VSIFReadL(1, len(s), f).decode('ascii') gdal.VSIFCloseL(f) assert data == s gdal.Unlink(zip_name)
def test_vsizip_create_zip64_stream_larger_than_4G(): if not gdaltest.run_slow_tests(): pytest.skip() zip_name = 'tmp/vsizip_create_zip64_stream_larger_than_4G.zip' gdal.Unlink(zip_name) niters = 999 s = ''.join([chr(random.randint(0, 127)) for i in range(5 * 1000 * 1000)]) with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test2', 'wb') for i in range(niters): gdal.VSIFWriteL(s, 1, len(s), f) gdal.VSIFCloseL(f) size = gdal.VSIStatL(zip_name).size assert size > 0xFFFFFFFF size = gdal.VSIStatL('/vsizip/' + zip_name + '/test2').size assert size == len(s) * niters f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test2', 'rb') data = gdal.VSIFReadL(1, len(s), f).decode('ascii') gdal.VSIFCloseL(f) assert data == s gdal.Unlink(zip_name)
def test_vsiaz_write_appendblob_retry(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() with gdaltest.config_options({'GDAL_HTTP_MAX_RETRY': '2', 'GDAL_HTTP_RETRY_DELAY': '0.01', 'VSIAZ_CHUNK_SIZE_BYTES': '10'}): f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 502) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 502) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 201) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 502) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 201) with webserver.install_http_handler(handler): assert gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) == 16 gdal.VSIFCloseL(f)
def test_vsiaz_write_blockblob_retry(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() # Test creation of BlockBob f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None with gdaltest.config_options({'GDAL_HTTP_MAX_RETRY': '2', 'GDAL_HTTP_RETRY_DELAY': '0.01'}): handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii')) content = request.rfile.read(3).decode('ascii') if len(content) != 3: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 502) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) with webserver.install_http_handler(handler): assert gdal.VSIFWriteL('foo', 1, 3, f) == 3 gdal.VSIFCloseL(f)
def test_vsizip_create_zip64(): if not gdaltest.run_slow_tests(): pytest.skip() niters = 1000 s = 'hello' * 1000 * 1000 zip_name = '/vsimem/vsizip_create_zip64.zip' with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): fmain = gdal.VSIFOpenL('/vsizip/' + zip_name, 'wb') f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test', 'wb') for i in range(niters): gdal.VSIFWriteL(s, 1, len(s), f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(fmain) size = gdal.VSIStatL(zip_name).size assert size <= 0xFFFFFFFF size = gdal.VSIStatL('/vsizip/' + zip_name + '/test').size assert size == len(s) * niters f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test', 'rb') data = gdal.VSIFReadL(1, len(s), f).decode('ascii') gdal.VSIFCloseL(f) assert data == s gdal.Unlink(zip_name)
def test_vsiaz_write_blockblob_retry(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() # Test creation of BlockBob f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None with gdaltest.config_options({'GDAL_HTTP_MAX_RETRY': '2', 'GDAL_HTTP_RETRY_DELAY': '0.01'}): handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii')) content = request.rfile.read(3).decode('ascii') if len(content) != 3: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 502) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) with webserver.install_http_handler(handler): assert gdal.VSIFWriteL('foo', 1, 3, f) == 3 gdal.VSIFCloseL(f)
def test_vsizip_create_zip64_stream_larger_than_4G(): if not gdaltest.run_slow_tests(): pytest.skip() zip_name = 'tmp/vsizip_create_zip64_stream_larger_than_4G.zip' gdal.Unlink(zip_name) niters = 999 s = ''.join([chr(random.randint(0, 127)) for i in range(5 * 1000 * 1000)]) with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test2', 'wb') for i in range(niters): gdal.VSIFWriteL(s, 1, len(s), f) gdal.VSIFCloseL(f) size = gdal.VSIStatL(zip_name).size assert size > 0xFFFFFFFF size = gdal.VSIStatL('/vsizip/' + zip_name + '/test2').size assert size == len(s) * niters f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test2', 'rb') data = gdal.VSIFReadL(1, len(s), f).decode('ascii') gdal.VSIFCloseL(f) assert data == s gdal.Unlink(zip_name)
def test_vsiaz_no_sign_request(): if not gdaltest.built_against_curl(): pytest.skip() with gdaltest.config_options({ 'AZURE_STORAGE_ACCOUNT': 'naipblobs', 'AZURE_NO_SIGN_REQUEST': 'YES' }): actual_url = gdal.GetActualURL( '/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif' ) assert actual_url == 'https://naipblobs.blob.core.windows.net/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif' assert actual_url == gdal.GetSignedURL( '/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif' ) f = open_for_read( '/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif' ) if f is None: if gdaltest.gdalurlopen( 'https://naipblobs.blob.core.windows.net/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif' ) is None: pytest.skip('cannot open URL') pytest.fail() gdal.VSIFCloseL(f) assert 'm_3008601_ne_16_1_20150804.tif' in gdal.ReadDir( '/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/')
def test_vsiaz_sas(): if not gdaltest.built_against_curl(): pytest.skip() # See https://azure.microsoft.com/en-us/services/open-datasets/catalog/naip/ for the value of AZURE_SAS with gdaltest.config_options({ 'AZURE_STORAGE_ACCOUNT': 'naipblobs', 'AZURE_SAS': 'st=2019-07-18T03%3A53%3A22Z&se=2035-07-19T03%3A53%3A00Z&sp=rl&sv=2018-03-28&sr=c&sig=2RIXmLbLbiagYnUd49rgx2kOXKyILrJOgafmkODhRAQ%3D' }): actual_url = gdal.GetActualURL( '/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif' ) assert actual_url == 'https://naipblobs.blob.core.windows.net/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif' assert gdal.GetSignedURL( '/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif' ) == 'https://naipblobs.blob.core.windows.net/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif?st=2019-07-18T03%3A53%3A22Z&se=2035-07-19T03%3A53%3A00Z&sp=rl&sv=2018-03-28&sr=c&sig=2RIXmLbLbiagYnUd49rgx2kOXKyILrJOgafmkODhRAQ%3D' f = open_for_read( '/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif' ) if f is None: if gdaltest.gdalurlopen( 'https://naipblobs.blob.core.windows.net/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif' ) is None: pytest.skip('cannot open URL') pytest.fail() gdal.VSIFCloseL(f) assert 'm_3008601_ne_16_1_20150804.tif' in gdal.ReadDir( '/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/')
def vsiwebhdfs_unlink(): if gdaltest.webserver_port == 0: return 'skip' gdal.VSICurlClearCache() # Success handler = webserver.SequentialHandler() handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE', 200, {}, '{"boolean":true}') with webserver.install_http_handler(handler): ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar') if ret != 0: gdaltest.post_reason('fail') return 'fail' gdal.VSICurlClearCache() # With permissions gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE&user.name=root&delegation=token', 200, {}, '{"boolean":true}') with gdaltest.config_options({'WEBHDFS_USERNAME': '******', 'WEBHDFS_DELEGATION': 'token'}): with webserver.install_http_handler(handler): ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar') if ret != 0: gdaltest.post_reason('fail') return 'fail' # Failure handler = webserver.SequentialHandler() handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE', 200, {}, '{"boolean":false}') with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar') if ret != -1: gdaltest.post_reason('fail') return 'fail' gdal.VSICurlClearCache() # Failure handler = webserver.SequentialHandler() handler.add('DELETE', '/webhdfs/v1/foo/bar?op=DELETE', 404, {}) with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.Unlink(gdaltest.webhdfs_base_connection + '/foo/bar') if ret != -1: gdaltest.post_reason('fail') return 'fail' return 'success'
def vsigzip_multi_thread(): with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS', 'CPL_VSIL_DEFLATE_CHUNK_SIZE': '32K'}): f = gdal.VSIFOpenL('/vsigzip//vsimem/vsigzip_multi_thread.gz', 'wb') for i in range(100000): gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) f = gdal.VSIFOpenL('/vsigzip//vsimem/vsigzip_multi_thread.gz', 'rb') data = gdal.VSIFReadL(100000, 5, f).decode('ascii') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/vsigzip_multi_thread.gz') if data != 'hello' * 100000: gdaltest.post_reason('fail') for i in range(10000): if data[i*5:i*5+5] != 'hello': print(i*5, data[i*5:i*5+5], data[i*5-5:i*5+5-5]) break return 'fail' return 'success'
def test_ogr_dwg_4(): with gdaltest.config_options({ 'DWG_INLINE_BLOCKS': 'FALSE', 'DWG_ATTRIBUTES': 'TRUE' }): ds = gdal.OpenEx('data/cad/Building_A_Floor_0_Mapwize.dwg', allowed_drivers=['DWG']) assert ds is not None layer = ds.GetLayer('entities') defn = layer.GetLayerDefn() foundMathieu = False for feature in layer: if feature.GetField("OCCUPANT") == 'Mathieu': foundMathieu = True assert defn.GetFieldCount() == 28, \ ('did not get expected number of fields in defn. got %d' % defn.GetFieldsCount()) assert defn.GetFieldIndex('AVAILABILITY') >-1, \ ('did not get the expected field in defn. AVAILABILITY') assert defn.GetFieldIndex('ROOM') >-1, \ ('did not get the expected field in defn. ROOM') assert foundMathieu, \ ('Mathieu was not found as OCCUPANT field value') ds = None
def vsigzip_multi_thread(): with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS', 'CPL_VSIL_DEFLATE_CHUNK_SIZE': '32K'}): f = gdal.VSIFOpenL('/vsigzip//vsimem/vsigzip_multi_thread.gz', 'wb') for i in range(100000): gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) f = gdal.VSIFOpenL('/vsigzip//vsimem/vsigzip_multi_thread.gz', 'rb') data = gdal.VSIFReadL(100000, 5, f).decode('ascii') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/vsigzip_multi_thread.gz') if data != 'hello' * 100000: gdaltest.post_reason('fail') for i in range(10000): if data[i*5:i*5+5] != 'hello': print(i*5, data[i*5:i*5+5], data[i*5-5:i*5+5-5]) break return 'fail' return 'success'
def test_ogr_idf_1_with_temp_sqlite_db(): if ogr.GetDriverByName('SQLite') is None: pytest.skip() options = {'OGR_IDF_TEMP_DB_THRESHOLD': '0'} if sys.platform == 'darwin': # Otherwise we get a failure with system's sqlite 3.32.3 of Big Sur # when chaining ogr_sqlite.py and ogr_vdv.py options['OGR_IDF_DELETE_TEMP_DB'] = 'NO' with gdaltest.config_options(options): return test_ogr_idf_1()
def test_vsiwebhdfs_open(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() # Download without redirect (not nominal) handler = webserver.SequentialHandler() handler.add('GET', '/webhdfs/v1/foo/bar?op=OPEN&offset=9999990784&length=16384', 200, {}, '0123456789data') with webserver.install_http_handler(handler): f = open_for_read(gdaltest.webhdfs_base_connection + '/foo/bar') assert f is not None gdal.VSIFSeekL(f, 9999990784 + 10, 0) assert gdal.VSIFReadL(1, 4, f).decode('ascii') == 'data' gdal.VSIFCloseL(f) # Download with redirect (nominal) and permissions gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add('GET', '/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384&user.name=root&delegation=token', 307, {'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384'}) handler.add('GET', '/redirected/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384', 200, {}, 'yeah') with gdaltest.config_options({'WEBHDFS_USERNAME': '******', 'WEBHDFS_DELEGATION': 'token', 'WEBHDFS_DATANODE_HOST': 'localhost'}): with webserver.install_http_handler(handler): f = open_for_read(gdaltest.webhdfs_base_connection + '/foo/bar') assert f is not None assert gdal.VSIFReadL(1, 4, f).decode('ascii') == 'yeah' gdal.VSIFCloseL(f) # Test error gdal.VSICurlClearCache() f = open_for_read(gdaltest.webhdfs_base_connection + '/foo/bar') assert f is not None handler = webserver.SequentialHandler() handler.add('GET', '/webhdfs/v1/foo/bar?op=OPEN&offset=0&length=16384', 404) with webserver.install_http_handler(handler): assert len(gdal.VSIFReadL(1, 4, f)) == 0 # Retry: shouldn't not cause network access assert len(gdal.VSIFReadL(1, 4, f)) == 0 gdal.VSIFCloseL(f)
def test_vsizip_multi_thread_error(): with gdaltest.error_handler(): with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS', 'CPL_VSIL_DEFLATE_CHUNK_SIZE': '16K'}): fmain = gdal.VSIFOpenL('/vsizip/{/vsimem/vsizip_multi_thread.zip||maxlength=1000}', 'wb') f = gdal.VSIFOpenL('/vsizip/{/vsimem/vsizip_multi_thread.zip||maxlength=1000}/test', 'wb') for i in range(100000): gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(fmain) gdal.Unlink('/vsimem/vsizip_multi_thread.zip')
def test_ogr_dwg_5(): with gdaltest.config_options({ 'DWG_INLINE_BLOCKS': 'FALSE', 'DWG_ATTRIBUTES': 'TRUE', 'DWG_ALL_ATTRIBUTES': 'FALSE' }): ds = gdal.OpenEx('data/cad/attribs.dwg', allowed_drivers=['DWG']) assert ds is not None layer = ds.GetLayer('entities') defn = layer.GetLayerDefn() assert defn.GetFieldCount() == 11, \ ('did not get expected number of fields in defn. got %d' % defn.GetFieldCount()) ds = None with gdaltest.config_options({ 'DWG_INLINE_BLOCKS': 'FALSE', 'DWG_ATTRIBUTES': 'TRUE' }): ds = gdal.OpenEx('data/cad/attribs.dwg', allowed_drivers=['DWG']) assert ds is not None layer = ds.GetLayer('entities') defn = layer.GetLayerDefn() assert defn.GetFieldCount() == 32, \ ('did not get expected number of fields in defn. got %d' % defn.GetFieldCount()) ds = None
def test_vsizip_multi_thread_below_threshold(): with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): fmain = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip', 'wb') f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'wb') gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(fmain) f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'rb') data = gdal.VSIFReadL(1, 5, f).decode('ascii') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/vsizip_multi_thread.zip') assert data == 'hello'
def test_vsizip_multi_thread_below_threshold(): with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): fmain = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip', 'wb') f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'wb') gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(fmain) f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'rb') data = gdal.VSIFReadL(1, 5, f).decode('ascii') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/vsizip_multi_thread.zip') assert data == 'hello'
def test_vsigs_write_retry(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() with gdaltest.config_options({ 'GDAL_HTTP_MAX_RETRY': '2', 'GDAL_HTTP_RETRY_DELAY': '0.01' }): f = gdal.VSIFOpenL('/vsigs/test_write_retry/put_with_retry.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.wfile.write( 'HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii')) content = '' while True: numchars = int(request.rfile.readline().strip(), 16) content += request.rfile.read(numchars).decode('ascii') request.rfile.read(2) if numchars == 0: break if len(content) != 3: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(200) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/test_write_retry/put_with_retry.bin', 502) handler.add('PUT', '/test_write_retry/put_with_retry.bin', custom_method=method) with webserver.install_http_handler(handler): assert gdal.VSIFWriteL('foo', 1, 3, f) == 3 gdal.VSIFCloseL(f)
def test_ogr_arrow_extension_type(): outfilename = '/vsimem/out.feather' with gdaltest.config_options({ 'OGR_ARROW_WRITE_GDAL_FOOTER': 'NO', 'OGR_ARROW_WRITE_GEO': 'NO' }): gdal.VectorTranslate(outfilename, 'data/arrow/test.feather') ds = ogr.Open(outfilename) assert ds is not None lyr = ds.GetLayer(0) assert lyr is not None assert lyr.GetGeometryColumn() assert lyr.GetLayerDefn().GetGeomFieldCount() == 1 lyr = None ds = None gdal.Unlink(outfilename)
def test_vsigs_write_retry(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() with gdaltest.config_options({'GDAL_HTTP_MAX_RETRY': '2', 'GDAL_HTTP_RETRY_DELAY': '0.01'}): f = gdal.VSIFOpenL('/vsigs/test_write_retry/put_with_retry.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii')) content = '' while True: numchars = int(request.rfile.readline().strip(), 16) content += request.rfile.read(numchars).decode('ascii') request.rfile.read(2) if numchars == 0: break if len(content) != 3: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(200) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/test_write_retry/put_with_retry.bin', 502) handler.add('PUT', '/test_write_retry/put_with_retry.bin', custom_method=method) with webserver.install_http_handler(handler): assert gdal.VSIFWriteL('foo', 1, 3, f) == 3 gdal.VSIFCloseL(f)
def test_vsicurl_test_CPL_CURL_VERBOSE(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() class MyHandler: def __init__(self): self.found_CURL_INFO = False self.found_CURL_INFO_HEADER_IN = False self.found_CURL_INFO_HEADER_OUT = False def handler(self, err_type, err_no, err_msg): if 'CURL_INFO_TEXT:' in err_msg: self.found_CURL_INFO_TEXT = True if 'CURL_INFO_HEADER_IN:' in err_msg: self.found_CURL_INFO_HEADER_IN = True if 'CURL_INFO_HEADER_OUT:' in err_msg: self.found_CURL_INFO_HEADER_OUT = True handler = webserver.SequentialHandler() handler.add('HEAD', '/test_vsicurl_test_CPL_CURL_VERBOSE', 200, {'Content-Length': '3'}) my_error_handler = MyHandler() with gdaltest.config_options({ 'CPL_CURL_VERBOSE': 'YES', 'CPL_DEBUG': 'ON' }): with gdaltest.error_handler(my_error_handler.handler): with webserver.install_http_handler(handler): statres = gdal.VSIStatL( '/vsicurl/http://localhost:%d/test_vsicurl_test_CPL_CURL_VERBOSE' % gdaltest.webserver_port) assert statres.size == 3 assert my_error_handler.found_CURL_INFO_TEXT assert my_error_handler.found_CURL_INFO_HEADER_IN assert my_error_handler.found_CURL_INFO_HEADER_OUT gdal.VSICurlClearCache()
def vsizip_multi_thread_below_threshold(): with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): fmain = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip', 'wb') f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'wb') gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(fmain) f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'rb') data = gdal.VSIFReadL(1, 5, f).decode('ascii') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/vsizip_multi_thread.zip') if data != 'hello': gdaltest.post_reason('fail') print(data) return 'fail' return 'success'
def vsizip_multi_thread_below_threshold(): with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): fmain = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip', 'wb') f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'wb') gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(fmain) f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'rb') data = gdal.VSIFReadL(1, 5, f).decode('ascii') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/vsizip_multi_thread.zip') if data != 'hello': gdaltest.post_reason('fail') print(data) return 'fail' return 'success'
def vsizip_create_zip64_stream_larger_than_4G(): if not gdaltest.run_slow_tests(): return 'skip' zip_name = 'tmp/vsizip_create_zip64_stream_larger_than_4G.zip' gdal.Unlink(zip_name) niters = 999 s = ''.join([chr(random.randint(0, 127)) for i in range(5 * 1000 * 1000)]) with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test2', 'wb') for i in range(niters): gdal.VSIFWriteL(s, 1, len(s), f) gdal.VSIFCloseL(f) size = gdal.VSIStatL(zip_name).size if size <= 0xFFFFFFFF: gdaltest.post_reason('fail') print(size) return 'fail' size = gdal.VSIStatL('/vsizip/' + zip_name + '/test2').size if size != len(s) * niters: gdaltest.post_reason('fail') print(size) return 'fail' f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test2', 'rb') data = gdal.VSIFReadL(1, len(s), f).decode('ascii') gdal.VSIFCloseL(f) if data != s: gdaltest.post_reason('fail') return 'fail' gdal.Unlink(zip_name) return 'success'
def vsizip_create_zip64_stream_larger_than_4G(): if not gdaltest.run_slow_tests(): return 'skip' zip_name = 'tmp/vsizip_create_zip64_stream_larger_than_4G.zip' gdal.Unlink(zip_name) niters = 999 s = ''.join([chr(random.randint(0, 127)) for i in range(5 * 1000 * 1000)]) with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test2', 'wb') for i in range(niters): gdal.VSIFWriteL(s, 1, len(s), f) gdal.VSIFCloseL(f) size = gdal.VSIStatL(zip_name).size if size <= 0xFFFFFFFF: gdaltest.post_reason('fail') print(size) return 'fail' size = gdal.VSIStatL('/vsizip/' + zip_name + '/test2').size if size != len(s) * niters: gdaltest.post_reason('fail') print(size) return 'fail' f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test2', 'rb') data = gdal.VSIFReadL(1, len(s), f).decode('ascii') gdal.VSIFCloseL(f) if data != s: gdaltest.post_reason('fail') return 'fail' gdal.Unlink(zip_name) return 'success'
def vsizip_create_zip64(): if not gdaltest.run_slow_tests(): return 'skip' niters = 1000 s = 'hello' * 1000 * 1000 zip_name = '/vsimem/vsizip_create_zip64.zip' with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): fmain = gdal.VSIFOpenL('/vsizip/' + zip_name, 'wb') f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test', 'wb') for i in range(niters): gdal.VSIFWriteL(s, 1, len(s), f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(fmain) size = gdal.VSIStatL(zip_name).size if size > 0xFFFFFFFF: gdaltest.post_reason('fail') print(size) return 'fail' size = gdal.VSIStatL('/vsizip/' + zip_name + '/test').size if size != len(s) * niters: gdaltest.post_reason('fail') print(size) return 'fail' f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test', 'rb') data = gdal.VSIFReadL(1, len(s), f).decode('ascii') gdal.VSIFCloseL(f) if data != s: gdaltest.post_reason('fail') return 'fail' gdal.Unlink(zip_name) return 'success'
def vsizip_create_zip64(): if not gdaltest.run_slow_tests(): return 'skip' niters = 1000 s = 'hello' * 1000 * 1000 zip_name = '/vsimem/vsizip_create_zip64.zip' with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS'}): fmain = gdal.VSIFOpenL('/vsizip/' + zip_name, 'wb') f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test', 'wb') for i in range(niters): gdal.VSIFWriteL(s, 1, len(s), f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(fmain) size = gdal.VSIStatL(zip_name).size if size > 0xFFFFFFFF: gdaltest.post_reason('fail') print(size) return 'fail' size = gdal.VSIStatL('/vsizip/' + zip_name + '/test').size if size != len(s) * niters: gdaltest.post_reason('fail') print(size) return 'fail' f = gdal.VSIFOpenL('/vsizip/' + zip_name + '/test', 'rb') data = gdal.VSIFReadL(1, len(s), f).decode('ascii') gdal.VSIFCloseL(f) if data != s: gdaltest.post_reason('fail') return 'fail' gdal.Unlink(zip_name) return 'success'
def test_vsizip_multi_thread(): with gdaltest.config_options({'GDAL_NUM_THREADS': 'ALL_CPUS', 'CPL_VSIL_DEFLATE_CHUNK_SIZE': '32K'}): fmain = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip', 'wb') f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'wb') for i in range(100000): gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(fmain) f = gdal.VSIFOpenL('/vsizip//vsimem/vsizip_multi_thread.zip/test', 'rb') data = gdal.VSIFReadL(100000, 5, f).decode('ascii') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/vsizip_multi_thread.zip') if data != 'hello' * 100000: for i in range(10000): if data[i*5:i*5+5] != 'hello': print(i*5, data[i*5:i*5+5], data[i*5-5:i*5+5-5]) break pytest.fail()
def vsiwebhdfs_write(): if gdaltest.webserver_port == 0: return 'skip' gdal.VSICurlClearCache() # Zero length file handler = webserver.SequentialHandler() with webserver.install_http_handler(handler): # Missing required config options with gdaltest.error_handler(): f = gdal.VSIFOpenL( gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') if f is not None: gdaltest.post_reason('fail') return 'fail' handler = webserver.SequentialHandler() handler.add('PUT', '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 307, {'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root'}, ) handler.add( 'PUT', '/redirected/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 201) with gdaltest.config_options({'WEBHDFS_USERNAME': '******', 'WEBHDFS_DATANODE_HOST': 'localhost'}): with webserver.install_http_handler(handler): f = gdal.VSIFOpenL( gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFCloseL(f) != 0: gdaltest.post_reason('fail') return 'fail' # Non-empty file gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add('PUT', '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 307, {'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root'}, ) handler.add( 'PUT', '/redirected/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 201) with gdaltest.config_options({'WEBHDFS_USERNAME': '******', 'WEBHDFS_DATANODE_HOST': 'localhost'}): with webserver.install_http_handler(handler): f = gdal.VSIFOpenL( gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFWriteL('foobar', 1, 6, f) != 6: gdaltest.post_reason('fail') return 'fail' handler = webserver.SequentialHandler() def method(request): h = request.headers if 'Content-Length' in h and h['Content-Length'] != 0: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() request.protocol_version = 'HTTP/1.1' request.send_response(307) request.send_header('Location', gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=APPEND&user.name=root') request.end_headers() handler.add('POST', '/webhdfs/v1/foo/bar?op=APPEND&user.name=root', 307, custom_method=method) handler.add('POST', '/redirected/webhdfs/v1/foo/bar?op=APPEND&user.name=root', 200, expected_body='foobar'.encode('ascii')) with webserver.install_http_handler(handler): if gdal.VSIFCloseL(f) != 0: gdaltest.post_reason('fail') return 'fail' # Errors during file creation gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add( 'PUT', '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 404) with gdaltest.config_options({'WEBHDFS_USERNAME': '******', 'WEBHDFS_DATANODE_HOST': 'localhost'}): with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = gdal.VSIFOpenL( gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') if f is not None: gdaltest.post_reason('fail') return 'fail' handler = webserver.SequentialHandler() handler.add('PUT', '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 307, {'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root'}, ) with gdaltest.config_options({'WEBHDFS_USERNAME': '******'}): with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = gdal.VSIFOpenL( gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') if f is not None: gdaltest.post_reason('fail') return 'fail' # Errors during POST gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add('PUT', '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 307, {'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root'}, ) handler.add( 'PUT', '/redirected/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 201) with gdaltest.config_options({'WEBHDFS_USERNAME': '******', 'WEBHDFS_DATANODE_HOST': 'localhost'}): with webserver.install_http_handler(handler): f = gdal.VSIFOpenL( gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFWriteL('foobar', 1, 6, f) != 6: gdaltest.post_reason('fail') return 'fail' handler = webserver.SequentialHandler() handler.add('POST', '/webhdfs/v1/foo/bar?op=APPEND&user.name=root', 307, {'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=APPEND&user.name=root'}) handler.add( 'POST', '/redirected/webhdfs/v1/foo/bar?op=APPEND&user.name=root', 400) with gdaltest.error_handler(): with webserver.install_http_handler(handler): if gdal.VSIFCloseL(f) == 0: gdaltest.post_reason('fail') return 'fail' return 'success'
def test_vsiwebhdfs_write(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() # Zero length file handler = webserver.SequentialHandler() with webserver.install_http_handler(handler): # Missing required config options with gdaltest.error_handler(): f = gdal.VSIFOpenL(gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') assert f is None handler = webserver.SequentialHandler() handler.add( 'PUT', '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 307, { 'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root' }, ) handler.add( 'PUT', '/redirected/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 201) with gdaltest.config_options({ 'WEBHDFS_USERNAME': '******', 'WEBHDFS_DATANODE_HOST': 'localhost' }): with webserver.install_http_handler(handler): f = gdal.VSIFOpenL(gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') assert f is not None assert gdal.VSIFCloseL(f) == 0 # Non-empty file gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add( 'PUT', '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 307, { 'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root' }, ) handler.add( 'PUT', '/redirected/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 201) with gdaltest.config_options({ 'WEBHDFS_USERNAME': '******', 'WEBHDFS_DATANODE_HOST': 'localhost' }): with webserver.install_http_handler(handler): f = gdal.VSIFOpenL(gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') assert f is not None assert gdal.VSIFWriteL('foobar', 1, 6, f) == 6 handler = webserver.SequentialHandler() def method(request): h = request.headers if 'Content-Length' in h and h['Content-Length'] != 0: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() request.protocol_version = 'HTTP/1.1' request.send_response(307) request.send_header( 'Location', gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=APPEND&user.name=root') request.end_headers() handler.add('POST', '/webhdfs/v1/foo/bar?op=APPEND&user.name=root', 307, custom_method=method) handler.add('POST', '/redirected/webhdfs/v1/foo/bar?op=APPEND&user.name=root', 200, expected_body='foobar'.encode('ascii')) with webserver.install_http_handler(handler): assert gdal.VSIFCloseL(f) == 0 # Errors during file creation gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add('PUT', '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 404) with gdaltest.config_options({ 'WEBHDFS_USERNAME': '******', 'WEBHDFS_DATANODE_HOST': 'localhost' }): with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = gdal.VSIFOpenL( gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') assert f is None handler = webserver.SequentialHandler() handler.add( 'PUT', '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 307, { 'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root' }, ) with gdaltest.config_options({'WEBHDFS_USERNAME': '******'}): with webserver.install_http_handler(handler): with gdaltest.error_handler(): f = gdal.VSIFOpenL( gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') assert f is None # Errors during POST gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add( 'PUT', '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 307, { 'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root' }, ) handler.add( 'PUT', '/redirected/webhdfs/v1/foo/bar?op=CREATE&overwrite=true&user.name=root', 201) with gdaltest.config_options({ 'WEBHDFS_USERNAME': '******', 'WEBHDFS_DATANODE_HOST': 'localhost' }): with webserver.install_http_handler(handler): f = gdal.VSIFOpenL(gdaltest.webhdfs_base_connection + '/foo/bar', 'wb') assert f is not None assert gdal.VSIFWriteL('foobar', 1, 6, f) == 6 handler = webserver.SequentialHandler() handler.add( 'POST', '/webhdfs/v1/foo/bar?op=APPEND&user.name=root', 307, { 'Location': gdaltest.webhdfs_redirected_url + '/webhdfs/v1/foo/bar?op=APPEND&user.name=root' }) handler.add('POST', '/redirected/webhdfs/v1/foo/bar?op=APPEND&user.name=root', 400) with gdaltest.error_handler(): with webserver.install_http_handler(handler): assert gdal.VSIFCloseL(f) != 0
def test_ogr_pds4_create_table_binary(): options = [ 'VAR_LOGICAL_IDENTIFIER=logical_identifier', 'VAR_TITLE=title', 'VAR_INVESTIGATION_AREA_NAME=ian', 'VAR_INVESTIGATION_AREA_LID_REFERENCE=INVESTIGATION_AREA_LID_REFERENCE', 'VAR_OBSERVING_SYSTEM_NAME=osn', 'VAR_TARGET=target', 'VAR_TARGET_TYPE=target' ] for signedness in ['Signed', 'Unsigned']: for endianness in ['LSB', 'MSB']: ds = ogr.GetDriverByName('PDS4').CreateDataSource( '/vsimem/test.xml', options=options) layername = endianness with gdaltest.config_options({ 'PDS4_ENDIANNESS': endianness, 'PDS4_SIGNEDNESS': signedness }): lyr = ds.CreateLayer(layername, options=['TABLE_TYPE=BINARY']) fld = ogr.FieldDefn('bool', ogr.OFTInteger) fld.SetSubType(ogr.OFSTBoolean) lyr.CreateField(fld) fld = ogr.FieldDefn('byte', ogr.OFTInteger) fld.SetWidth(2) lyr.CreateField(fld) fld = ogr.FieldDefn('int16', ogr.OFTInteger) fld.SetSubType(ogr.OFSTInt16) lyr.CreateField(fld) lyr.CreateField(ogr.FieldDefn('int', ogr.OFTInteger)) lyr.CreateField(ogr.FieldDefn('int64', ogr.OFTInteger64)) fld = ogr.FieldDefn('float', ogr.OFTReal) fld.SetSubType(ogr.OFSTFloat32) lyr.CreateField(fld) lyr.CreateField(ogr.FieldDefn('real', ogr.OFTReal)) lyr.CreateField(ogr.FieldDefn('str', ogr.OFTString)) lyr.CreateField(ogr.FieldDefn('datetime', ogr.OFTDateTime)) lyr.CreateField(ogr.FieldDefn('date', ogr.OFTDate)) lyr.CreateField(ogr.FieldDefn('time', ogr.OFTTime)) sign = -1 if signedness == 'Signed' else 1 f = ogr.Feature(lyr.GetLayerDefn()) f['bool'] = 1 f['byte'] = sign * 9 f['int16'] = sign * 12345 f['int'] = sign * 123456789 f['int64'] = sign * 1234567890123 f['float'] = 1.25 f['real'] = 1.2567 f['str'] = 'foo' f['datetime'] = '2019/01/24 12:34:56.789+00' f['date'] = '2019-01-24' f['time'] = '12:34:56.789' lyr.CreateFeature(f) ds = None f = gdal.VSIFOpenL('/vsimem/test.xml', 'rb') data = gdal.VSIFReadL(1, 100000, f).decode('ascii') gdal.VSIFCloseL(f) assert '_Binary' in data assert '_Character' not in data if endianness == 'LSB': assert 'LSB' in data, data assert 'MSB' not in data, data else: assert 'MSB' in data, data assert 'LSB' not in data, data if signedness == 'Signed': assert 'Signed' in data, data assert 'Unsigned' not in data, data else: assert 'Unsigned' in data, data assert 'Signed' not in data, data assert validate_xml('/vsimem/test.xml') ds = ogr.Open('/vsimem/test.xml') layername = endianness lyr = ds.GetLayerByName(layername) assert lyr.GetLayerDefn().GetFieldCount() == 11 f = lyr.GetNextFeature() assert f['bool'] assert f['byte'] == sign * 9 assert f['int16'] == sign * 12345 assert f['int'] == sign * 123456789 assert f['int64'] == sign * 1234567890123 assert f['float'] == 1.25 assert f['real'] == 1.2567 assert f['str'] == 'foo' assert f['datetime'] == '2019/01/24 12:34:56.789+00' assert f['date'] == '2019/01/24' assert f['time'] == '12:34:56.789' ds = None # Add new layer ds = ogr.Open('/vsimem/test.xml', update=1) sr = osr.SpatialReference() sr.SetFromUserInput('WGS84') lyr = ds.CreateLayer('bar', geom_type=ogr.wkbPoint25D, srs=sr, options=['TABLE_TYPE=BINARY']) f = ogr.Feature(lyr.GetLayerDefn()) f.SetGeometryDirectly(ogr.CreateGeometryFromWkt('POINT Z (1 2 3)')) lyr.CreateFeature(f) ds = None assert validate_xml('/vsimem/test.xml') ds = ogr.Open('/vsimem/test.xml') lyr = ds.GetLayerByName('bar') f = lyr.GetNextFeature() assert f.GetGeometryRef().ExportToIsoWkt() == 'POINT Z (1 2 3)' ds = None ogr.GetDriverByName('PDS4').DeleteDataSource('/vsimem/test.xml') gdal.Rmdir('/vsimem/test')
def test_vsiwebhdfs_mkdir_rmdir(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() # Invalid name ret = gdal.Mkdir('/vsiwebhdfs', 0) assert ret != 0 # Valid handler = webserver.SequentialHandler() handler.add('PUT', '/webhdfs/v1/foo/dir?op=MKDIRS', 200, {}, '{"boolean":true}') with webserver.install_http_handler(handler): ret = gdal.Mkdir(gdaltest.webhdfs_base_connection + '/foo/dir', 0) assert ret == 0 # Valid with all options handler = webserver.SequentialHandler() handler.add( 'PUT', '/webhdfs/v1/foo/dir?op=MKDIRS&user.name=root&delegation=token&permission=755', 200, {}, '{"boolean":true}') with gdaltest.config_options({ 'WEBHDFS_USERNAME': '******', 'WEBHDFS_DELEGATION': 'token' }): with webserver.install_http_handler(handler): ret = gdal.Mkdir(gdaltest.webhdfs_base_connection + '/foo/dir/', 493) # 0755 assert ret == 0 # Error handler = webserver.SequentialHandler() handler.add('PUT', '/webhdfs/v1/foo/dir_error?op=MKDIRS', 404) with webserver.install_http_handler(handler): ret = gdal.Mkdir(gdaltest.webhdfs_base_connection + '/foo/dir_error', 0) assert ret != 0 # Root name is invalid ret = gdal.Mkdir(gdaltest.webhdfs_base_connection + '/', 0) assert ret != 0 # Invalid name ret = gdal.Rmdir('/vsiwebhdfs') assert ret != 0 gdal.VSICurlClearCache() # Valid handler = webserver.SequentialHandler() handler.add('DELETE', '/webhdfs/v1/foo/dir?op=DELETE', 200, {}, '{"boolean":true}') with webserver.install_http_handler(handler): ret = gdal.Rmdir(gdaltest.webhdfs_base_connection + '/foo/dir') assert ret == 0 # Error handler = webserver.SequentialHandler() handler.add('DELETE', '/webhdfs/v1/foo/dir_error?op=DELETE', 404) with webserver.install_http_handler(handler): ret = gdal.Rmdir(gdaltest.webhdfs_base_connection + '/foo/dir_error') assert ret != 0
def test_ogr_pds4_create_table_binary(): options = ['VAR_LOGICAL_IDENTIFIER=logical_identifier', 'VAR_TITLE=title', 'VAR_INVESTIGATION_AREA_NAME=ian', 'VAR_INVESTIGATION_AREA_LID_REFERENCE=INVESTIGATION_AREA_LID_REFERENCE', 'VAR_OBSERVING_SYSTEM_NAME=osn', 'VAR_TARGET=target', 'VAR_TARGET_TYPE=target'] for signedness in ['Signed', 'Unsigned']: for endianness in ['LSB', 'MSB']: ds = ogr.GetDriverByName('PDS4').CreateDataSource('/vsimem/test.xml', options=options) layername = endianness with gdaltest.config_options( {'PDS4_ENDIANNESS': endianness, 'PDS4_SIGNEDNESS': signedness} ): lyr = ds.CreateLayer(layername, options = ['TABLE_TYPE=BINARY']) fld = ogr.FieldDefn('bool', ogr.OFTInteger) fld.SetSubType(ogr.OFSTBoolean) lyr.CreateField(fld) fld = ogr.FieldDefn('byte', ogr.OFTInteger) fld.SetWidth(2) lyr.CreateField(fld) fld = ogr.FieldDefn('int16', ogr.OFTInteger) fld.SetSubType(ogr.OFSTInt16) lyr.CreateField(fld) lyr.CreateField(ogr.FieldDefn('int', ogr.OFTInteger)) lyr.CreateField(ogr.FieldDefn('int64', ogr.OFTInteger64)) fld = ogr.FieldDefn('float', ogr.OFTReal) fld.SetSubType(ogr.OFSTFloat32) lyr.CreateField(fld) lyr.CreateField(ogr.FieldDefn('real', ogr.OFTReal)) lyr.CreateField(ogr.FieldDefn('str', ogr.OFTString)) lyr.CreateField(ogr.FieldDefn('datetime', ogr.OFTDateTime)) lyr.CreateField(ogr.FieldDefn('date', ogr.OFTDate)) lyr.CreateField(ogr.FieldDefn('time', ogr.OFTTime)) sign = -1 if signedness == 'Signed' else 1 f = ogr.Feature(lyr.GetLayerDefn()) f['bool'] = 1 f['byte'] = sign * 9 f['int16'] = sign * 12345 f['int'] = sign * 123456789 f['int64'] = sign * 1234567890123 f['float'] = 1.25 f['real'] = 1.2567 f['str'] = 'foo' f['datetime'] = '2019/01/24 12:34:56.789+00' f['date'] = '2019-01-24' f['time'] = '12:34:56.789' lyr.CreateFeature(f) ds = None f = gdal.VSIFOpenL('/vsimem/test.xml', 'rb') data = gdal.VSIFReadL(1, 100000, f).decode('ascii') gdal.VSIFCloseL(f) assert '_Binary' in data assert '_Character' not in data if endianness == 'LSB': assert 'LSB' in data, data assert 'MSB' not in data, data else: assert 'MSB' in data, data assert 'LSB' not in data, data if signedness == 'Signed': assert 'Signed' in data, data assert 'Unsigned' not in data, data else: assert 'Unsigned' in data, data assert 'Signed' not in data, data assert validate_xml('/vsimem/test.xml') ds = ogr.Open('/vsimem/test.xml') layername = endianness lyr = ds.GetLayerByName(layername) assert lyr.GetLayerDefn().GetFieldCount() == 11 f = lyr.GetNextFeature() assert f['bool'] assert f['byte'] == sign * 9 assert f['int16'] == sign * 12345 assert f['int'] == sign * 123456789 assert f['int64'] == sign * 1234567890123 assert f['float'] == 1.25 assert f['real'] == 1.2567 assert f['str'] == 'foo' assert f['datetime'] == '2019/01/24 12:34:56.789+00' assert f['date'] == '2019/01/24' assert f['time'] == '12:34:56.789' ds = None # Add new layer ds = ogr.Open('/vsimem/test.xml', update = 1) sr = osr.SpatialReference() sr.SetFromUserInput('WGS84') lyr = ds.CreateLayer('bar', geom_type = ogr.wkbPoint25D, srs = sr, options = ['TABLE_TYPE=BINARY']) f = ogr.Feature(lyr.GetLayerDefn()) f.SetGeometryDirectly(ogr.CreateGeometryFromWkt('POINT Z (1 2 3)')) lyr.CreateFeature(f) ds = None assert validate_xml('/vsimem/test.xml') ds = ogr.Open('/vsimem/test.xml') lyr = ds.GetLayerByName('bar') f = lyr.GetNextFeature() assert f.GetGeometryRef().ExportToIsoWkt() == 'POINT Z (1 2 3)' ds = None ogr.GetDriverByName('PDS4').DeleteDataSource('/vsimem/test.xml')
def vsiwebhdfs_mkdir_rmdir(): if gdaltest.webserver_port == 0: return 'skip' gdal.VSICurlClearCache() # Invalid name ret = gdal.Mkdir('/vsiwebhdfs', 0) if ret == 0: gdaltest.post_reason('fail') return 'fail' # Valid handler = webserver.SequentialHandler() handler.add('PUT', '/webhdfs/v1/foo/dir?op=MKDIRS', 200, {}, '{"boolean":true}') with webserver.install_http_handler(handler): ret = gdal.Mkdir(gdaltest.webhdfs_base_connection + '/foo/dir', 0) if ret != 0: gdaltest.post_reason('fail') return 'fail' # Valid with all options handler = webserver.SequentialHandler() handler.add('PUT', '/webhdfs/v1/foo/dir?op=MKDIRS&user.name=root&delegation=token&permission=755', 200, {}, '{"boolean":true}') with gdaltest.config_options({'WEBHDFS_USERNAME': '******', 'WEBHDFS_DELEGATION': 'token'}): with webserver.install_http_handler(handler): ret = gdal.Mkdir(gdaltest.webhdfs_base_connection + '/foo/dir/', 493) # 0755 if ret != 0: gdaltest.post_reason('fail') return 'fail' # Error handler = webserver.SequentialHandler() handler.add('PUT', '/webhdfs/v1/foo/dir_error?op=MKDIRS', 404) with webserver.install_http_handler(handler): ret = gdal.Mkdir(gdaltest.webhdfs_base_connection + '/foo/dir_error', 0) if ret == 0: gdaltest.post_reason('fail') return 'fail' # Root name is invalid ret = gdal.Mkdir(gdaltest.webhdfs_base_connection + '/', 0) if ret == 0: gdaltest.post_reason('fail') return 'fail' # Invalid name ret = gdal.Rmdir('/vsiwebhdfs') if ret == 0: gdaltest.post_reason('fail') return 'fail' gdal.VSICurlClearCache() # Valid handler = webserver.SequentialHandler() handler.add('DELETE', '/webhdfs/v1/foo/dir?op=DELETE', 200, {}, '{"boolean":true}') with webserver.install_http_handler(handler): ret = gdal.Rmdir(gdaltest.webhdfs_base_connection + '/foo/dir') if ret != 0: gdaltest.post_reason('fail') return 'fail' # Error handler = webserver.SequentialHandler() handler.add('DELETE', '/webhdfs/v1/foo/dir_error?op=DELETE', 404) with webserver.install_http_handler(handler): ret = gdal.Rmdir(gdaltest.webhdfs_base_connection + '/foo/dir_error') if ret == 0: gdaltest.post_reason('fail') return 'fail' return 'success'
def test_vsiswift_fake_auth_v3_application_credential_url(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() gdal.SetConfigOption('SWIFT_STORAGE_URL', '') gdal.SetConfigOption('SWIFT_AUTH_TOKEN', '') with gdaltest.config_options({ 'OS_IDENTITY_API_VERSION': '3', 'OS_AUTH_URL': 'http://127.0.0.1:%d/v3' % gdaltest.webserver_port, 'OS_AUTH_TYPE': 'v3applicationcredential', 'OS_APPLICATION_CREDENTIAL_ID': 'xxxyyycredential-idyyyxxx==', 'OS_APPLICATION_CREDENTIAL_SECRET': 'xxxyyycredential-secretyyyxxx==', 'OS_USER_DOMAIN_NAME': 'test_user_domain', 'OS_REGION_NAME': 'Test' }): handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' h = request.headers if 'Content-Type' not in h or h[ 'Content-Type'] != 'application/json': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request_len = int(h['Content-Length']) request_body = request.rfile.read(request_len).decode() request_json = json.loads(request_body) methods = request_json['auth']['identity']["methods"] assert "application_credential" in methods cred_id = request_json['auth']['identity'][ 'application_credential']['id'] cred_secret = request_json['auth']['identity'][ 'application_credential']['secret'] assert cred_id == 'xxxyyycredential-idyyyxxx==' assert cred_secret == 'xxxyyycredential-secretyyyxxx==' content = """{ "token" : { "catalog" : [ { "endpoints" : [ { "region" : "Test", "interface" : "admin", "url" : "http://127.0.0.1:8080/v1/admin/AUTH_something" }, { "region" : "Test", "interface" : "internal", "url" : "http://127.0.0.1:8081/v1/internal/AUTH_something" }, { "region" : "Test", "interface" : "public", "url" : "http://127.0.0.1:%d/v1/AUTH_something" } ], "type": "object-store", "name" : "swift" } ] } }""" % gdaltest.webserver_port content = content.encode('ascii') request.send_response(200) request.send_header('Content-Length', len(content)) request.send_header('Content-Type', 'application/json') request.send_header('X-Subject-Token', 'my_auth_token') request.end_headers() request.wfile.write(content) handler.add('POST', '/v3/auth/tokens', custom_method=method) def method(request): request.protocol_version = 'HTTP/1.1' h = request.headers if 'x-auth-token' not in h or \ h['x-auth-token'] != 'my_auth_token': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.send_header('Connection', 'close') request.end_headers() request.wfile.write('foo'.encode('ascii')) handler.add('GET', '/v1/AUTH_something/foo/bar', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsiswift/foo/bar') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') assert data == 'foo' gdal.VSIFCloseL(f)
def test_vsistdin_5(): f = open('tmp/test_vsistdin_5.bin', 'wb') f.write(b'0123456789' * (1024 * 1024)) f.close() with gdaltest.config_options({ 'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin', 'CPL_VSISTDIN_RESET_POSITION': 'YES', 'CPL_VSISTDIN_FILE_CLOSE': 'YES' }): f = gdal.VSIFOpenL('/vsistdin?buffer_limit=10MB', 'rb') assert f is not None assert gdal.VSIFEofL(f) == 0 assert gdal.VSIFReadL(10, 1, f) == b'0123456789' assert gdal.VSIFTellL(f) == 10 assert gdal.VSIFSeekL(f, 5, 0) == 0 assert gdal.VSIFTellL(f) == 5 assert gdal.VSIFReadL(3, 1, f) == b'567' assert gdal.VSIFTellL(f) == 8 assert gdal.VSIFReadL(4, 1, f) == b'8901' assert gdal.VSIFTellL(f) == 12 assert gdal.VSIFSeekL(f, 0, 2) == 0 assert gdal.VSIFTellL(f) == 10 * 1024 * 1024 assert gdal.VSIFReadL(1, 1, f) == b'' assert gdal.VSIFEofL(f) == 1 assert gdal.VSIFTellL(f) == 10 * 1024 * 1024 assert gdal.VSIFSeekL(f, 5, 0) == 0 assert gdal.VSIFTellL(f) == 5 assert gdal.VSIFReadL(3, 1, f) == b'567' assert gdal.VSIFTellL(f) == 8 assert gdal.VSIFSeekL(f, 10 * 1024 * 1024 - 10 + 1, 0) == 0 assert gdal.VSIFReadL(3, 1, f) == b'123' gdal.VSIFCloseL(f) with gdaltest.config_options({ 'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin', 'CPL_VSISTDIN_RESET_POSITION': 'YES', 'CPL_VSISTDIN_FILE_CLOSE': 'YES' }): f = gdal.VSIFOpenL('/vsistdin?buffer_limit=-1', 'rb') assert f is not None assert gdal.VSIFSeekL(f, 0, 2) == 0 assert gdal.VSIFTellL(f) == 10 * 1024 * 1024 gdal.VSIFCloseL(f) assert gdal.VSIStatL( '/vsistdin?buffer_limit=-1').size == 10 * 1024 * 1024 with gdaltest.config_options({ 'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin', 'CPL_VSISTDIN_RESET_POSITION': 'YES', 'CPL_VSISTDIN_FILE_CLOSE': 'YES' }): f = gdal.VSIFOpenL('/vsistdin?buffer_limit=1GB', 'rb') assert f is not None assert gdal.VSIFSeekL(f, 0, 2) == 0 assert gdal.VSIFTellL(f) == 10 * 1024 * 1024 gdal.VSIFCloseL(f) assert gdal.VSIStatL( '/vsistdin?buffer_limit=-1').size == 10 * 1024 * 1024 with gdaltest.config_options({ 'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin', 'CPL_VSISTDIN_RESET_POSITION': 'YES', 'CPL_VSISTDIN_FILE_CLOSE': 'YES' }): f = gdal.VSIFOpenL('/vsistdin?buffer_limit=10', 'rb') assert f is not None assert gdal.VSIFSeekL(f, 0, 2) == 0 assert gdal.VSIFTellL(f) == 10 * 1024 * 1024 assert gdal.VSIFSeekL(f, 0, 0) == 0 assert gdal.VSIFReadL(5, 1, f) == b'01234' assert gdal.VSIFReadL(3, 1, f) == b'567' with gdaltest.error_handler(): assert gdal.VSIFReadL(3, 1, f) == b'' gdal.VSIFCloseL(f) os.unlink('tmp/test_vsistdin_5.bin')