def test_vsigs_read_credentials_oauth2_service_account(): if gdaltest.webserver_port == 0: pytest.skip() gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '') gdal.SetConfigOption('GS_ACCESS_KEY_ID', '') # Generated with 'openssl genrsa -out rsa-openssl.pem 1024' and # 'openssl pkcs8 -nocrypt -in rsa-openssl.pem -inform PEM -topk8 -outform PEM -out rsa-openssl.pkcs8.pem' # DO NOT USE in production !!!! key = """-----BEGIN PRIVATE KEY----- MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk VNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm i2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ DVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy TabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I 2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd DrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+ bpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg Wz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3 4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5 Kvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj j+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI yzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq gwE6fxOLyJDxuWRf -----END PRIVATE KEY----- """ for i in range(2): gdal.SetConfigOption('GO2A_AUD', 'http://localhost:%d/oauth2/v4/token' % gdaltest.webserver_port) gdal.SetConfigOption('GOA2_NOW', '123456') if i == 0: gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY', key) else: gdal.FileFromMemBuffer('/vsimem/pkey', key) gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY_FILE', '/vsimem/pkey') gdal.SetConfigOption('GS_OAUTH2_CLIENT_EMAIL', 'CLIENT_EMAIL') gdal.VSICurlClearCache() handler = webserver.SequentialHandler() def method(request): content = request.rfile.read(int(request.headers['Content-Length'])).decode('ascii') content_8080 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODAvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.DAhqWtBgKpObxZ%2BGiXqwF%2Fa4SS%2FNWQRhLCI7DYZCuOTuf2w7dL8j4CdpiwwzQg1diIus7dyViRfzpsFmuZKAXwL%2B84iBoVVqnJJZ4TgwH49NdfMAnc4Rgm%2Bo2a2nEcMjX%2FbQ3jRY%2B9WNVl96hzULGvLrVeyego2f06wivqmvxHA%3D' content_8081 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODEvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.0abOEg4%2FRApWTSeAs6YTHaNzdwOgZLm8DTMO2MKlOA%2Fiagyb4cBJxDpkD5gECPvi7qhkg7LsyFuj0a%2BK48Bsuj%2FgLHOU4MpB0dHwYnDO2UXzH%2FUPdgFCVak1P1V%2ByiDA%2B%2Ft4aDI5fD9qefKQiu3wsMDHzP71MNLzayrjqaqKKS4%3D' if content not in [content_8080, content_8081]: sys.stderr.write('Bad POST content: %s\n' % content) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') content = """{ "access_token" : "ACCESS_TOKEN", "token_type" : "Bearer", "expires_in" : 3600, }""" request.send_header('Content-Length', len(content)) request.end_headers() request.wfile.write(content.encode('ascii')) handler.add('POST', '/oauth2/v4/token', custom_method=method) def method(request): if 'Authorization' not in request.headers: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) return expected_authorization = 'Bearer ACCESS_TOKEN' if request.headers['Authorization'] != expected_authorization: sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization'])) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.end_headers() request.wfile.write("""foo""".encode('ascii')) handler.add('GET', '/gs_fake_bucket/resource', custom_method=method) try: with webserver.install_http_handler(handler): f = open_for_read('/vsigs/gs_fake_bucket/resource') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) except: if gdal.GetLastErrorMsg().find('CPLRSASHA256Sign() not implemented') >= 0: pytest.skip() finally: gdal.SetConfigOption('GO2A_AUD', None) gdal.SetConfigOption('GO2A_NOW', None) gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY', '') gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY_FILE', '') gdal.SetConfigOption('GS_OAUTH2_CLIENT_EMAIL', '') assert data == 'foo' gdal.Unlink('/vsimem/pkey')
def vsiaz_extra_1(): if not gdaltest.built_against_curl(): return 'skip' az_resource = gdal.GetConfigOption('AZ_RESOURCE') if az_resource is None: print('Missing AZ_RESOURCE for running gdaltest_list_extra') return 'skip' if az_resource.find('/') < 0: path = '/vsiaz/' + az_resource statres = gdal.VSIStatL(path) if statres is None or not stat.S_ISDIR(statres.mode): gdaltest.post_reason('fail') print('%s is not a valid bucket' % path) return 'fail' readdir = gdal.ReadDir(path) if readdir is None: gdaltest.post_reason('fail') print('ReadDir() should not return empty list') return 'fail' for filename in readdir: if filename != '.': subpath = path + '/' + filename if gdal.VSIStatL(subpath) is None: gdaltest.post_reason('fail') print('Stat(%s) should not return an error' % subpath) return 'fail' unique_id = 'vsiaz_test' subpath = path + '/' + unique_id ret = gdal.Mkdir(subpath, 0) if ret < 0: gdaltest.post_reason('fail') print('Mkdir(%s) should not return an error' % subpath) return 'fail' readdir = gdal.ReadDir(path) if unique_id not in readdir: gdaltest.post_reason('fail') print('ReadDir(%s) should contain %s' % (path, unique_id)) print(readdir) return 'fail' ret = gdal.Mkdir(subpath, 0) if ret == 0: gdaltest.post_reason('fail') print('Mkdir(%s) repeated should return an error' % subpath) return 'fail' ret = gdal.Rmdir(subpath) if ret < 0: gdaltest.post_reason('fail') print('Rmdir(%s) should not return an error' % subpath) return 'fail' readdir = gdal.ReadDir(path) if unique_id in readdir: gdaltest.post_reason('fail') print('ReadDir(%s) should not contain %s' % (path, unique_id)) print(readdir) return 'fail' ret = gdal.Rmdir(subpath) if ret == 0: gdaltest.post_reason('fail') print('Rmdir(%s) repeated should return an error' % subpath) return 'fail' ret = gdal.Mkdir(subpath, 0) if ret < 0: gdaltest.post_reason('fail') print('Mkdir(%s) should not return an error' % subpath) return 'fail' f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) ret = gdal.Rmdir(subpath) if ret == 0: gdaltest.post_reason('fail') print('Rmdir(%s) on non empty directory should return an error' % subpath) return 'fail' f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 5, f).decode('utf-8') if data != 'hello': gdaltest.post_reason('fail') print(data) return 'fail' gdal.VSIFCloseL(f) ret = gdal.Unlink(subpath + '/test.txt') if ret < 0: gdaltest.post_reason('fail') print('Unlink(%s) should not return an error' % (subpath + '/test.txt')) return 'fail' ret = gdal.Rmdir(subpath) if ret < 0: gdaltest.post_reason('fail') print('Rmdir(%s) should not return an error' % subpath) return 'fail' return 'success' f = open_for_read('/vsiaz/' + az_resource) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' # Same with /vsiaz_streaming/ f = open_for_read('/vsiaz_streaming/' + az_resource) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' if False: # we actually try to read at read() time and bSetError = false # Invalid bucket : "The specified bucket does not exist" gdal.ErrorReset() f = open_for_read('/vsiaz/not_existing_bucket/foo') with gdaltest.error_handler(): gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if gdal.VSIGetLastErrorMsg() == '': gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' # Invalid resource gdal.ErrorReset() f = open_for_read('/vsiaz_streaming/' + az_resource + '/invalid_resource.baz') if f is not None: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' # Test GetSignedURL() signed_url = gdal.GetSignedURL('/vsiaz/' + az_resource) f = open_for_read('/vsicurl_streaming/' + signed_url) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' return 'success'
def vsifile_generic(filename): start_time = time.time() fp = gdal.VSIFOpenL(filename, 'wb+') if fp is None: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFWriteL('0123456789', 1, 10, fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFFlushL(fp) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTruncateL(fp, 20) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTruncateL(fp, 5) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFSeekL(fp, 0, 2) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 5: gdaltest.post_reason('failure') return 'fail' gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 7: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if abs(start_time - statBuf.mtime) > 2: gdaltest.post_reason('failure') print(statBuf.mtime) return 'fail' fp = gdal.VSIFOpenL(filename, 'rb') buf = gdal.VSIFReadL(1, 7, fp) if gdal.VSIFWriteL('a', 1, 1, fp) != 0: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFTruncateL(fp, 0) == 0: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(fp) if buf.decode('ascii') != '01234XX': gdaltest.post_reason('failure') print(buf.decode('ascii')) return 'fail' # Test append mode on existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 9: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if gdal.Unlink(filename) != 0: gdaltest.post_reason('failure') return 'fail' statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG) if statBuf is not None: gdaltest.post_reason('failure') return 'fail' # Test append mode on non existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 2: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if gdal.Unlink(filename) != 0: gdaltest.post_reason('failure') return 'fail' return 'success'
def vsigs_2(): if gdaltest.webserver_port == 0: return 'skip' gdal.SetConfigOption('CPL_GS_ENDPOINT', 'http://127.0.0.1:%d/' % gdaltest.webserver_port) # header file gdal.FileFromMemBuffer('/vsimem/my_headers.txt', 'foo: bar') gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', '/vsimem/my_headers.txt') f = open_for_read('/vsigs/gs_fake_bucket_http_header_file/resource') if f is None: gdal.Unlink('/vsimem/my_headers.txt') gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None) gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 1, f) gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None) gdal.Unlink('/vsimem/my_headers.txt') gdal.VSIFCloseL(f) if len(data) != 1: gdaltest.post_reason('fail') return 'fail' gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY') gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID') gdal.SetConfigOption('CPL_GS_TIMESTAMP', 'my_timestamp') f = open_for_read('/vsigs/gs_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' f = open_for_read('/vsigs_streaming/gs_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' #old_val = gdal.GetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN') #gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR') stat_res = gdal.VSIStatL('/vsigs/gs_fake_bucket/resource2.bin') #gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', old_val) if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' stat_res = gdal.VSIStatL('/vsigs_streaming/gs_fake_bucket/resource2.bin') if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' return 'success'
def vsiaz_fake_basic(): if gdaltest.webserver_port == 0: return 'skip' signed_url = gdal.GetSignedURL('/vsiaz/az_fake_bucket/resource', ['START_DATE=20180213T123456']) if signed_url not in ( 'http://127.0.0.1:8080/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12', 'http://127.0.0.1:8081/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12' ): gdaltest.post_reason('fail') print(signed_url) return 'fail' def method(request): request.protocol_version = 'HTTP/1.1' h = request.headers if 'Authorization' not in h or \ h['Authorization'] != 'SharedKey myaccount:zKb0EXnM/RinBjcUE9EU+qfRIGaIItoUElSWc+FE24E=' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.send_header('Connection', 'close') request.end_headers() request.wfile.write("""foo""".encode('ascii')) handler = webserver.SequentialHandler() handler.add('GET', '/azure/blob/myaccount/az_fake_bucket/resource', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsiaz/az_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' def method(request): request.protocol_version = 'HTTP/1.1' h = request.headers if 'Authorization' not in h or \ h['Authorization'] != 'SharedKey myaccount:8d6IEeOsl7qGpKAxaTTxx2xMNpvqWq8DGlFE67lsmQ4=' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'Accept-Encoding' not in h or h['Accept-Encoding'] != 'gzip': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.send_header('Connection', 'close') request.end_headers() request.wfile.write("""foo""".encode('ascii')) handler = webserver.SequentialHandler() handler.add('GET', '/azure/blob/myaccount/az_fake_bucket/resource', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsiaz_streaming/az_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' handler = webserver.SequentialHandler() handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin', 200, {'Content-Length': '1000000'}) with webserver.install_http_handler(handler): stat_res = gdal.VSIStatL('/vsiaz/az_fake_bucket/resource2.bin') if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' handler = webserver.SequentialHandler() handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin', 200, {'Content-Length': 1000000}) with webserver.install_http_handler(handler): stat_res = gdal.VSIStatL( '/vsiaz_streaming/az_fake_bucket/resource2.bin') if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' return 'success'
def gdal_cp_single(srcfile, targetfile, progress): try: if os.path.isdir(targetfile): (head, tail) = os.path.split(srcfile) targetfile = targetfile + '/' + tail except: pass fin = gdal.VSIFOpenL(srcfile, "rb") if fin is None: print('Cannot open %s' % srcfile) return -1 fout = gdal.VSIFOpenL(targetfile, "wb") if fout is None: print('Cannot create %s' % targetfile) gdal.VSIFCloseL(fin) return -1 version_num = int(gdal.VersionInfo('VERSION_NUM')) total_size = 0 if version_num < 1900 or progress is not None: gdal.VSIFSeekL(fin, 0, 2) total_size = gdal.VSIFTellL(fin) gdal.VSIFSeekL(fin, 0, 0) buffer_max_size = 4096 copied = 0 ret = 0 #print('Copying %s...' % srcfile) if progress is not None: if not progress.Progress(0.0, 'Copying %s' % srcfile): print('Copy stopped by user') ret = -2 while ret == 0: if total_size != 0 and copied + buffer_max_size > total_size: to_read = total_size - copied else: to_read = buffer_max_size buffer = gdal.VSIFReadL(1, to_read, fin) if buffer is None: if copied == 0: print('Cannot read %d bytes in %s' % (to_read, srcfile)) ret = -1 break buffer_size = len(buffer) if gdal.VSIFWriteL(buffer, 1, buffer_size, fout) != buffer_size: print('Error writing %d bytes' % buffer_size) ret = -1 break copied += buffer_size if progress is not None and total_size != 0: if not progress.Progress(copied * 1.0 / total_size, 'Copying %s' % srcfile): print('Copy stopped by user') ret = -2 break if to_read < buffer_max_size or buffer_size != buffer_max_size: break gdal.VSIFCloseL(fin) gdal.VSIFCloseL(fout) return ret
def eedai_geotiff(): if gdaltest.eedai_drv is None: return 'skip' gdal.FileFromMemBuffer( '/vsimem/ee/assets/image', json.dumps({ 'type': 'IMAGE', 'bands': [{ "id": "B1", "dataType": { "precision": "INTEGER", "range": { "max": 65535 } }, "grid": { "crsCode": "EPSG:32610", "affineTransform": { "translateX": 499980, "translateY": 4200000, "scaleX": 60, "scaleY": -60 }, "dimensions": { "width": 1830, "height": 1831 } } }] })) gdal.SetConfigOption('EEDA_BEARER', 'mybearer') gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/') ds = gdal.Open('EEDAI:image') gdal.SetConfigOption('EEDA_URL', None) mem_ds = gdal.GetDriverByName('MEM').Create('', 256, 256, 1, gdal.GDT_UInt16) mem_ds.GetRasterBand(1).Fill(12345) gdal.GetDriverByName('GTiff').CreateCopy('/vsimem/out.tif', mem_ds) f = gdal.VSIFOpenL('/vsimem/out.tif', 'rb') data = gdal.VSIFReadL(1, 1000000, f) gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/out.tif') gdal.FileFromMemBuffer( '/vsimem/ee/assets:getPixels&CUSTOMREQUEST=POST&POSTFIELDS={ "path": "image", "encoding": "GEOTIFF", "bandIds": [ "B1" ], "grid": { "origin": { "x": 499980.0, "y": 4200000.0 }, "pixelSize": { "x": 60.0, "y": -60.0 }, "dimensions": { "width": 256, "height": 256 } } }', data) got_data = ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1) got_data = struct.unpack('H', got_data)[0] if got_data != 12345: gdaltest.post_reason('fail') print(got_data) return 'fail' ds = None gdal.SetConfigOption('EEDA_BEARER', None) return 'success'
def ogr_pgdump_9(pg_use_copy='YES'): gdal.SetConfigOption('PG_USE_COPY', pg_use_copy) ds = ogr.GetDriverByName('PGDump').CreateDataSource( '/vsimem/ogr_pgdump_9.sql', options=['LINEFORMAT=LF']) lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone) fld = ogr.FieldDefn('str', ogr.OFTString) fld.SetWidth(5) lyr.CreateField(fld) fld = ogr.FieldDefn('str2', ogr.OFTString) lyr.CreateField(fld) feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField('str', '01234') lyr.CreateFeature(feat) feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField('str', 'ABCDEF') lyr.CreateFeature(feat) if sys.version_info >= (3, 0, 0): val4 = '\u00e9\u00e9\u00e9\u00e9' val5 = val4 + '\u00e9' val6 = val5 + '\u00e9' else: exec("val4 = u'\\u00e9\\u00e9\\u00e9\\u00e9'") exec("val5 = val4 + u'\\u00e9'") exec("val6 = val5 + u'\\u00e9'") feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField('str', val6) lyr.CreateFeature(feat) feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField('str', 'a' + val5) lyr.CreateFeature(feat) gdal.SetConfigOption('PG_USE_COPY', None) ds = None f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_9.sql', 'rb') sql = gdal.VSIFReadL(1, 10000, f).decode('utf8') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/ogr_pgdump_9.sql') if pg_use_copy == 'YES': eofield = '\t' else: eofield = "'" if sql.find("""01234%s""" % eofield) < 0 or \ sql.find("""ABCDE%s""" % eofield) < 0 or \ sql.find("""%s%s""" % (val5, eofield)) < 0 or \ sql.find("""%s%s""" % ('a' + val4, eofield)) < 0: print(sql) return 'fail' return 'success'
def ogr_pgdump_13(): tests = [ [ ogr.wkbUnknown, [], 'POINT ZM (1 2 3 4)', ["'GEOMETRY',2)", "0101000000000000000000F03F0000000000000040"] ], [ ogr.wkbUnknown, ['GEOM_TYPE=geography'], 'POINT ZM (1 2 3 4)', [ "geography(GEOMETRY", "0101000000000000000000F03F0000000000000040" ] ], [ ogr.wkbUnknown, ['DIM=XYZ'], 'POINT ZM (1 2 3 4)', [ "'GEOMETRY',3)", "0101000080000000000000F03F00000000000000400000000000000840" ] ], [ ogr.wkbUnknown, ['DIM=XYZ', 'GEOM_TYPE=geography'], 'POINT ZM (1 2 3 4)', [ "geography(GEOMETRYZ,", "0101000080000000000000F03F00000000000000400000000000000840" ] ], [ ogr.wkbPoint, ['DIM=XYZ'], 'POINT ZM (1 2 3 4)', [ "'POINT',3)", "0101000080000000000000F03F00000000000000400000000000000840" ] ], [ ogr.wkbPoint25D, [], 'POINT ZM (1 2 3 4)', [ "'POINT',3)", "0101000080000000000000F03F00000000000000400000000000000840" ] ], [ ogr.wkbPoint, ['DIM=XYZ', 'GEOM_TYPE=geography'], 'POINT ZM (1 2 3 4)', [ "geography(POINTZ,", "0101000080000000000000F03F00000000000000400000000000000840" ] ], [ ogr.wkbUnknown, ['DIM=XYM'], 'POINT ZM (1 2 3 4)', [ "'GEOMETRY',3)", "0101000040000000000000F03F00000000000000400000000000001040" ] ], [ ogr.wkbUnknown, ['DIM=XYM', 'GEOM_TYPE=geography'], 'POINT ZM (1 2 3 4)', [ "geography(GEOMETRYM,", "0101000040000000000000F03F00000000000000400000000000001040" ] ], [ ogr.wkbPoint, ['DIM=XYM'], 'POINT ZM (1 2 3 4)', [ "'POINTM',3)", "0101000040000000000000F03F00000000000000400000000000001040" ] ], [ ogr.wkbPointM, [], 'POINT ZM (1 2 3 4)', [ "'POINTM',3)", "0101000040000000000000F03F00000000000000400000000000001040" ] ], [ ogr.wkbPoint, ['DIM=XYM', 'GEOM_TYPE=geography'], 'POINT ZM (1 2 3 4)', [ "geography(POINTM,", "0101000040000000000000F03F00000000000000400000000000001040" ] ], [ ogr.wkbUnknown, ['DIM=XYZM'], 'POINT ZM (1 2 3 4)', [ "'GEOMETRY',4)", "01010000C0000000000000F03F000000000000004000000000000008400000000000001040" ] ], [ ogr.wkbUnknown, ['DIM=XYZM', 'GEOM_TYPE=geography'], 'POINT ZM (1 2 3 4)', [ "geography(GEOMETRYZM,", "01010000C0000000000000F03F000000000000004000000000000008400000000000001040" ] ], [ ogr.wkbPoint, ['DIM=XYZM'], 'POINT ZM (1 2 3 4)', [ "'POINT',4)", "01010000C0000000000000F03F000000000000004000000000000008400000000000001040" ] ], [ ogr.wkbPointZM, [], 'POINT ZM (1 2 3 4)', [ "'POINT',4)", "01010000C0000000000000F03F000000000000004000000000000008400000000000001040" ] ], [ ogr.wkbPoint, ['DIM=XYZM', 'GEOM_TYPE=geography'], 'POINT ZM (1 2 3 4)', [ "geography(POINTZM,", "01010000C0000000000000F03F000000000000004000000000000008400000000000001040" ] ], ] for (geom_type, options, wkt, expected_strings) in tests: ds = ogr.GetDriverByName('PGDump').CreateDataSource( '/vsimem/ogr_pgdump_13.sql', options=['LINEFORMAT=LF']) lyr = ds.CreateLayer('test', geom_type=geom_type, options=options) f = ogr.Feature(lyr.GetLayerDefn()) f.SetGeometryDirectly(ogr.CreateGeometryFromWkt(wkt)) lyr.CreateFeature(f) f = None ds = None f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_13.sql', 'rb') sql = gdal.VSIFReadL(1, 10000, f).decode('utf8') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/ogr_pgdump_13.sql') for expected_string in expected_strings: if sql.find(expected_string) < 0: gdaltest.post_reason('fail') print(geom_type, options, wkt, expected_string) print(sql) return 'fail' if 'GEOM_TYPE=geography' in options: continue ds = ogr.GetDriverByName('PGDump').CreateDataSource( '/vsimem/ogr_pgdump_13.sql', options=['LINEFORMAT=LF']) lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone, options=options) lyr.CreateGeomField(ogr.GeomFieldDefn("my_geom", geom_type)) f = ogr.Feature(lyr.GetLayerDefn()) f.SetGeometryDirectly(ogr.CreateGeometryFromWkt(wkt)) lyr.CreateFeature(f) f = None ds = None f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_13.sql', 'rb') sql = gdal.VSIFReadL(1, 10000, f).decode('utf8') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/ogr_pgdump_13.sql') for expected_string in expected_strings: if sql.find(expected_string) < 0: gdaltest.post_reason('fail') print(geom_type, options, wkt, expected_string) print(sql) return 'fail' return 'success'
def ogr_pgdump_6(): ds = ogr.GetDriverByName('PGDump').CreateDataSource( '/vsimem/ogr_pgdump_6.sql', options=['LINEFORMAT=LF']) lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone) field_defn = ogr.FieldDefn('field_string', ogr.OFTString) field_defn.SetDefault("'a''b'") lyr.CreateField(field_defn) field_defn = ogr.FieldDefn('field_int', ogr.OFTInteger) field_defn.SetDefault('123') lyr.CreateField(field_defn) field_defn = ogr.FieldDefn('field_real', ogr.OFTReal) field_defn.SetDefault('1.23') lyr.CreateField(field_defn) field_defn = ogr.FieldDefn('field_nodefault', ogr.OFTInteger) lyr.CreateField(field_defn) field_defn = ogr.FieldDefn('field_datetime', ogr.OFTDateTime) field_defn.SetDefault("CURRENT_TIMESTAMP") lyr.CreateField(field_defn) field_defn = ogr.FieldDefn('field_datetime2', ogr.OFTDateTime) field_defn.SetDefault("'2015/06/30 12:34:56'") lyr.CreateField(field_defn) field_defn = ogr.FieldDefn('field_date', ogr.OFTDate) field_defn.SetDefault("CURRENT_DATE") lyr.CreateField(field_defn) field_defn = ogr.FieldDefn('field_time', ogr.OFTTime) field_defn.SetDefault("CURRENT_TIME") lyr.CreateField(field_defn) gdal.SetConfigOption('PG_USE_COPY', 'YES') f = ogr.Feature(lyr.GetLayerDefn()) f.SetField('field_string', 'a') f.SetField('field_int', 456) f.SetField('field_real', 4.56) f.SetField('field_datetime', '2015/06/30 12:34:56') f.SetField('field_datetime2', '2015/06/30 12:34:56') f.SetField('field_date', '2015/06/30') f.SetField('field_time', '12:34:56') lyr.CreateFeature(f) f = None # Transition from COPY to INSERT f = ogr.Feature(lyr.GetLayerDefn()) lyr.CreateFeature(f) f = None # Transition from INSERT to COPY f = ogr.Feature(lyr.GetLayerDefn()) f.SetField('field_string', 'b') f.SetField('field_int', 456) f.SetField('field_real', 4.56) f.SetField('field_datetime', '2015/06/30 12:34:56') f.SetField('field_datetime2', '2015/06/30 12:34:56') f.SetField('field_date', '2015/06/30') f.SetField('field_time', '12:34:56') lyr.CreateFeature(f) f = None gdal.SetConfigOption('PG_USE_COPY', None) ds = None f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_6.sql', 'rb') sql = gdal.VSIFReadL(1, 10000, f).decode('ascii') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/ogr_pgdump_6.sql') if sql.find("""a\t456\t4.56\t\\N\t2015/06/30 12:34:56\t2015/06/30 12:34:56\t2015/06/30\t12:34:56""") < 0 or \ sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_string" VARCHAR DEFAULT 'a''b';""") == -1 or \ sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_int" INTEGER DEFAULT 123;""") == -1 or \ sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_real" FLOAT8 DEFAULT 1.23;""") == -1 or \ sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_datetime" timestamp with time zone DEFAULT CURRENT_TIMESTAMP;""") == -1 or \ sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_datetime2" timestamp with time zone DEFAULT '2015/06/30 12:34:56+00'::timestamp with time zone;""") == -1 or \ sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_date" date DEFAULT CURRENT_DATE;""") == -1 or \ sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_time" time DEFAULT CURRENT_TIME;""") == -1 or \ sql.find("""b\t456\t4.56\t\\N\t2015/06/30 12:34:56\t2015/06/30 12:34:56\t2015/06/30\t12:34:56""") < 0: print(sql) return 'fail' return 'success'
def ogr_pgdump_8(): ds = ogr.GetDriverByName('PGDump').CreateDataSource( '/vsimem/ogr_pgdump_8.sql', options=['LINEFORMAT=LF']) lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone, options=['FID=myfid']) lyr.CreateField(ogr.FieldDefn('str', ogr.OFTString)) gdal.PushErrorHandler() ret = lyr.CreateField(ogr.FieldDefn('myfid', ogr.OFTString)) gdal.PopErrorHandler() if ret == 0: gdaltest.post_reason('fail') return 'fail' ret = lyr.CreateField(ogr.FieldDefn('myfid', ogr.OFTInteger)) if ret != 0: gdaltest.post_reason('fail') return 'fail' lyr.CreateField(ogr.FieldDefn('str2', ogr.OFTString)) feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField('str', 'first string') feat.SetField('myfid', 10) feat.SetField('str2', 'second string') gdal.SetConfigOption('PG_USE_COPY', 'YES') ret = lyr.CreateFeature(feat) gdal.SetConfigOption('PG_USE_COPY', None) if ret != 0: gdaltest.post_reason('fail') return 'fail' if feat.GetFID() != 10: gdaltest.post_reason('fail') return 'fail' feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField('str2', 'second string') gdal.SetConfigOption('PG_USE_COPY', 'YES') ret = lyr.CreateFeature(feat) gdal.SetConfigOption('PG_USE_COPY', None) if ret != 0: gdaltest.post_reason('fail') return 'fail' if feat.GetFID() < 0: gdaltest.post_reason('fail') feat.DumpReadable() return 'fail' if feat.GetField('myfid') != feat.GetFID(): gdaltest.post_reason('fail') feat.DumpReadable() return 'fail' # feat.SetField('str', 'foo') # ret = lyr.SetFeature(feat) # if ret != 0: # gdaltest.post_reason('fail') # return 'fail' feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetFID(1) feat.SetField('myfid', 10) gdal.PushErrorHandler() gdal.SetConfigOption('PG_USE_COPY', 'YES') ret = lyr.CreateFeature(feat) gdal.SetConfigOption('PG_USE_COPY', None) gdal.PopErrorHandler() if ret == 0: gdaltest.post_reason('fail') return 'fail' # gdal.PushErrorHandler() # ret = lyr.SetFeature(feat) # gdal.PopErrorHandler() # if ret == 0: # gdaltest.post_reason('fail') # return 'fail' # feat.UnsetField('myfid') # gdal.PushErrorHandler() # ret = lyr.SetFeature(feat) # gdal.PopErrorHandler() # if ret == 0: # gdaltest.post_reason('fail') # return 'fail' feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField('str', 'first string') feat.SetField('myfid', 12) feat.SetField('str2', 'second string') gdal.SetConfigOption('PG_USE_COPY', 'YES') ret = lyr.CreateFeature(feat) gdal.SetConfigOption('PG_USE_COPY', None) if ret != 0: gdaltest.post_reason('fail') return 'fail' if feat.GetFID() != 12: gdaltest.post_reason('fail') return 'fail' ds = None f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_8.sql', 'rb') sql = gdal.VSIFReadL(1, 10000, f).decode('ascii') gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/ogr_pgdump_8.sql') if sql.find("""CREATE TABLE "public"."test" ( "myfid" SERIAL, CONSTRAINT "test_pk" PRIMARY KEY ("myfid") )""") < 0 or \ sql.find("""ALTER TABLE "public"."test" ADD COLUMN "myfid" """) >= 0 or \ sql.find("""10\tfirst string\tsecond string""") == -1 or \ sql.find("""INSERT INTO "public"."test" ("str2") VALUES ('second string');""") == -1 or \ sql.find("""12\tfirst string\tsecond string""") == -1: print(sql) return 'fail' return 'success'
def test_vsigs_read_credentials_gce_expiration(): if gdaltest.webserver_port == 0: pytest.skip() if sys.platform not in ('linux', 'linux2', 'win32'): pytest.skip() gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '') gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '') gdal.SetConfigOption('GS_ACCESS_KEY_ID', '') gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL', 'http://localhost:%d/computeMetadata/v1/instance/service-accounts/default/token' % gdaltest.webserver_port) # Disable hypervisor related check to test if we are really on EC2 gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', 'NO') gdal.VSICurlClearCache() def method(request): if 'Authorization' not in request.headers: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) return expected_authorization = 'Bearer ACCESS_TOKEN' if request.headers['Authorization'] != expected_authorization: sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization'])) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.end_headers() request.wfile.write("""foo""".encode('ascii')) handler = webserver.SequentialHandler() # First time is used when trying to establish if GCE authentication is available handler.add('GET', '/computeMetadata/v1/instance/service-accounts/default/token', 200, {}, """{ "access_token" : "ACCESS_TOKEN", "token_type" : "Bearer", "expires_in" : 0, }""") # Second time is needed because f the access to th file handler.add('GET', '/computeMetadata/v1/instance/service-accounts/default/token', 200, {}, """{ "access_token" : "ACCESS_TOKEN", "token_type" : "Bearer", "expires_in" : 0, }""") handler.add('GET', '/gs_fake_bucket/resource', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsigs/gs_fake_bucket/resource') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo' gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL', '') gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', None)
def test_vsigs_read_credentials_file_refresh_token(): if gdaltest.webserver_port == 0: pytest.skip() gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '') gdal.SetConfigOption('GS_ACCESS_KEY_ID', '') gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '/vsimem/.boto') gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN', 'http://localhost:%d/accounts.google.com/o/oauth2/token' % gdaltest.webserver_port) gdal.VSICurlClearCache() gdal.FileFromMemBuffer('/vsimem/.boto', """ [Credentials] gs_oauth2_refresh_token = REFRESH_TOKEN [OAuth2] client_id = CLIENT_ID client_secret = CLIENT_SECRET """) handler = webserver.SequentialHandler() def method(request): content = request.rfile.read(int(request.headers['Content-Length'])).decode('ascii') if content != 'refresh_token=REFRESH_TOKEN&client_id=CLIENT_ID&client_secret=CLIENT_SECRET&grant_type=refresh_token': sys.stderr.write('Bad POST content: %s\n' % content) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') content = """{ "access_token" : "ACCESS_TOKEN", "token_type" : "Bearer", "expires_in" : 3600, }""" request.send_header('Content-Length', len(content)) request.end_headers() request.wfile.write(content.encode('ascii')) handler.add('POST', '/accounts.google.com/o/oauth2/token', custom_method=method) def method(request): if 'Authorization' not in request.headers: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) return expected_authorization = 'Bearer ACCESS_TOKEN' if request.headers['Authorization'] != expected_authorization: sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization'])) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.end_headers() request.wfile.write("""foo""".encode('ascii')) handler.add('GET', '/gs_fake_bucket/resource', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsigs/gs_fake_bucket/resource') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo' gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '') gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN', None) gdal.Unlink('/vsimem/.boto')
def test_vsigs_read_credentials_oauth2_service_account_json_file(): if gdaltest.webserver_port == 0: pytest.skip() gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '') gdal.SetConfigOption('GS_ACCESS_KEY_ID', '') gdal.FileFromMemBuffer('/vsimem/service_account.json', """{ "private_key": "-----BEGIN PRIVATE KEY-----\nMIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk\nVNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm\ni2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ\nDVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy\nTabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I\n2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd\nDrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+\nbpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg\nWz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3\n4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5\nKvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj\nj+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI\nyzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq\ngwE6fxOLyJDxuWRf\n-----END PRIVATE KEY-----\n", "client_email": "CLIENT_EMAIL" }""") gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', '/vsimem/service_account.json') gdal.SetConfigOption('GO2A_AUD', 'http://localhost:%d/oauth2/v4/token' % gdaltest.webserver_port) gdal.SetConfigOption('GOA2_NOW', '123456') gdal.VSICurlClearCache() handler = webserver.SequentialHandler() def method(request): content = request.rfile.read(int(request.headers['Content-Length'])).decode('ascii') content_8080 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODAvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.DAhqWtBgKpObxZ%2BGiXqwF%2Fa4SS%2FNWQRhLCI7DYZCuOTuf2w7dL8j4CdpiwwzQg1diIus7dyViRfzpsFmuZKAXwL%2B84iBoVVqnJJZ4TgwH49NdfMAnc4Rgm%2Bo2a2nEcMjX%2FbQ3jRY%2B9WNVl96hzULGvLrVeyego2f06wivqmvxHA%3D' content_8081 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODEvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.0abOEg4%2FRApWTSeAs6YTHaNzdwOgZLm8DTMO2MKlOA%2Fiagyb4cBJxDpkD5gECPvi7qhkg7LsyFuj0a%2BK48Bsuj%2FgLHOU4MpB0dHwYnDO2UXzH%2FUPdgFCVak1P1V%2ByiDA%2B%2Ft4aDI5fD9qefKQiu3wsMDHzP71MNLzayrjqaqKKS4%3D' if content not in [content_8080, content_8081]: sys.stderr.write('Bad POST content: %s\n' % content) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') content = """{ "access_token" : "ACCESS_TOKEN", "token_type" : "Bearer", "expires_in" : 3600, }""" request.send_header('Content-Length', len(content)) request.end_headers() request.wfile.write(content.encode('ascii')) handler.add('POST', '/oauth2/v4/token', custom_method=method) def method(request): if 'Authorization' not in request.headers: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) return expected_authorization = 'Bearer ACCESS_TOKEN' if request.headers['Authorization'] != expected_authorization: sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization'])) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.end_headers() request.wfile.write("""foo""".encode('ascii')) handler.add('GET', '/gs_fake_bucket/resource', custom_method=method) try: with webserver.install_http_handler(handler): f = open_for_read('/vsigs/gs_fake_bucket/resource') if f is None: gdal.Unlink('/vsimem/service_account.json') pytest.fail() data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) signed_url = gdal.GetSignedURL('/vsigs/gs_fake_bucket/resource', ['START_DATE=20180212T123456Z']) if signed_url not in ('http://127.0.0.1:8080/gs_fake_bucket/resource?Expires=1518442496&GoogleAccessId=CLIENT_EMAIL&Signature=b19I62KdqV51DpWGxhxGXLGJIA8MHvSJofwOygoeQuIxkM6PmmQFvJYTNWRt9zUVTUoVC0UHVB7ee5Z35NqDC8K4i0quu1hb8Js2B4h0W6OAupvyF3nSQ5D0OJmiSbomGMq0Ehyro5cqJ%2FU%2Fd8oAaKrGKVQScKfXoFrSJBbWkNs%3D', 'http://127.0.0.1:8081/gs_fake_bucket/resource?Expires=1518442496&GoogleAccessId=CLIENT_EMAIL&Signature=b19I62KdqV51DpWGxhxGXLGJIA8MHvSJofwOygoeQuIxkM6PmmQFvJYTNWRt9zUVTUoVC0UHVB7ee5Z35NqDC8K4i0quu1hb8Js2B4h0W6OAupvyF3nSQ5D0OJmiSbomGMq0Ehyro5cqJ%2FU%2Fd8oAaKrGKVQScKfXoFrSJBbWkNs%3D'): gdal.Unlink('/vsimem/service_account.json') pytest.fail(signed_url) except: if gdal.GetLastErrorMsg().find('CPLRSASHA256Sign() not implemented') >= 0: pytest.skip() finally: gdal.SetConfigOption('GO2A_AUD', None) gdal.SetConfigOption('GO2A_NOW', None) gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', '') gdal.Unlink('/vsimem/service_account.json') assert data == 'foo'
def validate(ds, check_tiled=True, full_check=False): """Check if a file is a (Geo)TIFF with cloud optimized compatible structure. Args: ds: GDAL Dataset for the file to inspect. check_tiled: Set to False to ignore missing tiling. full_check: Set to TRUe to check tile/strip leader/trailer bytes. Might be slow on remote files Returns: A tuple, whose first element is an array of error messages (empty if there is no error), and the second element, a dictionary with the structure of the GeoTIFF file. Raises: ValidateCloudOptimizedGeoTIFFException: Unable to open the file or the file is not a Tiff. """ if int(gdal.VersionInfo('VERSION_NUM')) < 2020000: raise ValidateCloudOptimizedGeoTIFFException( 'GDAL 2.2 or above required') unicode_type = type(''.encode('utf-8').decode('utf-8')) if isinstance(ds, (str, unicode_type)): gdal.PushErrorHandler() ds = gdal.Open(ds) gdal.PopErrorHandler() if ds is None: raise ValidateCloudOptimizedGeoTIFFException( 'Invalid file : %s' % gdal.GetLastErrorMsg()) if ds.GetDriver().ShortName != 'GTiff': raise ValidateCloudOptimizedGeoTIFFException( 'The file is not a GeoTIFF') details = {} errors = [] warnings = [] filename = ds.GetDescription() main_band = ds.GetRasterBand(1) ovr_count = main_band.GetOverviewCount() filelist = ds.GetFileList() if filelist is not None and filename + '.ovr' in filelist: errors += [ 'Overviews found in external .ovr file. They should be internal' ] if main_band.XSize > 512 or main_band.YSize > 512: if check_tiled: block_size = main_band.GetBlockSize() if block_size[0] == main_band.XSize and block_size[0] > 1024: errors += [ 'The file is greater than 512xH or Wx512, but is not tiled' ] if ovr_count == 0: warnings += [ 'The file is greater than 512xH or Wx512, it is recommended ' 'to include internal overviews' ] ifd_offset = int(main_band.GetMetadataItem('IFD_OFFSET', 'TIFF')) ifd_offsets = [ifd_offset] block_order_row_major = False block_leader_size_as_uint4 = False block_trailer_last_4_bytes_repeated = False mask_interleaved_with_imagery = False if ifd_offset not in (8, 16): # Check if there is GDAL hidden structural metadata f = gdal.VSIFOpenL(filename, 'rb') if not f: raise ValidateCloudOptimizedGeoTIFFException("Cannot open file") signature = struct.unpack('B' * 4, gdal.VSIFReadL(4, 1, f)) bigtiff = signature in ((0x49, 0x49, 0x2B, 0x00), (0x4D, 0x4D, 0x00, 0x2B)) if bigtiff: expected_ifd_pos = 16 else: expected_ifd_pos = 8 gdal.VSIFSeekL(f, expected_ifd_pos, 0) pattern = "GDAL_STRUCTURAL_METADATA_SIZE=%06d bytes\n" % 0 got = gdal.VSIFReadL(len(pattern), 1, f).decode('LATIN1') if len(got) == len(pattern) and got.startswith( 'GDAL_STRUCTURAL_METADATA_SIZE='): size = int(got[len('GDAL_STRUCTURAL_METADATA_SIZE='):][0:6]) extra_md = gdal.VSIFReadL(size, 1, f).decode('LATIN1') block_order_row_major = 'BLOCK_ORDER=ROW_MAJOR' in extra_md block_leader_size_as_uint4 = 'BLOCK_LEADER=SIZE_AS_UINT4' in extra_md block_trailer_last_4_bytes_repeated = 'BLOCK_TRAILER=LAST_4_BYTES_REPEATED' in extra_md mask_interleaved_with_imagery = 'MASK_INTERLEAVED_WITH_IMAGERY=YES' in extra_md if 'KNOWN_INCOMPATIBLE_EDITION=YES' in extra_md: errors += [ "KNOWN_INCOMPATIBLE_EDITION=YES is declared in the file" ] expected_ifd_pos += len(pattern) + size expected_ifd_pos += expected_ifd_pos % 2 # IFD offset starts on a 2-byte boundary gdal.VSIFCloseL(f) if expected_ifd_pos != ifd_offsets[0]: errors += [ 'The offset of the main IFD should be %d. It is %d instead' % (expected_ifd_pos, ifd_offsets[0]) ] details['ifd_offsets'] = {} details['ifd_offsets']['main'] = ifd_offset for i in range(ovr_count): # Check that overviews are by descending sizes ovr_band = ds.GetRasterBand(1).GetOverview(i) if i == 0: if (ovr_band.XSize > main_band.XSize or ovr_band.YSize > main_band.YSize): errors += [ 'First overview has larger dimension than main band' ] else: prev_ovr_band = ds.GetRasterBand(1).GetOverview(i - 1) if (ovr_band.XSize > prev_ovr_band.XSize or ovr_band.YSize > prev_ovr_band.YSize): errors += [ 'Overview of index %d has larger dimension than ' 'overview of index %d' % (i, i - 1) ] if check_tiled: block_size = ovr_band.GetBlockSize() if block_size[0] == ovr_band.XSize and block_size[0] > 1024: errors += ['Overview of index %d is not tiled' % i] # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int(ovr_band.GetMetadataItem('IFD_OFFSET', 'TIFF')) ifd_offsets.append(ifd_offset) details['ifd_offsets']['overview_%d' % i] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if i == 0: errors += [ 'The offset of the IFD for overview of index %d is %d, ' 'whereas it should be greater than the one of the main ' 'image, which is at byte %d' % (i, ifd_offsets[-1], ifd_offsets[-2]) ] else: errors += [ 'The offset of the IFD for overview of index %d is %d, ' 'whereas it should be greater than the one of index %d, ' 'which is at byte %d' % (i, ifd_offsets[-1], i - 1, ifd_offsets[-2]) ] # Check that the imagery starts by the smallest overview and ends with # the main resolution dataset def get_block_offset(band): blockxsize, blockysize = band.GetBlockSize() for y in range(int((band.YSize + blockysize - 1) / blockysize)): for x in range(int((band.XSize + blockxsize - 1) / blockxsize)): block_offset = band.GetMetadataItem( 'BLOCK_OFFSET_%d_%d' % (x, y), 'TIFF') if block_offset: return int(block_offset) return 0 block_offset = get_block_offset(main_band) data_offsets = [block_offset] details['data_offsets'] = {} details['data_offsets']['main'] = block_offset for i in range(ovr_count): ovr_band = ds.GetRasterBand(1).GetOverview(i) block_offset = get_block_offset(ovr_band) data_offsets.append(block_offset) details['data_offsets']['overview_%d' % i] = block_offset if data_offsets[-1] != 0 and data_offsets[-1] < ifd_offsets[-1]: if ovr_count > 0: errors += [ 'The offset of the first block of the smallest overview ' 'should be after its IFD' ] else: errors += [ 'The offset of the first block of the image should ' 'be after its IFD' ] for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] != 0 and data_offsets[i] < data_offsets[i + 1]: errors += [ 'The offset of the first block of overview of index %d should ' 'be after the one of the overview of index %d' % (i - 1, i) ] if len(data_offsets) >= 2 and data_offsets[0] != 0 and data_offsets[ 0] < data_offsets[1]: errors += [ 'The offset of the first block of the main resolution image ' 'should be after the one of the overview of index %d' % (ovr_count - 1) ] if full_check and (block_order_row_major or block_leader_size_as_uint4 or block_trailer_last_4_bytes_repeated or mask_interleaved_with_imagery): f = gdal.VSIFOpenL(filename, 'rb') if not f: raise ValidateCloudOptimizedGeoTIFFException("Cannot open file") full_check_band(f, 'Main resolution image', main_band, errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, mask_interleaved_with_imagery) if main_band.GetMaskFlags() == gdal.GMF_PER_DATASET and \ (filename + '.msk') not in ds.GetFileList(): full_check_band(f, 'Mask band of main resolution image', main_band.GetMaskBand(), errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, False) for i in range(ovr_count): ovr_band = ds.GetRasterBand(1).GetOverview(i) full_check_band(f, 'Overview %d' % i, ovr_band, errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, mask_interleaved_with_imagery) if ovr_band.GetMaskFlags() == gdal.GMF_PER_DATASET and \ (filename + '.msk') not in ds.GetFileList(): full_check_band(f, 'Mask band of overview %d' % i, ovr_band.GetMaskBand(), errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, False) gdal.VSIFCloseL(f) return warnings, errors, details
def ogr_vdv_2(src_filename='tmp/test.x10'): out_filename = '/vsimem/vdv/ogr_vdv_2.x10' gdal.Unlink(out_filename) src_ds = ogr.Open(src_filename) out_ds = ogr.GetDriverByName('VDV').CreateDataSource(out_filename) layer_names = [ src_ds.GetLayer(idx).GetName() for idx in range(src_ds.GetLayerCount()) ] layer_names.sort() for layer_name in layer_names: src_lyr = src_ds.GetLayer(layer_name) options = [ 'HEADER_SRC_DATE=01.01.1970', 'HEADER_SRC_TIME=00.00.00', 'HEADER_foo=bar' ] dst_lyr = out_ds.CreateLayer(src_lyr.GetName(), options=options) for field_idx in range(src_lyr.GetLayerDefn().GetFieldCount()): dst_lyr.CreateField(src_lyr.GetLayerDefn().GetFieldDefn(field_idx)) for src_f in src_lyr: dst_f = ogr.Feature(dst_lyr.GetLayerDefn()) dst_f.SetFrom(src_f) dst_lyr.CreateFeature(dst_f) out_ds = None expected = """mod; DD.MM.YYYY; HH:MM:SS; free src; "UNKNOWN"; "01.01.1970"; "00.00.00" chs; "ISO8859-1" ver; "1.4" ifv; "1.4" dve; "1.4" fft; "" foo; "bar" tbl; another_layer atr; str_field frm; char[80] rec; "0" rec; "1" rec; "2" rec; "3" rec; "4" end; 5 tbl; lyr_1 atr; str_field; int_field; int64_field; bool_field; str2_field; int2_field frm; char[80]; num[10.0]; num[19.0]; boolean; char[2]; num[1.0] rec; "a""b"; 12; NULL; 1; NULL; NULL rec; NULL; NULL; NULL; NULL; NULL; NULL end; 2 tbl; empty atr; frm; end; 0 tbl; empty2 atr; frm; end; 0 eof; 4 """ f = gdal.VSIFOpenL(out_filename, 'rb') got = gdal.VSIFReadL(1, 10000, f).decode('latin1') gdal.VSIFCloseL(f) if got != expected: gdaltest.post_reason('fail') print(got) return 'fail' gdal.Unlink(out_filename) return 'success'
def full_check_band(f, band_name, band, errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, mask_interleaved_with_imagery): block_size = band.GetBlockSize() mask_band = None if mask_interleaved_with_imagery: mask_band = band.GetMaskBand() mask_block_size = mask_band.GetBlockSize() if block_size != mask_block_size: errors += [ band_name + ': mask block size is different from its imagery band' ] mask_band = None yblocks = (band.YSize + block_size[1] - 1) // block_size[1] xblocks = (band.XSize + block_size[0] - 1) // block_size[0] last_offset = 0 for y in range(yblocks): for x in range(xblocks): offset = band.GetMetadataItem('BLOCK_OFFSET_%d_%d' % (x, y), 'TIFF') offset = int(offset) if offset is not None else 0 bytecount = band.GetMetadataItem('BLOCK_SIZE_%d_%d' % (x, y), 'TIFF') bytecount = int(bytecount) if bytecount is not None else 0 if offset > 0: if block_order_row_major and offset < last_offset: errors += [ band_name + ': offset of block (%d, %d) is smaller than previous block' % (x, y) ] if block_leader_size_as_uint4: gdal.VSIFSeekL(f, offset - 4, 0) leader_size = struct.unpack('<I', gdal.VSIFReadL(4, 1, f))[0] if leader_size != bytecount: errors += [ band_name + ': for block (%d, %d), size in leader bytes is %d instead of %d' % (x, y, leader_size, bytecount) ] if block_trailer_last_4_bytes_repeated: if bytecount >= 4: gdal.VSIFSeekL(f, offset + bytecount - 4, 0) last_bytes = gdal.VSIFReadL(8, 1, f) if last_bytes[0:4] != last_bytes[4:8]: errors += [ band_name + ': for block (%d, %d), trailer bytes are invalid' % (x, y) ] if mask_band: offset_mask = mask_band.GetMetadataItem( 'BLOCK_OFFSET_%d_%d' % (x, y), 'TIFF') offset_mask = int( offset_mask) if offset_mask is not None else 0 if offset > 0 and offset_mask > 0: #bytecount_mask = int(mask_band.GetMetadataItem('BLOCK_SIZE_%d_%d' % (x,y), 'TIFF')) expected_offset_mask = offset + bytecount + \ (4 if block_leader_size_as_uint4 else 0) + \ (4 if block_trailer_last_4_bytes_repeated else 0) if offset_mask != expected_offset_mask: errors += [ 'Mask of ' + band_name + ': for block (%d, %d), offset is %d, whereas %d was expected' % (x, y, offset_mask, expected_offset_mask) ] elif offset == 0 and offset_mask > 0: if block_order_row_major and offset_mask < last_offset: errors += [ 'Mask of ' + band_name + ': offset of block (%d, %d) is smaller than previous block' % (x, y) ] offset = offset_mask last_offset = offset
def ogr_vdv_8(): gdal.PushErrorHandler() ds = ogr.GetDriverByName('VDV').CreateDataSource('/does/not_exist') gdal.PopErrorHandler() if ds is not None: gdaltest.post_reason('fail') return 'fail' gdal.PushErrorHandler() ds = ogr.GetDriverByName('VDV').CreateDataSource( '/does/not_exist', options=['SINGLE_FILE=FALSE']) gdal.PopErrorHandler() if ds is not None: gdaltest.post_reason('fail') return 'fail' # Add layer in non writable directory if sys.platform.startswith('linux'): os.mkdir('tmp/ogr_vdv_8') open('tmp/ogr_vdv_8/empty.x10', 'wb').write('tbl; foo\natr;\nfrm;\n'.encode('latin1')) # 0555 = 365 os.chmod('tmp/ogr_vdv_8', 365) try: open('tmp/ogr_vdv_8/another_file', 'wb').close() shutil.rmtree('tmp/ogr_vdv_8') do_test = False except: do_test = True if do_test: ds = ogr.Open('tmp/ogr_vdv_8', update=1) gdal.PushErrorHandler() lyr = ds.CreateLayer('another_layer') gdal.PopErrorHandler() # 0755 = 493 os.chmod('tmp/ogr_vdv_8', 493) ds = None shutil.rmtree('tmp/ogr_vdv_8') if lyr is not None: gdaltest.post_reason('fail') return 'fail' out_filename = '/vsimem/vdv/ogr_vdv_8.x10' ds = ogr.GetDriverByName('VDV').CreateDataSource(out_filename) # File already exists gdal.PushErrorHandler() ds2 = ogr.GetDriverByName('VDV').CreateDataSource(out_filename) gdal.PopErrorHandler() if ds2 is not None: gdaltest.post_reason('fail') return 'fail' if ds.TestCapability(ogr.ODsCCreateLayer) != 1: gdaltest.post_reason('fail') return 'fail' lyr1 = ds.CreateLayer("lyr1") if lyr1.TestCapability(ogr.OLCSequentialWrite) != 1: gdaltest.post_reason('fail') return 'fail' if lyr1.TestCapability(ogr.OLCCreateField) != 1: gdaltest.post_reason('fail') return 'fail' lyr1.ResetReading() gdal.PushErrorHandler() lyr1.GetNextFeature() gdal.PopErrorHandler() lyr1.CreateFeature(ogr.Feature(lyr1.GetLayerDefn())) # Layer structure is now frozen if lyr1.TestCapability(ogr.OLCCreateField) != 0: gdaltest.post_reason('fail') return 'fail' gdal.PushErrorHandler() ret = lyr1.CreateField(ogr.FieldDefn('not_allowed')) gdal.PopErrorHandler() if ret == 0: gdaltest.post_reason('fail') return 'fail' lyr2 = ds.CreateLayer("lyr2") lyr2.CreateFeature(ogr.Feature(lyr2.GetLayerDefn())) # Test interleaved writing if lyr1.TestCapability(ogr.OLCSequentialWrite) != 0: gdaltest.post_reason('fail') return 'fail' gdal.PushErrorHandler() ret = lyr1.CreateFeature(ogr.Feature(lyr1.GetLayerDefn())) gdal.PopErrorHandler() if ret == 0: gdaltest.post_reason('fail') return 'fail' if lyr1.GetFeatureCount() != 1: gdaltest.post_reason('fail') return 'fail' ds = None # Test appending new layer to file without eof gdal.FileFromMemBuffer(out_filename, 'tbl; foo\natr; atr\nfrm; char[40]\nrec; "foo"\n') ds = ogr.Open(out_filename, update=1) lyr = ds.CreateLayer('new_layer') lyr.CreateField(ogr.FieldDefn('atr')) f = ogr.Feature(lyr.GetLayerDefn()) f['atr'] = 'bar' lyr.CreateFeature(f) f = None ds = None expected = """tbl; foo atr; atr frm; char[40] rec; "foo" tbl; new_layer atr; atr frm; char[80] rec; "bar" end; 1 eof; 2 """ f = gdal.VSIFOpenL(out_filename, 'rb') got = gdal.VSIFReadL(1, 10000, f).decode('latin1') gdal.VSIFCloseL(f) if got != expected: gdaltest.post_reason('fail') print(got) return 'fail' # Test we are robust against missing end; ds = ogr.Open(out_filename) for i in range(2): lyr = ds.GetLayer(i) if lyr.GetFeatureCount() != 1: gdaltest.post_reason('fail') return 'fail' lyr.ResetReading() fc = 0 for f in lyr: fc += 1 if fc != 1: gdaltest.post_reason('fail') return 'fail' lyr = None ds = None # Test appending new layer to file without terminating \n gdal.FileFromMemBuffer( out_filename, 'tbl; foo\natr; atr\nfrm; char[40]\nrec; "foo"\neof; 1') ds = ogr.Open(out_filename, update=1) lyr = ds.CreateLayer('new_layer') lyr.CreateField(ogr.FieldDefn('atr')) f = ogr.Feature(lyr.GetLayerDefn()) f['atr'] = 'bar' lyr.CreateFeature(f) f = None ds = None f = gdal.VSIFOpenL(out_filename, 'rb') got = gdal.VSIFReadL(1, 10000, f).decode('latin1') gdal.VSIFCloseL(f) if got != expected: gdaltest.post_reason('fail') print(got) return 'fail' gdal.Unlink(out_filename) return 'success'
def eedai_4(): if gdaltest.eedai_drv is None: return 'skip' gdal.FileFromMemBuffer( '/vsimem/ee/assets/image', json.dumps({ 'type': 'IMAGE', 'bands': [{ "id": "B1", "dataType": { "precision": "INTEGER", "range": { "max": 255 } }, "grid": { "crsCode": "EPSG:32610", "affineTransform": { "translateX": 499980, "translateY": 4200000, "scaleX": 60, "scaleY": -60 }, "dimensions": { "width": 1830, "height": 1831 } } }, { "id": "B2", "dataType": { "precision": "INTEGER", "range": { "max": 255 } }, "grid": { "crsCode": "EPSG:32610", "affineTransform": { "translateX": 499980, "translateY": 4200000, "scaleX": 60, "scaleY": -60 }, "dimensions": { "width": 1830, "height": 1831 } } }, { "id": "B3", "dataType": { "precision": "INTEGER", "range": { "max": 255 } }, "grid": { "crsCode": "EPSG:32610", "affineTransform": { "translateX": 499980, "translateY": 4200000, "scaleX": 60, "scaleY": -60 }, "dimensions": { "width": 1830, "height": 1831 } } }] })) gdal.SetConfigOption('EEDA_BEARER', 'mybearer') gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/') ds = gdal.Open('EEDAI:image') gdal.SetConfigOption('EEDA_URL', None) mem_ds = gdal.GetDriverByName('MEM').Create('', 256, 256, 3) mem_ds.GetRasterBand(1).Fill(127) mem_ds.GetRasterBand(2).Fill(128) mem_ds.GetRasterBand(3).Fill(129) gdal.GetDriverByName('PNG').CreateCopy('/vsimem/out.png', mem_ds) f = gdal.VSIFOpenL('/vsimem/out.png', 'rb') png_data = gdal.VSIFReadL(1, 1000000, f) gdal.VSIFCloseL(f) gdal.Unlink('/vsimem/out.png') gdal.FileFromMemBuffer( '/vsimem/ee/assets:getPixels&CUSTOMREQUEST=POST&POSTFIELDS={ "path": "image", "encoding": "PNG", "bandIds": [ "B1", "B2", "B3" ], "grid": { "origin": { "x": 499980.0, "y": 4200000.0 }, "pixelSize": { "x": 60.0, "y": -60.0 }, "dimensions": { "width": 256, "height": 256 } } }', png_data) got_data = ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1) got_data = struct.unpack('B', got_data)[0] if got_data != 127: gdaltest.post_reason('fail') print(got_data) return 'fail' # Same with dataset RasterIO got_data = ds.ReadRaster(0, 0, 1, 1) got_data = struct.unpack('B' * 3, got_data) if got_data != (127, 128, 129): gdaltest.post_reason('fail') print(got_data) return 'fail' # Same after flushing cache ds.FlushCache() got_data = ds.ReadRaster(0, 0, 1, 1) got_data = struct.unpack('B' * 3, got_data) if got_data != (127, 128, 129): gdaltest.post_reason('fail') print(got_data) return 'fail' # Sub-sampled query gdal.FileFromMemBuffer( '/vsimem/ee/assets:getPixels&CUSTOMREQUEST=POST&POSTFIELDS={ "path": "image", "encoding": "PNG", "bandIds": [ "B1", "B2", "B3" ], "grid": { "origin": { "x": 499980.0, "y": 4200000.0 }, "pixelSize": { "x": 120.0, "y": -120.06557377049181 }, "dimensions": { "width": 256, "height": 256 } } }', png_data) got_data = ds.GetRasterBand(1).ReadRaster(0, 0, 2, 2, buf_xsize=1, buf_ysize=1) got_data = struct.unpack('B', got_data)[0] if got_data != 127: gdaltest.post_reason('fail') print(got_data) return 'fail' # Same after flushing cache with dataset RasterIO ds.FlushCache() got_data = ds.ReadRaster(0, 0, 2, 2, buf_xsize=1, buf_ysize=1) got_data = struct.unpack('B' * 3, got_data) if got_data != (127, 128, 129): gdaltest.post_reason('fail') print(got_data) return 'fail' ds = None gdal.SetConfigOption('EEDA_BEARER', None) return 'success'
def jpeg_19(): for (width, height, iX) in [ (32, 32, 12), (25, 25, 8), (24, 25, 8) ]: src_ds = gdal.GetDriverByName('GTiff').Create('/vsimem/jpeg_19.tif', width, height, 1) src_ds.CreateMaskBand(gdal.GMF_PER_DATASET) src_ds.GetRasterBand(1).GetMaskBand().WriteRaster( 0, 0, iX,height, struct.pack('B' * 1, 255), 1, 1) src_ds.GetRasterBand(1).GetMaskBand().WriteRaster( iX, 0, width-iX, height, struct.pack('B' * 1, 0), 1, 1) tiff_mask_data = src_ds.GetRasterBand(1).GetMaskBand().ReadRaster( 0, 0, width, height) # Generate a JPEG file with a (default) LSB bit mask order out_ds = gdal.GetDriverByName('JPEG').CreateCopy('/vsimem/jpeg_19.jpg', src_ds) out_ds = None # Generate a JPEG file with a MSB bit mask order gdal.SetConfigOption('JPEG_WRITE_MASK_BIT_ORDER', 'MSB') out_ds = gdal.GetDriverByName('JPEG').CreateCopy( '/vsimem/jpeg_19_msb.jpg', src_ds) del out_ds gdal.SetConfigOption('JPEG_WRITE_MASK_BIT_ORDER', None) src_ds = None # Check that the file are indeed different statBuf = gdal.VSIStatL('/vsimem/jpeg_19.jpg') f = gdal.VSIFOpenL('/vsimem/jpeg_19.jpg', 'rb') data1 = gdal.VSIFReadL(1, statBuf.size, f) gdal.VSIFCloseL(f) statBuf = gdal.VSIStatL('/vsimem/jpeg_19_msb.jpg') f = gdal.VSIFOpenL('/vsimem/jpeg_19_msb.jpg', 'rb') data2 = gdal.VSIFReadL(1, statBuf.size, f) gdal.VSIFCloseL(f) if (width, height, iX) == (24, 25, 8): if data1 != data2: gdaltest.post_reason('fail') return 'fail' else: if data1 == data2: gdaltest.post_reason('fail') return 'fail' # Check the file with the LSB bit mask order ds = gdal.Open('/vsimem/jpeg_19.jpg') jpg_mask_data = ds.GetRasterBand(1).GetMaskBand().ReadRaster( 0, 0, width, height) ds = None if tiff_mask_data != jpg_mask_data: gdaltest.post_reason('fail') return 'fail' # Check the file with the MSB bit mask order ds = gdal.Open('/vsimem/jpeg_19_msb.jpg') jpg_mask_data = ds.GetRasterBand(1).GetMaskBand().ReadRaster( 0, 0, width, height) ds = None if tiff_mask_data != jpg_mask_data: gdaltest.post_reason('fail') return 'fail' gdal.GetDriverByName('GTiff').Delete('/vsimem/jpeg_19.tif') gdal.GetDriverByName('JPEG').Delete('/vsimem/jpeg_19.jpg') gdal.GetDriverByName('JPEG').Delete('/vsimem/jpeg_19_msb.jpg') return 'success'
def vsigs_1(): try: drv = gdal.GetDriverByName( 'HTTP' ) except: drv = None if drv is None: return 'skip' # RETODO: Bind to swig, change test # Invalid header filename gdal.ErrorReset() gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', '/i_dont/exist.py') f = open_for_read('/vsigs/foo/bar') if f is None: gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None) gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): data = gdal.VSIFReadL(1, 1, f) last_err = gdal.GetLastErrorMsg() gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None) gdal.VSIFCloseL(f) if len(data) != 0: gdaltest.post_reason('fail') return 'fail' if last_err.find('Cannot read') < 0: gdaltest.post_reason('fail') print(last_err) return 'fail' # Invalid content for header file gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', 'vsigs.py') f = open_for_read('/vsigs/foo/bar') if f is None: gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None) gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 1, f) gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None) gdal.VSIFCloseL(f) if len(data) != 0: gdaltest.post_reason('fail') return 'fail' # Missing GS_SECRET_ACCESS_KEY gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsigs/foo/bar') if f is not None or gdal.VSIGetLastErrorMsg().find('GS_SECRET_ACCESS_KEY') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsigs_streaming/foo/bar') if f is not None or gdal.VSIGetLastErrorMsg().find('GS_SECRET_ACCESS_KEY') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY') # Missing GS_ACCESS_KEY_ID gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsigs/foo/bar') if f is not None or gdal.VSIGetLastErrorMsg().find('GS_ACCESS_KEY_ID') < 0: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID') # ERROR 1: The User Id you provided does not exist in our records. gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsigs/foo/bar.baz') if f is not None or gdal.VSIGetLastErrorMsg() == '': if f is not None: gdal.VSIFCloseL(f) if gdal.GetConfigOption('APPVEYOR') is not None: return 'success' gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): f = open_for_read('/vsigs_streaming/foo/bar.baz') if f is not None or gdal.VSIGetLastErrorMsg() == '': gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' return 'success'
def vsicrypt_4(): if not gdaltest.has_vsicrypt: return 'skip' test_file = '/vsicrypt/key=DONT_USE_IN_PROD,sector_size=32,file=/vsimem/file_enc.bin' ref_file = '/vsimem/file.bin' for seed in range(1000): gdal.Unlink(test_file) gdal.Unlink(ref_file) test_f = gdal.VSIFOpenL(test_file, 'wb+') ref_f = gdal.VSIFOpenL(ref_file, 'wb+') import random random.seed(seed) for _ in range(20): random_offset = random.randint(0, 1000) gdal.VSIFSeekL(test_f, random_offset, 0) gdal.VSIFSeekL(ref_f, random_offset, 0) random_size = random.randint(1, 80) random_content = ''.join([chr(40 + int(10 * random.random())) for _ in range(random_size)]) gdal.VSIFWriteL(random_content, 1, random_size, test_f) gdal.VSIFWriteL(random_content, 1, random_size, ref_f) if random.randint(0, 1) == 0: random_offset = random.randint(0, 1500) gdal.VSIFSeekL(test_f, random_offset, 0) gdal.VSIFSeekL(ref_f, random_offset, 0) random_size = random.randint(1, 80) test_content = gdal.VSIFReadL(1, random_size, test_f) ref_content = gdal.VSIFReadL(1, random_size, ref_f) if test_content != ref_content: print(seed) print('Test content (%d):' % len(test_content)) print(test_content) print('') print('Ref content (%d):' % len(ref_content)) print(ref_content) return 'fail' gdal.VSIFSeekL(test_f, 0, 0) gdal.VSIFSeekL(ref_f, 0, 0) test_content = gdal.VSIFReadL(1, 100000, test_f) ref_content = gdal.VSIFReadL(1, 100000, ref_f) if test_content != ref_content: print(seed) print('Test content (%d):' % len(test_content)) print(test_content) print('') print('Ref content (%d):' % len(ref_content)) print(ref_content) return 'fail' gdal.Unlink(test_file) gdal.Unlink(ref_file) return 'success'
def vsigs_extra_1(): try: drv = gdal.GetDriverByName( 'HTTP' ) except: drv = None if drv is None: return 'skip' if gdal.GetConfigOption('GS_SECRET_ACCESS_KEY') is None: print('Missing GS_SECRET_ACCESS_KEY for running gdaltest_list_extra') return 'skip' elif gdal.GetConfigOption('GS_ACCESS_KEY_ID') is None: print('Missing GS_ACCESS_KEY_ID for running gdaltest_list_extra') return 'skip' elif gdal.GetConfigOption('GS_RESOURCE') is None: print('Missing GS_RESOURCE for running gdaltest_list_extra') return 'skip' f = open_for_read('/vsigs/' + gdal.GetConfigOption('GS_RESOURCE')) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' # Same with /vsigs_streaming/ f = open_for_read('/vsigs_streaming/' + gdal.GetConfigOption('GS_RESOURCE')) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' # Invalid bucket : "The specified bucket does not exist" gdal.ErrorReset() f = open_for_read('/vsigs/not_existing_bucket/foo') with gdaltest.error_handler(): gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if gdal.VSIGetLastErrorMsg() == '': gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' # Invalid resource gdal.ErrorReset() f = open_for_read('/vsigs_streaming/' + gdal.GetConfigOption('GS_RESOURCE') + '/invalid_resource.baz') if f is not None: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' return 'success'
def vsicrypt_6(): try: import ctypes except ImportError: return 'skip' import testnonboundtoswig testnonboundtoswig.testnonboundtoswig_init() if testnonboundtoswig.gdal_handle is None: return 'skip' testnonboundtoswig.gdal_handle.VSISetCryptKey.argtypes = [ctypes.c_char_p, ctypes.c_int] testnonboundtoswig.gdal_handle.VSISetCryptKey.restype = None # Set a valid key testnonboundtoswig.gdal_handle.VSISetCryptKey('DONT_USE_IN_PROD'.encode('ASCII'), 16) if not gdaltest.has_vsicrypt: return 'skip' fp = gdal.VSIFOpenL('/vsicrypt/add_key_check=yes,file=/vsimem/file.bin', 'wb+') if fp is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') return 'fail' fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+') if fp is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') return 'fail' # Set a too short key testnonboundtoswig.gdal_handle.VSISetCryptKey('bbc'.encode('ASCII'), 3) with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Erase key testnonboundtoswig.gdal_handle.VSISetCryptKey(None, 0) with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+') if fp is not None: gdaltest.post_reason('fail') return 'fail' gdal.Unlink('/vsimem/file.bin') return 'success'
def vsiaz_fake_write(): if gdaltest.webserver_port == 0: return 'skip' gdal.VSICurlClearCache() # Test creation of BlockBob f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' handler = webserver.SequentialHandler() def method(request): h = request.headers if 'Authorization' not in h or \ h['Authorization'] != 'SharedKey myaccount:AigkrY7q66WCrx3JRKBte56k7kxV2cxB/ZyGNubxk5I=' or \ 'Expect' not in h or h['Expect'] != '100-continue' or \ 'Content-Length' not in h or h['Content-Length'] != '40000' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'BlockBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii')) content = request.rfile.read(40000).decode('ascii') if len(content) != 40000: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) with webserver.install_http_handler(handler): ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f) ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f) if ret != 40000: gdaltest.post_reason('fail') print(ret) gdal.VSIFCloseL(f) return 'fail' gdal.VSIFCloseL(f) # Simulate illegal read f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): ret = gdal.VSIFReadL(1, 1, f) if len(ret) != 0: gdaltest.post_reason('fail') print(ret) return 'fail' gdal.VSIFCloseL(f) # Simulate illegal seek f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): ret = gdal.VSIFSeekL(f, 1, 0) if ret == 0: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) # Simulate failure when putting BlockBob f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) if gdal.VSIFSeekL(f, 0, 0) != 0: gdaltest.post_reason('fail') gdal.VSIFCloseL(f) return 'fail' gdal.VSIFWriteL('x' * 35000, 1, 35000, f) if gdal.VSIFTellL(f) != 35000: gdaltest.post_reason('fail') gdal.VSIFCloseL(f) return 'fail' if gdal.VSIFSeekL(f, 35000, 0) != 0: gdaltest.post_reason('fail') gdal.VSIFCloseL(f) return 'fail' if gdal.VSIFSeekL(f, 0, 1) != 0: gdaltest.post_reason('fail') gdal.VSIFCloseL(f) return 'fail' if gdal.VSIFSeekL(f, 0, 2) != 0: gdaltest.post_reason('fail') gdal.VSIFCloseL(f) return 'fail' if gdal.VSIFEofL(f) != 0: gdaltest.post_reason('fail') gdal.VSIFCloseL(f) return 'fail' with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.VSIFCloseL(f) if ret == 0: gdaltest.post_reason('fail') print(ret) gdal.VSIFCloseL(f) return 'fail' # Simulate creation of BlockBob over an existing blob of incompatible type f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' handler = webserver.SequentialHandler() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 409) handler.add('DELETE', '/azure/blob/myaccount/test_copy/file.bin', 202) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201) with webserver.install_http_handler(handler): gdal.VSIFCloseL(f) # Test creation of AppendBlob gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10') f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None) if f is None: gdaltest.post_reason('fail') return 'fail' handler = webserver.SequentialHandler() def method(request): h = request.headers if 'Authorization' not in h or \ h['Authorization'] != 'SharedKey myaccount:KimVui3ptY9D5ftLlsI7CNOgK36CNAEzsXqcuHskdEY=' or \ 'Content-Length' not in h or h['Content-Length'] != '0' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) def method(request): h = request.headers if 'Content-Length' not in h or h['Content-Length'] != '10' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' content = request.rfile.read(10).decode('ascii') if content != '0123456789': sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', custom_method=method) def method(request): h = request.headers if 'Content-Length' not in h or h['Content-Length'] != '6' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' content = request.rfile.read(6).decode('ascii') if content != 'abcdef': sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', custom_method=method) with webserver.install_http_handler(handler): ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) if ret != 16: gdaltest.post_reason('fail') print(ret) gdal.VSIFCloseL(f) return 'fail' gdal.VSIFCloseL(f) # Test failed creation of AppendBlob gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10') f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None) if f is None: gdaltest.post_reason('fail') return 'fail' handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) if ret != 0: gdaltest.post_reason('fail') print(ret) gdal.VSIFCloseL(f) return 'fail' gdal.VSIFCloseL(f) # Test failed writing of a block of an AppendBlob gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10') f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None) if f is None: gdaltest.post_reason('fail') return 'fail' handler = webserver.SequentialHandler() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 403) with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) if ret != 0: gdaltest.post_reason('fail') print(ret) gdal.VSIFCloseL(f) return 'fail' gdal.VSIFCloseL(f) return 'success'
def vsicrypt_2(): if not gdaltest.has_vsicrypt: return 'skip' # Missing key with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Invalid file with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'wb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Invalid file with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Invalid file with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'ab') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Invalid access with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'foo') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Key to short with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=a,file=/vsimem/file.bin', 'wb+') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Invalid signature gdal.FileFromMemBuffer('/vsimem/file.bin', 'foo') with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Generate empty file fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'wb') gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') if fp is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsimem/file.bin', 'rb') header = gdal.VSIFReadL(1, 1000, fp) gdal.VSIFCloseL(fp) if len(header) != 46: gdaltest.post_reason('fail') print(len(header)) return 'fail' # Test shortening header for i in range(46): fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb') gdal.VSIFWriteL(header, 1, 46 - 1 - i, fp) gdal.VSIFCloseL(fp) with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Test corrupting all bytes of header for i in range(46): for val in (0, 127, 255): fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb') try: new_byte = chr(val).encode('latin1') except (UnicodeDecodeError, UnicodeEncodeError): new_byte = chr(val) header_new = header[0:i] + new_byte + header[i + 1:] gdal.VSIFWriteL(header_new, 1, 46, fp) gdal.VSIFCloseL(fp) with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=' '/vsimem/file.bin', 'rb') if fp is not None: gdal.VSIFCloseL(fp) gdal.SetConfigOption('VSICRYPT_IV', 'TOO_SHORT') with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=' '/vsimem/file.bin', 'wb') gdal.SetConfigOption('VSICRYPT_IV', None) if fp is not None: gdal.VSIFCloseL(fp) # Inconsistent initial vector. header = struct.pack('B' * 38, 86, 83, 73, 67, 82, 89, 80, 84, # signature 38, 0, # header size 1, # major 0, # minor 0, 2, # sector size 0, # alg 0, # mode 8, # size of IV (should be 16) 32, 13, 169, 71, 154, 208, 22, 32, # IV 0, 0, # size of free text 0, # size of key check 0, 0, 0, 0, 0, 0, 0, 0, # size of unencrypted file 0, 0 # size of extra content ) fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb') gdal.VSIFWriteL(header, 1, len(header), fp) gdal.VSIFCloseL(fp) with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Inconsistent initial vector with key check. header = struct.pack('B' * 39, 86, 83, 73, 67, 82, 89, 80, 84, # signature 39, 0, # header size 1, # major 0, # minor 0, 2, # sector size 0, # alg 0, # mode 8, # size of IV (should be 16) 32, 13, 169, 71, 154, 208, 22, 32, # IV 0, 0, # size of free text 1, # size of key check 0, # key check 0, 0, 0, 0, 0, 0, 0, 0, # size of unencrypted file 0, 0 # size of extra content ) fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb') gdal.VSIFWriteL(header, 1, len(header), fp) gdal.VSIFCloseL(fp) with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Test reading with wrong key fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'wb') gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin', 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content == 'hello': gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=short_key,file=/vsimem/file.bin', 'ab') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Test reading with wrong key with add_key_check fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,add_key_check=yes,file=/vsimem/file.bin', 'wb') gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=short_key,file=/vsimem/file.bin', 'ab') if fp is not None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin', 'ab') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Test creating with potentially not build-in alg: with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/alg=blowfish,key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'wb') if fp is not None: gdal.VSIFCloseL(fp) # Invalid sector_size with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,sector_size=1,file=/vsimem/file.bin', 'wb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Sector size (16) should be at least twice larger than the block size (16) in CBC_CTS with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,sector_size=16,mode=CBC_CTS,file=/vsimem/file.bin', 'wb') if fp is not None: gdaltest.post_reason('fail') return 'fail' gdal.Unlink('/vsimem/file.bin') return 'success'
def vsifile_5(): fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'wb') ref_data = ''.join(['%08X' % i for i in range(5 * 32768)]) gdal.VSIFWriteL(ref_data, 1, len(ref_data), fp) gdal.VSIFCloseL(fp) gdal.SetConfigOption('VSI_CACHE', 'YES') for i in range(3): if i == 0: gdal.SetConfigOption('VSI_CACHE_SIZE', '0') elif i == 1: gdal.SetConfigOption('VSI_CACHE_SIZE', '65536') else: gdal.SetConfigOption('VSI_CACHE_SIZE', None) fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'rb') gdal.VSIFSeekL(fp, 50000, 0) if gdal.VSIFTellL(fp) != 50000: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFSeekL(fp, 50000, 1) if gdal.VSIFTellL(fp) != 100000: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFSeekL(fp, 0, 2) if gdal.VSIFTellL(fp) != 5 * 32768 * 8: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFReadL(1, 1, fp) gdal.VSIFSeekL(fp, 0, 0) data = gdal.VSIFReadL(1, 3 * 32768, fp) if data.decode('ascii') != ref_data[0:3 * 32768]: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFSeekL(fp, 16384, 0) data = gdal.VSIFReadL(1, 5 * 32768, fp) if data.decode('ascii') != ref_data[16384:16384 + 5 * 32768]: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' data = gdal.VSIFReadL(1, 50 * 32768, fp) if data[0:1130496].decode('ascii') != ref_data[16384 + 5 * 32768:]: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFCloseL(fp) gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) gdal.Unlink('tmp/vsifile_5.bin') return 'success'
def test_vrt_read_10(): src_ds = gdal.Open('data/byte.tif') mem_ds = gdal.GetDriverByName('GTiff').CreateCopy( '/vsimem/vrt_read_10.tif', src_ds) vrt_ds = gdal.GetDriverByName('VRT').CreateCopy('/vsimem/vrt_read_10.vrt', mem_ds) vrt_hist = vrt_ds.GetRasterBand(1).GetHistogram() mem_hist = mem_ds.GetRasterBand(1).GetHistogram() mem_ds = None vrt_ds = None f = gdal.VSIFOpenL('/vsimem/vrt_read_10.vrt', 'rb') content = gdal.VSIFReadL(1, 10000, f).decode('ascii') gdal.VSIFCloseL(f) assert vrt_hist == mem_hist assert '<Histograms>' in content # Single source optimization for i in range(2): gdal.FileFromMemBuffer( '/vsimem/vrt_read_10.vrt', """<VRTDataset rasterXSize="20" rasterYSize="20"> <VRTRasterBand dataType="Byte" band="1"> <SimpleSource> <SourceFilename relativeToVRT="1">vrt_read_10.tif</SourceFilename> </SimpleSource> </VRTRasterBand> </VRTDataset>""") ds = gdal.Open('/vsimem/vrt_read_10.vrt') if i == 0: ds.GetRasterBand(1).GetDefaultHistogram() else: ds.GetRasterBand(1).GetHistogram() ds = None f = gdal.VSIFOpenL('/vsimem/vrt_read_10.vrt', 'rb') content = gdal.VSIFReadL(1, 10000, f).decode('ascii') gdal.VSIFCloseL(f) assert '<Histograms>' in content # Two sources general case for i in range(2): gdal.FileFromMemBuffer( '/vsimem/vrt_read_10.vrt', """<VRTDataset rasterXSize="20" rasterYSize="20"> <VRTRasterBand dataType="Byte" band="1"> <SimpleSource> <SourceFilename relativeToVRT="1">vrt_read_10.tif</SourceFilename> </SimpleSource> <SimpleSource> <SourceFilename relativeToVRT="1">vrt_read_10.tif</SourceFilename> </SimpleSource> </VRTRasterBand> </VRTDataset>""") ds = gdal.Open('/vsimem/vrt_read_10.vrt') if i == 0: ds.GetRasterBand(1).GetDefaultHistogram() else: ds.GetRasterBand(1).GetHistogram() ds = None f = gdal.VSIFOpenL('/vsimem/vrt_read_10.vrt', 'rb') content = gdal.VSIFReadL(1, 10000, f).decode('ascii') gdal.VSIFCloseL(f) assert '<Histograms>' in content gdal.GetDriverByName('GTiff').Delete('/vsimem/vrt_read_10.tif') gdal.GetDriverByName('VRT').Delete('/vsimem/vrt_read_10.vrt')
def vsicrypt_3(): if not gdaltest.has_vsicrypt: return 'skip' for options in [ 'sector_size=16', 'alg=AES', 'alg=DES_EDE2', 'alg=DES_EDE3', 'alg=SKIPJACK', 'alg=invalid', 'mode=CBC', 'mode=CFB', 'mode=OFB', 'mode=CTR', 'mode=CBC_CTS', 'mode=invalid', 'freetext=my_free_text', 'add_key_check=yes' ]: gdal.Unlink('/vsimem/file.bin') if options == 'alg=invalid' or options == 'mode=invalid': with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,%s,file=/vsimem/file.bin' % options, 'wb') else: fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,%s,file=/vsimem/file.bin' % options, 'wb') if fp is None: gdaltest.post_reason('fail') print(options) return 'fail' gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,file=/vsimem/file.bin', 'r') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') print(options) return 'fail' # Some of those algs might be missing for options in [ 'alg=Blowfish', 'alg=Camellia', 'alg=CAST256', 'alg=MARS', 'alg=IDEA', 'alg=RC5', 'alg=RC6', 'alg=Serpent', 'alg=SHACAL2', 'alg=Twofish', 'alg=XTEA' ]: gdal.Unlink('/vsimem/file.bin') with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,%s,file=/vsimem/file.bin' % options, 'wb') if fp is not None: gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') print(options) return 'fail' # Test key generation # Do NOT set VSICRYPT_CRYPTO_RANDOM=NO in production. This is just to speed up tests ! gdal.SetConfigOption("VSICRYPT_CRYPTO_RANDOM", "NO") fp = gdal.VSIFOpenL( '/vsicrypt/key=GENERATE_IT,add_key_check=yes,file=/vsimem/file.bin', 'wb') gdal.SetConfigOption("VSICRYPT_CRYPTO_RANDOM", None) # Get the generated random key key_b64 = gdal.GetConfigOption('VSICRYPT_KEY_B64') if key_b64 is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') print(options) return 'fail' gdal.SetConfigOption('VSICRYPT_KEY_B64', None) fp = gdal.VSIFOpenL('/vsicrypt/key_b64=%s,file=/vsimem/file.bin' % key_b64, 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') print(options) return 'fail' with gdaltest.error_handler(): statRes = gdal.VSIStatL('/vsicrypt//vsimem/file.bin') if statRes is not None: gdaltest.post_reason('fail') return 'fail' ret = gdal.Rename('/vsicrypt//vsimem/file.bin', '/vsicrypt//vsimem/subdir_crypt/file.bin') if ret != 0: gdaltest.post_reason('fail') print(ret) return 'fail' ret = gdal.Rename('/vsicrypt//vsimem/subdir_crypt/file.bin', '/vsimem/subdir_crypt/file2.bin') if ret != 0: gdaltest.post_reason('fail') print(ret) return 'fail' dir_content = gdal.ReadDir('/vsicrypt//vsimem/subdir_crypt') if dir_content != ['file2.bin']: gdaltest.post_reason('fail') print(dir_content) return 'fail' gdal.Unlink('/vsimem/subdir_crypt/file2.bin') return 'success'
def test_vsigs_read_credentials_refresh_token_default_gdal_app(): if gdaltest.webserver_port == 0: pytest.skip() gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '') gdal.SetConfigOption('GS_ACCESS_KEY_ID', '') gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN', 'http://localhost:%d/accounts.google.com/o/oauth2/token' % gdaltest.webserver_port) gdal.SetConfigOption('GS_OAUTH2_REFRESH_TOKEN', 'REFRESH_TOKEN') with gdaltest.error_handler(): assert gdal.GetSignedURL('/vsigs/foo/bar') is None gdal.VSICurlClearCache() handler = webserver.SequentialHandler() def method(request): content = request.rfile.read(int(request.headers['Content-Length'])).decode('ascii') if content != 'refresh_token=REFRESH_TOKEN&client_id=265656308688.apps.googleusercontent.com&client_secret=0IbTUDOYzaL6vnIdWTuQnvLz&grant_type=refresh_token': sys.stderr.write('Bad POST content: %s\n' % content) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') content = """{ "access_token" : "ACCESS_TOKEN", "token_type" : "Bearer", "expires_in" : 3600, }""" request.send_header('Content-Length', len(content)) request.end_headers() request.wfile.write(content.encode('ascii')) handler.add('POST', '/accounts.google.com/o/oauth2/token', custom_method=method) def method(request): if 'Authorization' not in request.headers: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) return expected_authorization = 'Bearer ACCESS_TOKEN' if request.headers['Authorization'] != expected_authorization: sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization'])) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.end_headers() request.wfile.write("""foo""".encode('ascii')) handler.add('GET', '/gs_fake_bucket/resource', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsigs/gs_fake_bucket/resource') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo' gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN', None) gdal.SetConfigOption('GS_OAUTH2_REFRESH_TOKEN', '')