def test_vsigs_read_credentials_oauth2_service_account():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
# Generated with 'openssl genrsa -out rsa-openssl.pem 1024' and
# 'openssl pkcs8 -nocrypt -in rsa-openssl.pem -inform PEM -topk8 -outform PEM -out rsa-openssl.pkcs8.pem'
# DO NOT USE in production !!!!
key = """-----BEGIN PRIVATE KEY-----
MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk
VNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm
i2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ
DVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy
TabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I
2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd
DrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+
bpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg
Wz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3
4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5
Kvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj
j+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI
yzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq
gwE6fxOLyJDxuWRf
-----END PRIVATE KEY-----
"""
for i in range(2):
gdal.SetConfigOption('GO2A_AUD',
'http://localhost:%d/oauth2/v4/token' % gdaltest.webserver_port)
gdal.SetConfigOption('GOA2_NOW', '123456')
if i == 0:
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY', key)
else:
gdal.FileFromMemBuffer('/vsimem/pkey', key)
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY_FILE', '/vsimem/pkey')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_EMAIL', 'CLIENT_EMAIL')
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(request.headers['Content-Length'])).decode('ascii')
content_8080 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODAvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.DAhqWtBgKpObxZ%2BGiXqwF%2Fa4SS%2FNWQRhLCI7DYZCuOTuf2w7dL8j4CdpiwwzQg1diIus7dyViRfzpsFmuZKAXwL%2B84iBoVVqnJJZ4TgwH49NdfMAnc4Rgm%2Bo2a2nEcMjX%2FbQ3jRY%2B9WNVl96hzULGvLrVeyego2f06wivqmvxHA%3D'
content_8081 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODEvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.0abOEg4%2FRApWTSeAs6YTHaNzdwOgZLm8DTMO2MKlOA%2Fiagyb4cBJxDpkD5gECPvi7qhkg7LsyFuj0a%2BK48Bsuj%2FgLHOU4MpB0dHwYnDO2UXzH%2FUPdgFCVak1P1V%2ByiDA%2B%2Ft4aDI5fD9qefKQiu3wsMDHzP71MNLzayrjqaqKKS4%3D'
if content not in [content_8080, content_8081]:
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST', '/oauth2/v4/token', custom_method=method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
try:
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
except:
if gdal.GetLastErrorMsg().find('CPLRSASHA256Sign() not implemented') >= 0:
pytest.skip()
finally:
gdal.SetConfigOption('GO2A_AUD', None)
gdal.SetConfigOption('GO2A_NOW', None)
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY', '')
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY_FILE', '')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_EMAIL', '')
assert data == 'foo'
gdal.Unlink('/vsimem/pkey')
def vsiaz_extra_1():
if not gdaltest.built_against_curl():
return 'skip'
az_resource = gdal.GetConfigOption('AZ_RESOURCE')
if az_resource is None:
print('Missing AZ_RESOURCE for running gdaltest_list_extra')
return 'skip'
if az_resource.find('/') < 0:
path = '/vsiaz/' + az_resource
statres = gdal.VSIStatL(path)
if statres is None or not stat.S_ISDIR(statres.mode):
gdaltest.post_reason('fail')
print('%s is not a valid bucket' % path)
return 'fail'
readdir = gdal.ReadDir(path)
if readdir is None:
gdaltest.post_reason('fail')
print('ReadDir() should not return empty list')
return 'fail'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
if gdal.VSIStatL(subpath) is None:
gdaltest.post_reason('fail')
print('Stat(%s) should not return an error' % subpath)
return 'fail'
unique_id = 'vsiaz_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
if ret < 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) should not return an error' % subpath)
return 'fail'
readdir = gdal.ReadDir(path)
if unique_id not in readdir:
gdaltest.post_reason('fail')
print('ReadDir(%s) should contain %s' % (path, unique_id))
print(readdir)
return 'fail'
ret = gdal.Mkdir(subpath, 0)
if ret == 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) repeated should return an error' % subpath)
return 'fail'
ret = gdal.Rmdir(subpath)
if ret < 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) should not return an error' % subpath)
return 'fail'
readdir = gdal.ReadDir(path)
if unique_id in readdir:
gdaltest.post_reason('fail')
print('ReadDir(%s) should not contain %s' % (path, unique_id))
print(readdir)
return 'fail'
ret = gdal.Rmdir(subpath)
if ret == 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) repeated should return an error' % subpath)
return 'fail'
ret = gdal.Mkdir(subpath, 0)
if ret < 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) should not return an error' % subpath)
return 'fail'
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
if ret == 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) on non empty directory should return an error' %
subpath)
return 'fail'
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
if data != 'hello':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test.txt')
if ret < 0:
gdaltest.post_reason('fail')
print('Unlink(%s) should not return an error' %
(subpath + '/test.txt'))
return 'fail'
ret = gdal.Rmdir(subpath)
if ret < 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) should not return an error' % subpath)
return 'fail'
return 'success'
f = open_for_read('/vsiaz/' + az_resource)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Same with /vsiaz_streaming/
f = open_for_read('/vsiaz_streaming/' + az_resource)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
if False: # we actually try to read at read() time and bSetError = false
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsiaz/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if gdal.VSIGetLastErrorMsg() == '':
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsiaz_streaming/' + az_resource +
'/invalid_resource.baz')
if f is not None:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Test GetSignedURL()
signed_url = gdal.GetSignedURL('/vsiaz/' + az_resource)
f = open_for_read('/vsicurl_streaming/' + signed_url)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
return 'success'
def vsifile_generic(filename):
start_time = time.time()
fp = gdal.VSIFOpenL(filename, 'wb+')
if fp is None:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFWriteL('0123456789', 1, 10, fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFFlushL(fp) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTruncateL(fp, 20) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTruncateL(fp, 5) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFSeekL(fp, 0, 2) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 5:
gdaltest.post_reason('failure')
return 'fail'
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 7:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if abs(start_time - statBuf.mtime) > 2:
gdaltest.post_reason('failure')
print(statBuf.mtime)
return 'fail'
fp = gdal.VSIFOpenL(filename, 'rb')
buf = gdal.VSIFReadL(1, 7, fp)
if gdal.VSIFWriteL('a', 1, 1, fp) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFTruncateL(fp, 0) == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(fp)
if buf.decode('ascii') != '01234XX':
gdaltest.post_reason('failure')
print(buf.decode('ascii'))
return 'fail'
# Test append mode on existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 9:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if gdal.Unlink(filename) != 0:
gdaltest.post_reason('failure')
return 'fail'
statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG)
if statBuf is not None:
gdaltest.post_reason('failure')
return 'fail'
# Test append mode on non existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 2:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if gdal.Unlink(filename) != 0:
gdaltest.post_reason('failure')
return 'fail'
return 'success'
def vsigs_2():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('CPL_GS_ENDPOINT', 'http://127.0.0.1:%d/' % gdaltest.webserver_port)
# header file
gdal.FileFromMemBuffer('/vsimem/my_headers.txt', 'foo: bar')
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', '/vsimem/my_headers.txt')
f = open_for_read('/vsigs/gs_fake_bucket_http_header_file/resource')
if f is None:
gdal.Unlink('/vsimem/my_headers.txt')
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None)
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 1, f)
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None)
gdal.Unlink('/vsimem/my_headers.txt')
gdal.VSIFCloseL(f)
if len(data) != 1:
gdaltest.post_reason('fail')
return 'fail'
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID')
gdal.SetConfigOption('CPL_GS_TIMESTAMP', 'my_timestamp')
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
f = open_for_read('/vsigs_streaming/gs_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
#old_val = gdal.GetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN')
#gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR')
stat_res = gdal.VSIStatL('/vsigs/gs_fake_bucket/resource2.bin')
#gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', old_val)
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
stat_res = gdal.VSIStatL('/vsigs_streaming/gs_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
return 'success'
def vsiaz_fake_basic():
if gdaltest.webserver_port == 0:
return 'skip'
signed_url = gdal.GetSignedURL('/vsiaz/az_fake_bucket/resource',
['START_DATE=20180213T123456'])
if signed_url not in (
'http://127.0.0.1:8080/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12',
'http://127.0.0.1:8081/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12'
):
gdaltest.post_reason('fail')
print(signed_url)
return 'fail'
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:zKb0EXnM/RinBjcUE9EU+qfRIGaIItoUElSWc+FE24E=' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET',
'/azure/blob/myaccount/az_fake_bucket/resource',
custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz/az_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:8d6IEeOsl7qGpKAxaTTxx2xMNpvqWq8DGlFE67lsmQ4=' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'Accept-Encoding' not in h or h['Accept-Encoding'] != 'gzip':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET',
'/azure/blob/myaccount/az_fake_bucket/resource',
custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz_streaming/az_fake_bucket/resource')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(data)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin',
200, {'Content-Length': '1000000'})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsiaz/az_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin',
200, {'Content-Length': 1000000})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL(
'/vsiaz_streaming/az_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
gdaltest.post_reason('fail')
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
return 'fail'
return 'success'
def gdal_cp_single(srcfile, targetfile, progress):
try:
if os.path.isdir(targetfile):
(head, tail) = os.path.split(srcfile)
targetfile = targetfile + '/' + tail
except:
pass
fin = gdal.VSIFOpenL(srcfile, "rb")
if fin is None:
print('Cannot open %s' % srcfile)
return -1
fout = gdal.VSIFOpenL(targetfile, "wb")
if fout is None:
print('Cannot create %s' % targetfile)
gdal.VSIFCloseL(fin)
return -1
version_num = int(gdal.VersionInfo('VERSION_NUM'))
total_size = 0
if version_num < 1900 or progress is not None:
gdal.VSIFSeekL(fin, 0, 2)
total_size = gdal.VSIFTellL(fin)
gdal.VSIFSeekL(fin, 0, 0)
buffer_max_size = 4096
copied = 0
ret = 0
#print('Copying %s...' % srcfile)
if progress is not None:
if not progress.Progress(0.0, 'Copying %s' % srcfile):
print('Copy stopped by user')
ret = -2
while ret == 0:
if total_size != 0 and copied + buffer_max_size > total_size:
to_read = total_size - copied
else:
to_read = buffer_max_size
buffer = gdal.VSIFReadL(1, to_read, fin)
if buffer is None:
if copied == 0:
print('Cannot read %d bytes in %s' % (to_read, srcfile))
ret = -1
break
buffer_size = len(buffer)
if gdal.VSIFWriteL(buffer, 1, buffer_size, fout) != buffer_size:
print('Error writing %d bytes' % buffer_size)
ret = -1
break
copied += buffer_size
if progress is not None and total_size != 0:
if not progress.Progress(copied * 1.0 / total_size, 'Copying %s' % srcfile):
print('Copy stopped by user')
ret = -2
break
if to_read < buffer_max_size or buffer_size != buffer_max_size:
break
gdal.VSIFCloseL(fin)
gdal.VSIFCloseL(fout)
return ret
def eedai_geotiff():
if gdaltest.eedai_drv is None:
return 'skip'
gdal.FileFromMemBuffer(
'/vsimem/ee/assets/image',
json.dumps({
'type':
'IMAGE',
'bands': [{
"id": "B1",
"dataType": {
"precision": "INTEGER",
"range": {
"max": 65535
}
},
"grid": {
"crsCode": "EPSG:32610",
"affineTransform": {
"translateX": 499980,
"translateY": 4200000,
"scaleX": 60,
"scaleY": -60
},
"dimensions": {
"width": 1830,
"height": 1831
}
}
}]
}))
gdal.SetConfigOption('EEDA_BEARER', 'mybearer')
gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/')
ds = gdal.Open('EEDAI:image')
gdal.SetConfigOption('EEDA_URL', None)
mem_ds = gdal.GetDriverByName('MEM').Create('', 256, 256, 1,
gdal.GDT_UInt16)
mem_ds.GetRasterBand(1).Fill(12345)
gdal.GetDriverByName('GTiff').CreateCopy('/vsimem/out.tif', mem_ds)
f = gdal.VSIFOpenL('/vsimem/out.tif', 'rb')
data = gdal.VSIFReadL(1, 1000000, f)
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/out.tif')
gdal.FileFromMemBuffer(
'/vsimem/ee/assets:getPixels&CUSTOMREQUEST=POST&POSTFIELDS={ "path": "image", "encoding": "GEOTIFF", "bandIds": [ "B1" ], "grid": { "origin": { "x": 499980.0, "y": 4200000.0 }, "pixelSize": { "x": 60.0, "y": -60.0 }, "dimensions": { "width": 256, "height": 256 } } }',
data)
got_data = ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1)
got_data = struct.unpack('H', got_data)[0]
if got_data != 12345:
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
ds = None
gdal.SetConfigOption('EEDA_BEARER', None)
return 'success'
def ogr_pgdump_9(pg_use_copy='YES'):
gdal.SetConfigOption('PG_USE_COPY', pg_use_copy)
ds = ogr.GetDriverByName('PGDump').CreateDataSource(
'/vsimem/ogr_pgdump_9.sql', options=['LINEFORMAT=LF'])
lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone)
fld = ogr.FieldDefn('str', ogr.OFTString)
fld.SetWidth(5)
lyr.CreateField(fld)
fld = ogr.FieldDefn('str2', ogr.OFTString)
lyr.CreateField(fld)
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', '01234')
lyr.CreateFeature(feat)
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', 'ABCDEF')
lyr.CreateFeature(feat)
if sys.version_info >= (3, 0, 0):
val4 = '\u00e9\u00e9\u00e9\u00e9'
val5 = val4 + '\u00e9'
val6 = val5 + '\u00e9'
else:
exec("val4 = u'\\u00e9\\u00e9\\u00e9\\u00e9'")
exec("val5 = val4 + u'\\u00e9'")
exec("val6 = val5 + u'\\u00e9'")
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', val6)
lyr.CreateFeature(feat)
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', 'a' + val5)
lyr.CreateFeature(feat)
gdal.SetConfigOption('PG_USE_COPY', None)
ds = None
f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_9.sql', 'rb')
sql = gdal.VSIFReadL(1, 10000, f).decode('utf8')
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/ogr_pgdump_9.sql')
if pg_use_copy == 'YES':
eofield = '\t'
else:
eofield = "'"
if sql.find("""01234%s""" % eofield) < 0 or \
sql.find("""ABCDE%s""" % eofield) < 0 or \
sql.find("""%s%s""" % (val5, eofield)) < 0 or \
sql.find("""%s%s""" % ('a' + val4, eofield)) < 0:
print(sql)
return 'fail'
return 'success'
def ogr_pgdump_13():
tests = [
[
ogr.wkbUnknown, [], 'POINT ZM (1 2 3 4)',
["'GEOMETRY',2)", "0101000000000000000000F03F0000000000000040"]
],
[
ogr.wkbUnknown, ['GEOM_TYPE=geography'], 'POINT ZM (1 2 3 4)',
[
"geography(GEOMETRY",
"0101000000000000000000F03F0000000000000040"
]
],
[
ogr.wkbUnknown, ['DIM=XYZ'], 'POINT ZM (1 2 3 4)',
[
"'GEOMETRY',3)",
"0101000080000000000000F03F00000000000000400000000000000840"
]
],
[
ogr.wkbUnknown, ['DIM=XYZ', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(GEOMETRYZ,",
"0101000080000000000000F03F00000000000000400000000000000840"
]
],
[
ogr.wkbPoint, ['DIM=XYZ'], 'POINT ZM (1 2 3 4)',
[
"'POINT',3)",
"0101000080000000000000F03F00000000000000400000000000000840"
]
],
[
ogr.wkbPoint25D, [], 'POINT ZM (1 2 3 4)',
[
"'POINT',3)",
"0101000080000000000000F03F00000000000000400000000000000840"
]
],
[
ogr.wkbPoint, ['DIM=XYZ', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(POINTZ,",
"0101000080000000000000F03F00000000000000400000000000000840"
]
],
[
ogr.wkbUnknown, ['DIM=XYM'], 'POINT ZM (1 2 3 4)',
[
"'GEOMETRY',3)",
"0101000040000000000000F03F00000000000000400000000000001040"
]
],
[
ogr.wkbUnknown, ['DIM=XYM', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(GEOMETRYM,",
"0101000040000000000000F03F00000000000000400000000000001040"
]
],
[
ogr.wkbPoint, ['DIM=XYM'], 'POINT ZM (1 2 3 4)',
[
"'POINTM',3)",
"0101000040000000000000F03F00000000000000400000000000001040"
]
],
[
ogr.wkbPointM, [], 'POINT ZM (1 2 3 4)',
[
"'POINTM',3)",
"0101000040000000000000F03F00000000000000400000000000001040"
]
],
[
ogr.wkbPoint, ['DIM=XYM', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(POINTM,",
"0101000040000000000000F03F00000000000000400000000000001040"
]
],
[
ogr.wkbUnknown, ['DIM=XYZM'], 'POINT ZM (1 2 3 4)',
[
"'GEOMETRY',4)",
"01010000C0000000000000F03F000000000000004000000000000008400000000000001040"
]
],
[
ogr.wkbUnknown, ['DIM=XYZM', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(GEOMETRYZM,",
"01010000C0000000000000F03F000000000000004000000000000008400000000000001040"
]
],
[
ogr.wkbPoint, ['DIM=XYZM'], 'POINT ZM (1 2 3 4)',
[
"'POINT',4)",
"01010000C0000000000000F03F000000000000004000000000000008400000000000001040"
]
],
[
ogr.wkbPointZM, [], 'POINT ZM (1 2 3 4)',
[
"'POINT',4)",
"01010000C0000000000000F03F000000000000004000000000000008400000000000001040"
]
],
[
ogr.wkbPoint, ['DIM=XYZM', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(POINTZM,",
"01010000C0000000000000F03F000000000000004000000000000008400000000000001040"
]
],
]
for (geom_type, options, wkt, expected_strings) in tests:
ds = ogr.GetDriverByName('PGDump').CreateDataSource(
'/vsimem/ogr_pgdump_13.sql', options=['LINEFORMAT=LF'])
lyr = ds.CreateLayer('test', geom_type=geom_type, options=options)
f = ogr.Feature(lyr.GetLayerDefn())
f.SetGeometryDirectly(ogr.CreateGeometryFromWkt(wkt))
lyr.CreateFeature(f)
f = None
ds = None
f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_13.sql', 'rb')
sql = gdal.VSIFReadL(1, 10000, f).decode('utf8')
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/ogr_pgdump_13.sql')
for expected_string in expected_strings:
if sql.find(expected_string) < 0:
gdaltest.post_reason('fail')
print(geom_type, options, wkt, expected_string)
print(sql)
return 'fail'
if 'GEOM_TYPE=geography' in options:
continue
ds = ogr.GetDriverByName('PGDump').CreateDataSource(
'/vsimem/ogr_pgdump_13.sql', options=['LINEFORMAT=LF'])
lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone, options=options)
lyr.CreateGeomField(ogr.GeomFieldDefn("my_geom", geom_type))
f = ogr.Feature(lyr.GetLayerDefn())
f.SetGeometryDirectly(ogr.CreateGeometryFromWkt(wkt))
lyr.CreateFeature(f)
f = None
ds = None
f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_13.sql', 'rb')
sql = gdal.VSIFReadL(1, 10000, f).decode('utf8')
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/ogr_pgdump_13.sql')
for expected_string in expected_strings:
if sql.find(expected_string) < 0:
gdaltest.post_reason('fail')
print(geom_type, options, wkt, expected_string)
print(sql)
return 'fail'
return 'success'
def ogr_pgdump_6():
ds = ogr.GetDriverByName('PGDump').CreateDataSource(
'/vsimem/ogr_pgdump_6.sql', options=['LINEFORMAT=LF'])
lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone)
field_defn = ogr.FieldDefn('field_string', ogr.OFTString)
field_defn.SetDefault("'a''b'")
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_int', ogr.OFTInteger)
field_defn.SetDefault('123')
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_real', ogr.OFTReal)
field_defn.SetDefault('1.23')
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_nodefault', ogr.OFTInteger)
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_datetime', ogr.OFTDateTime)
field_defn.SetDefault("CURRENT_TIMESTAMP")
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_datetime2', ogr.OFTDateTime)
field_defn.SetDefault("'2015/06/30 12:34:56'")
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_date', ogr.OFTDate)
field_defn.SetDefault("CURRENT_DATE")
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_time', ogr.OFTTime)
field_defn.SetDefault("CURRENT_TIME")
lyr.CreateField(field_defn)
gdal.SetConfigOption('PG_USE_COPY', 'YES')
f = ogr.Feature(lyr.GetLayerDefn())
f.SetField('field_string', 'a')
f.SetField('field_int', 456)
f.SetField('field_real', 4.56)
f.SetField('field_datetime', '2015/06/30 12:34:56')
f.SetField('field_datetime2', '2015/06/30 12:34:56')
f.SetField('field_date', '2015/06/30')
f.SetField('field_time', '12:34:56')
lyr.CreateFeature(f)
f = None
# Transition from COPY to INSERT
f = ogr.Feature(lyr.GetLayerDefn())
lyr.CreateFeature(f)
f = None
# Transition from INSERT to COPY
f = ogr.Feature(lyr.GetLayerDefn())
f.SetField('field_string', 'b')
f.SetField('field_int', 456)
f.SetField('field_real', 4.56)
f.SetField('field_datetime', '2015/06/30 12:34:56')
f.SetField('field_datetime2', '2015/06/30 12:34:56')
f.SetField('field_date', '2015/06/30')
f.SetField('field_time', '12:34:56')
lyr.CreateFeature(f)
f = None
gdal.SetConfigOption('PG_USE_COPY', None)
ds = None
f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_6.sql', 'rb')
sql = gdal.VSIFReadL(1, 10000, f).decode('ascii')
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/ogr_pgdump_6.sql')
if sql.find("""a\t456\t4.56\t\\N\t2015/06/30 12:34:56\t2015/06/30 12:34:56\t2015/06/30\t12:34:56""") < 0 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_string" VARCHAR DEFAULT 'a''b';""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_int" INTEGER DEFAULT 123;""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_real" FLOAT8 DEFAULT 1.23;""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_datetime" timestamp with time zone DEFAULT CURRENT_TIMESTAMP;""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_datetime2" timestamp with time zone DEFAULT '2015/06/30 12:34:56+00'::timestamp with time zone;""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_date" date DEFAULT CURRENT_DATE;""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_time" time DEFAULT CURRENT_TIME;""") == -1 or \
sql.find("""b\t456\t4.56\t\\N\t2015/06/30 12:34:56\t2015/06/30 12:34:56\t2015/06/30\t12:34:56""") < 0:
print(sql)
return 'fail'
return 'success'
def ogr_pgdump_8():
ds = ogr.GetDriverByName('PGDump').CreateDataSource(
'/vsimem/ogr_pgdump_8.sql', options=['LINEFORMAT=LF'])
lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone, options=['FID=myfid'])
lyr.CreateField(ogr.FieldDefn('str', ogr.OFTString))
gdal.PushErrorHandler()
ret = lyr.CreateField(ogr.FieldDefn('myfid', ogr.OFTString))
gdal.PopErrorHandler()
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
ret = lyr.CreateField(ogr.FieldDefn('myfid', ogr.OFTInteger))
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
lyr.CreateField(ogr.FieldDefn('str2', ogr.OFTString))
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', 'first string')
feat.SetField('myfid', 10)
feat.SetField('str2', 'second string')
gdal.SetConfigOption('PG_USE_COPY', 'YES')
ret = lyr.CreateFeature(feat)
gdal.SetConfigOption('PG_USE_COPY', None)
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
if feat.GetFID() != 10:
gdaltest.post_reason('fail')
return 'fail'
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str2', 'second string')
gdal.SetConfigOption('PG_USE_COPY', 'YES')
ret = lyr.CreateFeature(feat)
gdal.SetConfigOption('PG_USE_COPY', None)
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
if feat.GetFID() < 0:
gdaltest.post_reason('fail')
feat.DumpReadable()
return 'fail'
if feat.GetField('myfid') != feat.GetFID():
gdaltest.post_reason('fail')
feat.DumpReadable()
return 'fail'
# feat.SetField('str', 'foo')
# ret = lyr.SetFeature(feat)
# if ret != 0:
# gdaltest.post_reason('fail')
# return 'fail'
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetFID(1)
feat.SetField('myfid', 10)
gdal.PushErrorHandler()
gdal.SetConfigOption('PG_USE_COPY', 'YES')
ret = lyr.CreateFeature(feat)
gdal.SetConfigOption('PG_USE_COPY', None)
gdal.PopErrorHandler()
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
# gdal.PushErrorHandler()
# ret = lyr.SetFeature(feat)
# gdal.PopErrorHandler()
# if ret == 0:
# gdaltest.post_reason('fail')
# return 'fail'
# feat.UnsetField('myfid')
# gdal.PushErrorHandler()
# ret = lyr.SetFeature(feat)
# gdal.PopErrorHandler()
# if ret == 0:
# gdaltest.post_reason('fail')
# return 'fail'
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', 'first string')
feat.SetField('myfid', 12)
feat.SetField('str2', 'second string')
gdal.SetConfigOption('PG_USE_COPY', 'YES')
ret = lyr.CreateFeature(feat)
gdal.SetConfigOption('PG_USE_COPY', None)
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
if feat.GetFID() != 12:
gdaltest.post_reason('fail')
return 'fail'
ds = None
f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_8.sql', 'rb')
sql = gdal.VSIFReadL(1, 10000, f).decode('ascii')
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/ogr_pgdump_8.sql')
if sql.find("""CREATE TABLE "public"."test" ( "myfid" SERIAL, CONSTRAINT "test_pk" PRIMARY KEY ("myfid") )""") < 0 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "myfid" """) >= 0 or \
sql.find("""10\tfirst string\tsecond string""") == -1 or \
sql.find("""INSERT INTO "public"."test" ("str2") VALUES ('second string');""") == -1 or \
sql.find("""12\tfirst string\tsecond string""") == -1:
print(sql)
return 'fail'
return 'success'
def test_vsigs_read_credentials_gce_expiration():
if gdaltest.webserver_port == 0:
pytest.skip()
if sys.platform not in ('linux', 'linux2', 'win32'):
pytest.skip()
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '')
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL',
'http://localhost:%d/computeMetadata/v1/instance/service-accounts/default/token' % gdaltest.webserver_port)
# Disable hypervisor related check to test if we are really on EC2
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', 'NO')
gdal.VSICurlClearCache()
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
# First time is used when trying to establish if GCE authentication is available
handler.add('GET', '/computeMetadata/v1/instance/service-accounts/default/token', 200, {},
"""{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 0,
}""")
# Second time is needed because f the access to th file
handler.add('GET', '/computeMetadata/v1/instance/service-accounts/default/token', 200, {},
"""{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 0,
}""")
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL', '')
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', None)
def test_vsigs_read_credentials_file_refresh_token():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '/vsimem/.boto')
gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN',
'http://localhost:%d/accounts.google.com/o/oauth2/token' % gdaltest.webserver_port)
gdal.VSICurlClearCache()
gdal.FileFromMemBuffer('/vsimem/.boto', """
[Credentials]
gs_oauth2_refresh_token = REFRESH_TOKEN
[OAuth2]
client_id = CLIENT_ID
client_secret = CLIENT_SECRET
""")
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(request.headers['Content-Length'])).decode('ascii')
if content != 'refresh_token=REFRESH_TOKEN&client_id=CLIENT_ID&client_secret=CLIENT_SECRET&grant_type=refresh_token':
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST', '/accounts.google.com/o/oauth2/token', custom_method=method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '')
gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN', None)
gdal.Unlink('/vsimem/.boto')
def test_vsigs_read_credentials_oauth2_service_account_json_file():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.FileFromMemBuffer('/vsimem/service_account.json', """{
"private_key": "-----BEGIN PRIVATE KEY-----\nMIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk\nVNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm\ni2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ\nDVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy\nTabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I\n2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd\nDrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+\nbpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg\nWz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3\n4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5\nKvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj\nj+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI\nyzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq\ngwE6fxOLyJDxuWRf\n-----END PRIVATE KEY-----\n",
"client_email": "CLIENT_EMAIL"
}""")
gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', '/vsimem/service_account.json')
gdal.SetConfigOption('GO2A_AUD',
'http://localhost:%d/oauth2/v4/token' % gdaltest.webserver_port)
gdal.SetConfigOption('GOA2_NOW', '123456')
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(request.headers['Content-Length'])).decode('ascii')
content_8080 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODAvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.DAhqWtBgKpObxZ%2BGiXqwF%2Fa4SS%2FNWQRhLCI7DYZCuOTuf2w7dL8j4CdpiwwzQg1diIus7dyViRfzpsFmuZKAXwL%2B84iBoVVqnJJZ4TgwH49NdfMAnc4Rgm%2Bo2a2nEcMjX%2FbQ3jRY%2B9WNVl96hzULGvLrVeyego2f06wivqmvxHA%3D'
content_8081 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODEvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.0abOEg4%2FRApWTSeAs6YTHaNzdwOgZLm8DTMO2MKlOA%2Fiagyb4cBJxDpkD5gECPvi7qhkg7LsyFuj0a%2BK48Bsuj%2FgLHOU4MpB0dHwYnDO2UXzH%2FUPdgFCVak1P1V%2ByiDA%2B%2Ft4aDI5fD9qefKQiu3wsMDHzP71MNLzayrjqaqKKS4%3D'
if content not in [content_8080, content_8081]:
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST', '/oauth2/v4/token', custom_method=method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
try:
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdal.Unlink('/vsimem/service_account.json')
pytest.fail()
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
signed_url = gdal.GetSignedURL('/vsigs/gs_fake_bucket/resource',
['START_DATE=20180212T123456Z'])
if signed_url not in ('http://127.0.0.1:8080/gs_fake_bucket/resource?Expires=1518442496&GoogleAccessId=CLIENT_EMAIL&Signature=b19I62KdqV51DpWGxhxGXLGJIA8MHvSJofwOygoeQuIxkM6PmmQFvJYTNWRt9zUVTUoVC0UHVB7ee5Z35NqDC8K4i0quu1hb8Js2B4h0W6OAupvyF3nSQ5D0OJmiSbomGMq0Ehyro5cqJ%2FU%2Fd8oAaKrGKVQScKfXoFrSJBbWkNs%3D',
'http://127.0.0.1:8081/gs_fake_bucket/resource?Expires=1518442496&GoogleAccessId=CLIENT_EMAIL&Signature=b19I62KdqV51DpWGxhxGXLGJIA8MHvSJofwOygoeQuIxkM6PmmQFvJYTNWRt9zUVTUoVC0UHVB7ee5Z35NqDC8K4i0quu1hb8Js2B4h0W6OAupvyF3nSQ5D0OJmiSbomGMq0Ehyro5cqJ%2FU%2Fd8oAaKrGKVQScKfXoFrSJBbWkNs%3D'):
gdal.Unlink('/vsimem/service_account.json')
pytest.fail(signed_url)
except:
if gdal.GetLastErrorMsg().find('CPLRSASHA256Sign() not implemented') >= 0:
pytest.skip()
finally:
gdal.SetConfigOption('GO2A_AUD', None)
gdal.SetConfigOption('GO2A_NOW', None)
gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', '')
gdal.Unlink('/vsimem/service_account.json')
assert data == 'foo'
def validate(ds, check_tiled=True, full_check=False):
"""Check if a file is a (Geo)TIFF with cloud optimized compatible structure.
Args:
ds: GDAL Dataset for the file to inspect.
check_tiled: Set to False to ignore missing tiling.
full_check: Set to TRUe to check tile/strip leader/trailer bytes. Might be slow on remote files
Returns:
A tuple, whose first element is an array of error messages
(empty if there is no error), and the second element, a dictionary
with the structure of the GeoTIFF file.
Raises:
ValidateCloudOptimizedGeoTIFFException: Unable to open the file or the
file is not a Tiff.
"""
if int(gdal.VersionInfo('VERSION_NUM')) < 2020000:
raise ValidateCloudOptimizedGeoTIFFException(
'GDAL 2.2 or above required')
unicode_type = type(''.encode('utf-8').decode('utf-8'))
if isinstance(ds, (str, unicode_type)):
gdal.PushErrorHandler()
ds = gdal.Open(ds)
gdal.PopErrorHandler()
if ds is None:
raise ValidateCloudOptimizedGeoTIFFException(
'Invalid file : %s' % gdal.GetLastErrorMsg())
if ds.GetDriver().ShortName != 'GTiff':
raise ValidateCloudOptimizedGeoTIFFException(
'The file is not a GeoTIFF')
details = {}
errors = []
warnings = []
filename = ds.GetDescription()
main_band = ds.GetRasterBand(1)
ovr_count = main_band.GetOverviewCount()
filelist = ds.GetFileList()
if filelist is not None and filename + '.ovr' in filelist:
errors += [
'Overviews found in external .ovr file. They should be internal'
]
if main_band.XSize > 512 or main_band.YSize > 512:
if check_tiled:
block_size = main_band.GetBlockSize()
if block_size[0] == main_band.XSize and block_size[0] > 1024:
errors += [
'The file is greater than 512xH or Wx512, but is not tiled'
]
if ovr_count == 0:
warnings += [
'The file is greater than 512xH or Wx512, it is recommended '
'to include internal overviews'
]
ifd_offset = int(main_band.GetMetadataItem('IFD_OFFSET', 'TIFF'))
ifd_offsets = [ifd_offset]
block_order_row_major = False
block_leader_size_as_uint4 = False
block_trailer_last_4_bytes_repeated = False
mask_interleaved_with_imagery = False
if ifd_offset not in (8, 16):
# Check if there is GDAL hidden structural metadata
f = gdal.VSIFOpenL(filename, 'rb')
if not f:
raise ValidateCloudOptimizedGeoTIFFException("Cannot open file")
signature = struct.unpack('B' * 4, gdal.VSIFReadL(4, 1, f))
bigtiff = signature in ((0x49, 0x49, 0x2B, 0x00), (0x4D, 0x4D, 0x00,
0x2B))
if bigtiff:
expected_ifd_pos = 16
else:
expected_ifd_pos = 8
gdal.VSIFSeekL(f, expected_ifd_pos, 0)
pattern = "GDAL_STRUCTURAL_METADATA_SIZE=%06d bytes\n" % 0
got = gdal.VSIFReadL(len(pattern), 1, f).decode('LATIN1')
if len(got) == len(pattern) and got.startswith(
'GDAL_STRUCTURAL_METADATA_SIZE='):
size = int(got[len('GDAL_STRUCTURAL_METADATA_SIZE='):][0:6])
extra_md = gdal.VSIFReadL(size, 1, f).decode('LATIN1')
block_order_row_major = 'BLOCK_ORDER=ROW_MAJOR' in extra_md
block_leader_size_as_uint4 = 'BLOCK_LEADER=SIZE_AS_UINT4' in extra_md
block_trailer_last_4_bytes_repeated = 'BLOCK_TRAILER=LAST_4_BYTES_REPEATED' in extra_md
mask_interleaved_with_imagery = 'MASK_INTERLEAVED_WITH_IMAGERY=YES' in extra_md
if 'KNOWN_INCOMPATIBLE_EDITION=YES' in extra_md:
errors += [
"KNOWN_INCOMPATIBLE_EDITION=YES is declared in the file"
]
expected_ifd_pos += len(pattern) + size
expected_ifd_pos += expected_ifd_pos % 2 # IFD offset starts on a 2-byte boundary
gdal.VSIFCloseL(f)
if expected_ifd_pos != ifd_offsets[0]:
errors += [
'The offset of the main IFD should be %d. It is %d instead' %
(expected_ifd_pos, ifd_offsets[0])
]
details['ifd_offsets'] = {}
details['ifd_offsets']['main'] = ifd_offset
for i in range(ovr_count):
# Check that overviews are by descending sizes
ovr_band = ds.GetRasterBand(1).GetOverview(i)
if i == 0:
if (ovr_band.XSize > main_band.XSize
or ovr_band.YSize > main_band.YSize):
errors += [
'First overview has larger dimension than main band'
]
else:
prev_ovr_band = ds.GetRasterBand(1).GetOverview(i - 1)
if (ovr_band.XSize > prev_ovr_band.XSize
or ovr_band.YSize > prev_ovr_band.YSize):
errors += [
'Overview of index %d has larger dimension than '
'overview of index %d' % (i, i - 1)
]
if check_tiled:
block_size = ovr_band.GetBlockSize()
if block_size[0] == ovr_band.XSize and block_size[0] > 1024:
errors += ['Overview of index %d is not tiled' % i]
# Check that the IFD of descending overviews are sorted by increasing
# offsets
ifd_offset = int(ovr_band.GetMetadataItem('IFD_OFFSET', 'TIFF'))
ifd_offsets.append(ifd_offset)
details['ifd_offsets']['overview_%d' % i] = ifd_offset
if ifd_offsets[-1] < ifd_offsets[-2]:
if i == 0:
errors += [
'The offset of the IFD for overview of index %d is %d, '
'whereas it should be greater than the one of the main '
'image, which is at byte %d' %
(i, ifd_offsets[-1], ifd_offsets[-2])
]
else:
errors += [
'The offset of the IFD for overview of index %d is %d, '
'whereas it should be greater than the one of index %d, '
'which is at byte %d' %
(i, ifd_offsets[-1], i - 1, ifd_offsets[-2])
]
# Check that the imagery starts by the smallest overview and ends with
# the main resolution dataset
def get_block_offset(band):
blockxsize, blockysize = band.GetBlockSize()
for y in range(int((band.YSize + blockysize - 1) / blockysize)):
for x in range(int((band.XSize + blockxsize - 1) / blockxsize)):
block_offset = band.GetMetadataItem(
'BLOCK_OFFSET_%d_%d' % (x, y), 'TIFF')
if block_offset:
return int(block_offset)
return 0
block_offset = get_block_offset(main_band)
data_offsets = [block_offset]
details['data_offsets'] = {}
details['data_offsets']['main'] = block_offset
for i in range(ovr_count):
ovr_band = ds.GetRasterBand(1).GetOverview(i)
block_offset = get_block_offset(ovr_band)
data_offsets.append(block_offset)
details['data_offsets']['overview_%d' % i] = block_offset
if data_offsets[-1] != 0 and data_offsets[-1] < ifd_offsets[-1]:
if ovr_count > 0:
errors += [
'The offset of the first block of the smallest overview '
'should be after its IFD'
]
else:
errors += [
'The offset of the first block of the image should '
'be after its IFD'
]
for i in range(len(data_offsets) - 2, 0, -1):
if data_offsets[i] != 0 and data_offsets[i] < data_offsets[i + 1]:
errors += [
'The offset of the first block of overview of index %d should '
'be after the one of the overview of index %d' % (i - 1, i)
]
if len(data_offsets) >= 2 and data_offsets[0] != 0 and data_offsets[
0] < data_offsets[1]:
errors += [
'The offset of the first block of the main resolution image '
'should be after the one of the overview of index %d' %
(ovr_count - 1)
]
if full_check and (block_order_row_major or block_leader_size_as_uint4
or block_trailer_last_4_bytes_repeated
or mask_interleaved_with_imagery):
f = gdal.VSIFOpenL(filename, 'rb')
if not f:
raise ValidateCloudOptimizedGeoTIFFException("Cannot open file")
full_check_band(f, 'Main resolution image', main_band, errors,
block_order_row_major, block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery)
if main_band.GetMaskFlags() == gdal.GMF_PER_DATASET and \
(filename + '.msk') not in ds.GetFileList():
full_check_band(f, 'Mask band of main resolution image',
main_band.GetMaskBand(), errors,
block_order_row_major, block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated, False)
for i in range(ovr_count):
ovr_band = ds.GetRasterBand(1).GetOverview(i)
full_check_band(f, 'Overview %d' % i, ovr_band, errors,
block_order_row_major, block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery)
if ovr_band.GetMaskFlags() == gdal.GMF_PER_DATASET and \
(filename + '.msk') not in ds.GetFileList():
full_check_band(f, 'Mask band of overview %d' % i,
ovr_band.GetMaskBand(), errors,
block_order_row_major,
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated, False)
gdal.VSIFCloseL(f)
return warnings, errors, details
def ogr_vdv_2(src_filename='tmp/test.x10'):
out_filename = '/vsimem/vdv/ogr_vdv_2.x10'
gdal.Unlink(out_filename)
src_ds = ogr.Open(src_filename)
out_ds = ogr.GetDriverByName('VDV').CreateDataSource(out_filename)
layer_names = [
src_ds.GetLayer(idx).GetName() for idx in range(src_ds.GetLayerCount())
]
layer_names.sort()
for layer_name in layer_names:
src_lyr = src_ds.GetLayer(layer_name)
options = [
'HEADER_SRC_DATE=01.01.1970', 'HEADER_SRC_TIME=00.00.00',
'HEADER_foo=bar'
]
dst_lyr = out_ds.CreateLayer(src_lyr.GetName(), options=options)
for field_idx in range(src_lyr.GetLayerDefn().GetFieldCount()):
dst_lyr.CreateField(src_lyr.GetLayerDefn().GetFieldDefn(field_idx))
for src_f in src_lyr:
dst_f = ogr.Feature(dst_lyr.GetLayerDefn())
dst_f.SetFrom(src_f)
dst_lyr.CreateFeature(dst_f)
out_ds = None
expected = """mod; DD.MM.YYYY; HH:MM:SS; free
src; "UNKNOWN"; "01.01.1970"; "00.00.00"
chs; "ISO8859-1"
ver; "1.4"
ifv; "1.4"
dve; "1.4"
fft; ""
foo; "bar"
tbl; another_layer
atr; str_field
frm; char[80]
rec; "0"
rec; "1"
rec; "2"
rec; "3"
rec; "4"
end; 5
tbl; lyr_1
atr; str_field; int_field; int64_field; bool_field; str2_field; int2_field
frm; char[80]; num[10.0]; num[19.0]; boolean; char[2]; num[1.0]
rec; "a""b"; 12; NULL; 1; NULL; NULL
rec; NULL; NULL; NULL; NULL; NULL; NULL
end; 2
tbl; empty
atr;
frm;
end; 0
tbl; empty2
atr;
frm;
end; 0
eof; 4
"""
f = gdal.VSIFOpenL(out_filename, 'rb')
got = gdal.VSIFReadL(1, 10000, f).decode('latin1')
gdal.VSIFCloseL(f)
if got != expected:
gdaltest.post_reason('fail')
print(got)
return 'fail'
gdal.Unlink(out_filename)
return 'success'
def full_check_band(f, band_name, band, errors, block_order_row_major,
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery):
block_size = band.GetBlockSize()
mask_band = None
if mask_interleaved_with_imagery:
mask_band = band.GetMaskBand()
mask_block_size = mask_band.GetBlockSize()
if block_size != mask_block_size:
errors += [
band_name +
': mask block size is different from its imagery band'
]
mask_band = None
yblocks = (band.YSize + block_size[1] - 1) // block_size[1]
xblocks = (band.XSize + block_size[0] - 1) // block_size[0]
last_offset = 0
for y in range(yblocks):
for x in range(xblocks):
offset = band.GetMetadataItem('BLOCK_OFFSET_%d_%d' % (x, y),
'TIFF')
offset = int(offset) if offset is not None else 0
bytecount = band.GetMetadataItem('BLOCK_SIZE_%d_%d' % (x, y),
'TIFF')
bytecount = int(bytecount) if bytecount is not None else 0
if offset > 0:
if block_order_row_major and offset < last_offset:
errors += [
band_name +
': offset of block (%d, %d) is smaller than previous block'
% (x, y)
]
if block_leader_size_as_uint4:
gdal.VSIFSeekL(f, offset - 4, 0)
leader_size = struct.unpack('<I', gdal.VSIFReadL(4, 1,
f))[0]
if leader_size != bytecount:
errors += [
band_name +
': for block (%d, %d), size in leader bytes is %d instead of %d'
% (x, y, leader_size, bytecount)
]
if block_trailer_last_4_bytes_repeated:
if bytecount >= 4:
gdal.VSIFSeekL(f, offset + bytecount - 4, 0)
last_bytes = gdal.VSIFReadL(8, 1, f)
if last_bytes[0:4] != last_bytes[4:8]:
errors += [
band_name +
': for block (%d, %d), trailer bytes are invalid'
% (x, y)
]
if mask_band:
offset_mask = mask_band.GetMetadataItem(
'BLOCK_OFFSET_%d_%d' % (x, y), 'TIFF')
offset_mask = int(
offset_mask) if offset_mask is not None else 0
if offset > 0 and offset_mask > 0:
#bytecount_mask = int(mask_band.GetMetadataItem('BLOCK_SIZE_%d_%d' % (x,y), 'TIFF'))
expected_offset_mask = offset + bytecount + \
(4 if block_leader_size_as_uint4 else 0) + \
(4 if block_trailer_last_4_bytes_repeated else 0)
if offset_mask != expected_offset_mask:
errors += [
'Mask of ' + band_name +
': for block (%d, %d), offset is %d, whereas %d was expected'
% (x, y, offset_mask, expected_offset_mask)
]
elif offset == 0 and offset_mask > 0:
if block_order_row_major and offset_mask < last_offset:
errors += [
'Mask of ' + band_name +
': offset of block (%d, %d) is smaller than previous block'
% (x, y)
]
offset = offset_mask
last_offset = offset
def ogr_vdv_8():
gdal.PushErrorHandler()
ds = ogr.GetDriverByName('VDV').CreateDataSource('/does/not_exist')
gdal.PopErrorHandler()
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
gdal.PushErrorHandler()
ds = ogr.GetDriverByName('VDV').CreateDataSource(
'/does/not_exist', options=['SINGLE_FILE=FALSE'])
gdal.PopErrorHandler()
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# Add layer in non writable directory
if sys.platform.startswith('linux'):
os.mkdir('tmp/ogr_vdv_8')
open('tmp/ogr_vdv_8/empty.x10',
'wb').write('tbl; foo\natr;\nfrm;\n'.encode('latin1'))
# 0555 = 365
os.chmod('tmp/ogr_vdv_8', 365)
try:
open('tmp/ogr_vdv_8/another_file', 'wb').close()
shutil.rmtree('tmp/ogr_vdv_8')
do_test = False
except:
do_test = True
if do_test:
ds = ogr.Open('tmp/ogr_vdv_8', update=1)
gdal.PushErrorHandler()
lyr = ds.CreateLayer('another_layer')
gdal.PopErrorHandler()
# 0755 = 493
os.chmod('tmp/ogr_vdv_8', 493)
ds = None
shutil.rmtree('tmp/ogr_vdv_8')
if lyr is not None:
gdaltest.post_reason('fail')
return 'fail'
out_filename = '/vsimem/vdv/ogr_vdv_8.x10'
ds = ogr.GetDriverByName('VDV').CreateDataSource(out_filename)
# File already exists
gdal.PushErrorHandler()
ds2 = ogr.GetDriverByName('VDV').CreateDataSource(out_filename)
gdal.PopErrorHandler()
if ds2 is not None:
gdaltest.post_reason('fail')
return 'fail'
if ds.TestCapability(ogr.ODsCCreateLayer) != 1:
gdaltest.post_reason('fail')
return 'fail'
lyr1 = ds.CreateLayer("lyr1")
if lyr1.TestCapability(ogr.OLCSequentialWrite) != 1:
gdaltest.post_reason('fail')
return 'fail'
if lyr1.TestCapability(ogr.OLCCreateField) != 1:
gdaltest.post_reason('fail')
return 'fail'
lyr1.ResetReading()
gdal.PushErrorHandler()
lyr1.GetNextFeature()
gdal.PopErrorHandler()
lyr1.CreateFeature(ogr.Feature(lyr1.GetLayerDefn()))
# Layer structure is now frozen
if lyr1.TestCapability(ogr.OLCCreateField) != 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.PushErrorHandler()
ret = lyr1.CreateField(ogr.FieldDefn('not_allowed'))
gdal.PopErrorHandler()
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
lyr2 = ds.CreateLayer("lyr2")
lyr2.CreateFeature(ogr.Feature(lyr2.GetLayerDefn()))
# Test interleaved writing
if lyr1.TestCapability(ogr.OLCSequentialWrite) != 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.PushErrorHandler()
ret = lyr1.CreateFeature(ogr.Feature(lyr1.GetLayerDefn()))
gdal.PopErrorHandler()
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
if lyr1.GetFeatureCount() != 1:
gdaltest.post_reason('fail')
return 'fail'
ds = None
# Test appending new layer to file without eof
gdal.FileFromMemBuffer(out_filename,
'tbl; foo\natr; atr\nfrm; char[40]\nrec; "foo"\n')
ds = ogr.Open(out_filename, update=1)
lyr = ds.CreateLayer('new_layer')
lyr.CreateField(ogr.FieldDefn('atr'))
f = ogr.Feature(lyr.GetLayerDefn())
f['atr'] = 'bar'
lyr.CreateFeature(f)
f = None
ds = None
expected = """tbl; foo
atr; atr
frm; char[40]
rec; "foo"
tbl; new_layer
atr; atr
frm; char[80]
rec; "bar"
end; 1
eof; 2
"""
f = gdal.VSIFOpenL(out_filename, 'rb')
got = gdal.VSIFReadL(1, 10000, f).decode('latin1')
gdal.VSIFCloseL(f)
if got != expected:
gdaltest.post_reason('fail')
print(got)
return 'fail'
# Test we are robust against missing end;
ds = ogr.Open(out_filename)
for i in range(2):
lyr = ds.GetLayer(i)
if lyr.GetFeatureCount() != 1:
gdaltest.post_reason('fail')
return 'fail'
lyr.ResetReading()
fc = 0
for f in lyr:
fc += 1
if fc != 1:
gdaltest.post_reason('fail')
return 'fail'
lyr = None
ds = None
# Test appending new layer to file without terminating \n
gdal.FileFromMemBuffer(
out_filename, 'tbl; foo\natr; atr\nfrm; char[40]\nrec; "foo"\neof; 1')
ds = ogr.Open(out_filename, update=1)
lyr = ds.CreateLayer('new_layer')
lyr.CreateField(ogr.FieldDefn('atr'))
f = ogr.Feature(lyr.GetLayerDefn())
f['atr'] = 'bar'
lyr.CreateFeature(f)
f = None
ds = None
f = gdal.VSIFOpenL(out_filename, 'rb')
got = gdal.VSIFReadL(1, 10000, f).decode('latin1')
gdal.VSIFCloseL(f)
if got != expected:
gdaltest.post_reason('fail')
print(got)
return 'fail'
gdal.Unlink(out_filename)
return 'success'
def eedai_4():
if gdaltest.eedai_drv is None:
return 'skip'
gdal.FileFromMemBuffer(
'/vsimem/ee/assets/image',
json.dumps({
'type':
'IMAGE',
'bands': [{
"id": "B1",
"dataType": {
"precision": "INTEGER",
"range": {
"max": 255
}
},
"grid": {
"crsCode": "EPSG:32610",
"affineTransform": {
"translateX": 499980,
"translateY": 4200000,
"scaleX": 60,
"scaleY": -60
},
"dimensions": {
"width": 1830,
"height": 1831
}
}
}, {
"id": "B2",
"dataType": {
"precision": "INTEGER",
"range": {
"max": 255
}
},
"grid": {
"crsCode": "EPSG:32610",
"affineTransform": {
"translateX": 499980,
"translateY": 4200000,
"scaleX": 60,
"scaleY": -60
},
"dimensions": {
"width": 1830,
"height": 1831
}
}
}, {
"id": "B3",
"dataType": {
"precision": "INTEGER",
"range": {
"max": 255
}
},
"grid": {
"crsCode": "EPSG:32610",
"affineTransform": {
"translateX": 499980,
"translateY": 4200000,
"scaleX": 60,
"scaleY": -60
},
"dimensions": {
"width": 1830,
"height": 1831
}
}
}]
}))
gdal.SetConfigOption('EEDA_BEARER', 'mybearer')
gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/')
ds = gdal.Open('EEDAI:image')
gdal.SetConfigOption('EEDA_URL', None)
mem_ds = gdal.GetDriverByName('MEM').Create('', 256, 256, 3)
mem_ds.GetRasterBand(1).Fill(127)
mem_ds.GetRasterBand(2).Fill(128)
mem_ds.GetRasterBand(3).Fill(129)
gdal.GetDriverByName('PNG').CreateCopy('/vsimem/out.png', mem_ds)
f = gdal.VSIFOpenL('/vsimem/out.png', 'rb')
png_data = gdal.VSIFReadL(1, 1000000, f)
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/out.png')
gdal.FileFromMemBuffer(
'/vsimem/ee/assets:getPixels&CUSTOMREQUEST=POST&POSTFIELDS={ "path": "image", "encoding": "PNG", "bandIds": [ "B1", "B2", "B3" ], "grid": { "origin": { "x": 499980.0, "y": 4200000.0 }, "pixelSize": { "x": 60.0, "y": -60.0 }, "dimensions": { "width": 256, "height": 256 } } }',
png_data)
got_data = ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1)
got_data = struct.unpack('B', got_data)[0]
if got_data != 127:
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
# Same with dataset RasterIO
got_data = ds.ReadRaster(0, 0, 1, 1)
got_data = struct.unpack('B' * 3, got_data)
if got_data != (127, 128, 129):
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
# Same after flushing cache
ds.FlushCache()
got_data = ds.ReadRaster(0, 0, 1, 1)
got_data = struct.unpack('B' * 3, got_data)
if got_data != (127, 128, 129):
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
# Sub-sampled query
gdal.FileFromMemBuffer(
'/vsimem/ee/assets:getPixels&CUSTOMREQUEST=POST&POSTFIELDS={ "path": "image", "encoding": "PNG", "bandIds": [ "B1", "B2", "B3" ], "grid": { "origin": { "x": 499980.0, "y": 4200000.0 }, "pixelSize": { "x": 120.0, "y": -120.06557377049181 }, "dimensions": { "width": 256, "height": 256 } } }',
png_data)
got_data = ds.GetRasterBand(1).ReadRaster(0,
0,
2,
2,
buf_xsize=1,
buf_ysize=1)
got_data = struct.unpack('B', got_data)[0]
if got_data != 127:
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
# Same after flushing cache with dataset RasterIO
ds.FlushCache()
got_data = ds.ReadRaster(0, 0, 2, 2, buf_xsize=1, buf_ysize=1)
got_data = struct.unpack('B' * 3, got_data)
if got_data != (127, 128, 129):
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
ds = None
gdal.SetConfigOption('EEDA_BEARER', None)
return 'success'
def jpeg_19():
for (width, height, iX) in [ (32, 32, 12), (25, 25, 8), (24, 25, 8) ]:
src_ds = gdal.GetDriverByName('GTiff').Create('/vsimem/jpeg_19.tif',
width, height, 1)
src_ds.CreateMaskBand(gdal.GMF_PER_DATASET)
src_ds.GetRasterBand(1).GetMaskBand().WriteRaster(
0, 0, iX,height, struct.pack('B' * 1, 255), 1, 1)
src_ds.GetRasterBand(1).GetMaskBand().WriteRaster(
iX, 0, width-iX, height, struct.pack('B' * 1, 0), 1, 1)
tiff_mask_data = src_ds.GetRasterBand(1).GetMaskBand().ReadRaster(
0, 0, width, height)
# Generate a JPEG file with a (default) LSB bit mask order
out_ds = gdal.GetDriverByName('JPEG').CreateCopy('/vsimem/jpeg_19.jpg',
src_ds)
out_ds = None
# Generate a JPEG file with a MSB bit mask order
gdal.SetConfigOption('JPEG_WRITE_MASK_BIT_ORDER', 'MSB')
out_ds = gdal.GetDriverByName('JPEG').CreateCopy(
'/vsimem/jpeg_19_msb.jpg', src_ds)
del out_ds
gdal.SetConfigOption('JPEG_WRITE_MASK_BIT_ORDER', None)
src_ds = None
# Check that the file are indeed different
statBuf = gdal.VSIStatL('/vsimem/jpeg_19.jpg')
f = gdal.VSIFOpenL('/vsimem/jpeg_19.jpg', 'rb')
data1 = gdal.VSIFReadL(1, statBuf.size, f)
gdal.VSIFCloseL(f)
statBuf = gdal.VSIStatL('/vsimem/jpeg_19_msb.jpg')
f = gdal.VSIFOpenL('/vsimem/jpeg_19_msb.jpg', 'rb')
data2 = gdal.VSIFReadL(1, statBuf.size, f)
gdal.VSIFCloseL(f)
if (width, height, iX) == (24, 25, 8):
if data1 != data2:
gdaltest.post_reason('fail')
return 'fail'
else:
if data1 == data2:
gdaltest.post_reason('fail')
return 'fail'
# Check the file with the LSB bit mask order
ds = gdal.Open('/vsimem/jpeg_19.jpg')
jpg_mask_data = ds.GetRasterBand(1).GetMaskBand().ReadRaster(
0, 0, width, height)
ds = None
if tiff_mask_data != jpg_mask_data:
gdaltest.post_reason('fail')
return 'fail'
# Check the file with the MSB bit mask order
ds = gdal.Open('/vsimem/jpeg_19_msb.jpg')
jpg_mask_data = ds.GetRasterBand(1).GetMaskBand().ReadRaster(
0, 0, width, height)
ds = None
if tiff_mask_data != jpg_mask_data:
gdaltest.post_reason('fail')
return 'fail'
gdal.GetDriverByName('GTiff').Delete('/vsimem/jpeg_19.tif')
gdal.GetDriverByName('JPEG').Delete('/vsimem/jpeg_19.jpg')
gdal.GetDriverByName('JPEG').Delete('/vsimem/jpeg_19_msb.jpg')
return 'success'
def vsigs_1():
try:
drv = gdal.GetDriverByName( 'HTTP' )
except:
drv = None
if drv is None:
return 'skip'
# RETODO: Bind to swig, change test
# Invalid header filename
gdal.ErrorReset()
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', '/i_dont/exist.py')
f = open_for_read('/vsigs/foo/bar')
if f is None:
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None)
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
data = gdal.VSIFReadL(1, 1, f)
last_err = gdal.GetLastErrorMsg()
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None)
gdal.VSIFCloseL(f)
if len(data) != 0:
gdaltest.post_reason('fail')
return 'fail'
if last_err.find('Cannot read') < 0:
gdaltest.post_reason('fail')
print(last_err)
return 'fail'
# Invalid content for header file
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', 'vsigs.py')
f = open_for_read('/vsigs/foo/bar')
if f is None:
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None)
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 1, f)
gdal.SetConfigOption('GDAL_HTTP_HEADER_FILE', None)
gdal.VSIFCloseL(f)
if len(data) != 0:
gdaltest.post_reason('fail')
return 'fail'
# Missing GS_SECRET_ACCESS_KEY
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find('GS_SECRET_ACCESS_KEY') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs_streaming/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find('GS_SECRET_ACCESS_KEY') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY')
# Missing GS_ACCESS_KEY_ID
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
if f is not None or gdal.VSIGetLastErrorMsg().find('GS_ACCESS_KEY_ID') < 0:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID')
# ERROR 1: The User Id you provided does not exist in our records.
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar.baz')
if f is not None or gdal.VSIGetLastErrorMsg() == '':
if f is not None:
gdal.VSIFCloseL(f)
if gdal.GetConfigOption('APPVEYOR') is not None:
return 'success'
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs_streaming/foo/bar.baz')
if f is not None or gdal.VSIGetLastErrorMsg() == '':
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def vsicrypt_4():
if not gdaltest.has_vsicrypt:
return 'skip'
test_file = '/vsicrypt/key=DONT_USE_IN_PROD,sector_size=32,file=/vsimem/file_enc.bin'
ref_file = '/vsimem/file.bin'
for seed in range(1000):
gdal.Unlink(test_file)
gdal.Unlink(ref_file)
test_f = gdal.VSIFOpenL(test_file, 'wb+')
ref_f = gdal.VSIFOpenL(ref_file, 'wb+')
import random
random.seed(seed)
for _ in range(20):
random_offset = random.randint(0, 1000)
gdal.VSIFSeekL(test_f, random_offset, 0)
gdal.VSIFSeekL(ref_f, random_offset, 0)
random_size = random.randint(1, 80)
random_content = ''.join([chr(40 + int(10 * random.random())) for _ in range(random_size)])
gdal.VSIFWriteL(random_content, 1, random_size, test_f)
gdal.VSIFWriteL(random_content, 1, random_size, ref_f)
if random.randint(0, 1) == 0:
random_offset = random.randint(0, 1500)
gdal.VSIFSeekL(test_f, random_offset, 0)
gdal.VSIFSeekL(ref_f, random_offset, 0)
random_size = random.randint(1, 80)
test_content = gdal.VSIFReadL(1, random_size, test_f)
ref_content = gdal.VSIFReadL(1, random_size, ref_f)
if test_content != ref_content:
print(seed)
print('Test content (%d):' % len(test_content))
print(test_content)
print('')
print('Ref content (%d):' % len(ref_content))
print(ref_content)
return 'fail'
gdal.VSIFSeekL(test_f, 0, 0)
gdal.VSIFSeekL(ref_f, 0, 0)
test_content = gdal.VSIFReadL(1, 100000, test_f)
ref_content = gdal.VSIFReadL(1, 100000, ref_f)
if test_content != ref_content:
print(seed)
print('Test content (%d):' % len(test_content))
print(test_content)
print('')
print('Ref content (%d):' % len(ref_content))
print(ref_content)
return 'fail'
gdal.Unlink(test_file)
gdal.Unlink(ref_file)
return 'success'
def vsigs_extra_1():
try:
drv = gdal.GetDriverByName( 'HTTP' )
except:
drv = None
if drv is None:
return 'skip'
if gdal.GetConfigOption('GS_SECRET_ACCESS_KEY') is None:
print('Missing GS_SECRET_ACCESS_KEY for running gdaltest_list_extra')
return 'skip'
elif gdal.GetConfigOption('GS_ACCESS_KEY_ID') is None:
print('Missing GS_ACCESS_KEY_ID for running gdaltest_list_extra')
return 'skip'
elif gdal.GetConfigOption('GS_RESOURCE') is None:
print('Missing GS_RESOURCE for running gdaltest_list_extra')
return 'skip'
f = open_for_read('/vsigs/' + gdal.GetConfigOption('GS_RESOURCE'))
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Same with /vsigs_streaming/
f = open_for_read('/vsigs_streaming/' + gdal.GetConfigOption('GS_RESOURCE'))
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsigs/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if gdal.VSIGetLastErrorMsg() == '':
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsigs_streaming/' + gdal.GetConfigOption('GS_RESOURCE') + '/invalid_resource.baz')
if f is not None:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def vsicrypt_6():
try:
import ctypes
except ImportError:
return 'skip'
import testnonboundtoswig
testnonboundtoswig.testnonboundtoswig_init()
if testnonboundtoswig.gdal_handle is None:
return 'skip'
testnonboundtoswig.gdal_handle.VSISetCryptKey.argtypes = [ctypes.c_char_p, ctypes.c_int]
testnonboundtoswig.gdal_handle.VSISetCryptKey.restype = None
# Set a valid key
testnonboundtoswig.gdal_handle.VSISetCryptKey('DONT_USE_IN_PROD'.encode('ASCII'), 16)
if not gdaltest.has_vsicrypt:
return 'skip'
fp = gdal.VSIFOpenL('/vsicrypt/add_key_check=yes,file=/vsimem/file.bin', 'wb+')
if fp is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
return 'fail'
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+')
if fp is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
return 'fail'
# Set a too short key
testnonboundtoswig.gdal_handle.VSISetCryptKey('bbc'.encode('ASCII'), 3)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Erase key
testnonboundtoswig.gdal_handle.VSISetCryptKey(None, 0)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
gdal.Unlink('/vsimem/file.bin')
return 'success'
def vsiaz_fake_write():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
# Test creation of BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:AigkrY7q66WCrx3JRKBte56k7kxV2cxB/ZyGNubxk5I=' or \
'Expect' not in h or h['Expect'] != '100-continue' or \
'Content-Length' not in h or h['Content-Length'] != '40000' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'BlockBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(40000).decode('ascii')
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Simulate illegal read
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
if len(ret) != 0:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
gdal.VSIFCloseL(f)
# Simulate illegal seek
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
# Simulate failure when putting BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
if gdal.VSIFSeekL(f, 0, 0) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if gdal.VSIFTellL(f) != 35000:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 35000, 0) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 0, 1) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFSeekL(f, 0, 2) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
if gdal.VSIFEofL(f) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(f)
return 'fail'
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFCloseL(f)
if ret == 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
# Simulate creation of BlockBob over an existing blob of incompatible type
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 409)
handler.add('DELETE', '/azure/blob/myaccount/test_copy/file.bin', 202)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
# Test creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:KimVui3ptY9D5ftLlsI7CNOgK36CNAEzsXqcuHskdEY=' or \
'Content-Length' not in h or h['Content-Length'] != '0' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '10' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(10).decode('ascii')
if content != '0123456789':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '6' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(6).decode('ascii')
if content != 'abcdef':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 16:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Test failed creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
# Test failed writing of a block of an AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
403)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
return 'success'
def vsicrypt_2():
if not gdaltest.has_vsicrypt:
return 'skip'
# Missing key
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid file
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'wb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid file
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid file
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'ab')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid access
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'foo')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Key to short
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=a,file=/vsimem/file.bin', 'wb+')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid signature
gdal.FileFromMemBuffer('/vsimem/file.bin', 'foo')
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Generate empty file
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'wb')
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
if fp is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsimem/file.bin', 'rb')
header = gdal.VSIFReadL(1, 1000, fp)
gdal.VSIFCloseL(fp)
if len(header) != 46:
gdaltest.post_reason('fail')
print(len(header))
return 'fail'
# Test shortening header
for i in range(46):
fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb')
gdal.VSIFWriteL(header, 1, 46 - 1 - i, fp)
gdal.VSIFCloseL(fp)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test corrupting all bytes of header
for i in range(46):
for val in (0, 127, 255):
fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb')
try:
new_byte = chr(val).encode('latin1')
except (UnicodeDecodeError, UnicodeEncodeError):
new_byte = chr(val)
header_new = header[0:i] + new_byte + header[i + 1:]
gdal.VSIFWriteL(header_new, 1, 46, fp)
gdal.VSIFCloseL(fp)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file='
'/vsimem/file.bin', 'rb')
if fp is not None:
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSICRYPT_IV', 'TOO_SHORT')
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file='
'/vsimem/file.bin', 'wb')
gdal.SetConfigOption('VSICRYPT_IV', None)
if fp is not None:
gdal.VSIFCloseL(fp)
# Inconsistent initial vector.
header = struct.pack('B' * 38,
86, 83, 73, 67, 82, 89, 80, 84, # signature
38, 0, # header size
1, # major
0, # minor
0, 2, # sector size
0, # alg
0, # mode
8, # size of IV (should be 16)
32, 13, 169, 71, 154, 208, 22, 32, # IV
0, 0, # size of free text
0, # size of key check
0, 0, 0, 0, 0, 0, 0, 0, # size of unencrypted file
0, 0 # size of extra content
)
fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb')
gdal.VSIFWriteL(header, 1, len(header), fp)
gdal.VSIFCloseL(fp)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Inconsistent initial vector with key check.
header = struct.pack('B' * 39,
86, 83, 73, 67, 82, 89, 80, 84, # signature
39, 0, # header size
1, # major
0, # minor
0, 2, # sector size
0, # alg
0, # mode
8, # size of IV (should be 16)
32, 13, 169, 71, 154, 208, 22, 32, # IV
0, 0, # size of free text
1, # size of key check
0, # key check
0, 0, 0, 0, 0, 0, 0, 0, # size of unencrypted file
0, 0 # size of extra content
)
fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb')
gdal.VSIFWriteL(header, 1, len(header), fp)
gdal.VSIFCloseL(fp)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test reading with wrong key
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'wb')
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin', 'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content == 'hello':
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=short_key,file=/vsimem/file.bin', 'ab')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test reading with wrong key with add_key_check
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,add_key_check=yes,file=/vsimem/file.bin', 'wb')
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=short_key,file=/vsimem/file.bin', 'ab')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin', 'ab')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test creating with potentially not build-in alg:
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/alg=blowfish,key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'wb')
if fp is not None:
gdal.VSIFCloseL(fp)
# Invalid sector_size
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,sector_size=1,file=/vsimem/file.bin', 'wb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Sector size (16) should be at least twice larger than the block size (16) in CBC_CTS
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,sector_size=16,mode=CBC_CTS,file=/vsimem/file.bin', 'wb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
gdal.Unlink('/vsimem/file.bin')
return 'success'
def vsifile_5():
fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'wb')
ref_data = ''.join(['%08X' % i for i in range(5 * 32768)])
gdal.VSIFWriteL(ref_data, 1, len(ref_data), fp)
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSI_CACHE', 'YES')
for i in range(3):
if i == 0:
gdal.SetConfigOption('VSI_CACHE_SIZE', '0')
elif i == 1:
gdal.SetConfigOption('VSI_CACHE_SIZE', '65536')
else:
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'rb')
gdal.VSIFSeekL(fp, 50000, 0)
if gdal.VSIFTellL(fp) != 50000:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 50000, 1)
if gdal.VSIFTellL(fp) != 100000:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 0, 2)
if gdal.VSIFTellL(fp) != 5 * 32768 * 8:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFReadL(1, 1, fp)
gdal.VSIFSeekL(fp, 0, 0)
data = gdal.VSIFReadL(1, 3 * 32768, fp)
if data.decode('ascii') != ref_data[0:3 * 32768]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 16384, 0)
data = gdal.VSIFReadL(1, 5 * 32768, fp)
if data.decode('ascii') != ref_data[16384:16384 + 5 * 32768]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
data = gdal.VSIFReadL(1, 50 * 32768, fp)
if data[0:1130496].decode('ascii') != ref_data[16384 + 5 * 32768:]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
gdal.Unlink('tmp/vsifile_5.bin')
return 'success'
def test_vrt_read_10():
src_ds = gdal.Open('data/byte.tif')
mem_ds = gdal.GetDriverByName('GTiff').CreateCopy(
'/vsimem/vrt_read_10.tif', src_ds)
vrt_ds = gdal.GetDriverByName('VRT').CreateCopy('/vsimem/vrt_read_10.vrt',
mem_ds)
vrt_hist = vrt_ds.GetRasterBand(1).GetHistogram()
mem_hist = mem_ds.GetRasterBand(1).GetHistogram()
mem_ds = None
vrt_ds = None
f = gdal.VSIFOpenL('/vsimem/vrt_read_10.vrt', 'rb')
content = gdal.VSIFReadL(1, 10000, f).decode('ascii')
gdal.VSIFCloseL(f)
assert vrt_hist == mem_hist
assert '<Histograms>' in content
# Single source optimization
for i in range(2):
gdal.FileFromMemBuffer(
'/vsimem/vrt_read_10.vrt',
"""<VRTDataset rasterXSize="20" rasterYSize="20">
<VRTRasterBand dataType="Byte" band="1">
<SimpleSource>
<SourceFilename relativeToVRT="1">vrt_read_10.tif</SourceFilename>
</SimpleSource>
</VRTRasterBand>
</VRTDataset>""")
ds = gdal.Open('/vsimem/vrt_read_10.vrt')
if i == 0:
ds.GetRasterBand(1).GetDefaultHistogram()
else:
ds.GetRasterBand(1).GetHistogram()
ds = None
f = gdal.VSIFOpenL('/vsimem/vrt_read_10.vrt', 'rb')
content = gdal.VSIFReadL(1, 10000, f).decode('ascii')
gdal.VSIFCloseL(f)
assert '<Histograms>' in content
# Two sources general case
for i in range(2):
gdal.FileFromMemBuffer(
'/vsimem/vrt_read_10.vrt',
"""<VRTDataset rasterXSize="20" rasterYSize="20">
<VRTRasterBand dataType="Byte" band="1">
<SimpleSource>
<SourceFilename relativeToVRT="1">vrt_read_10.tif</SourceFilename>
</SimpleSource>
<SimpleSource>
<SourceFilename relativeToVRT="1">vrt_read_10.tif</SourceFilename>
</SimpleSource>
</VRTRasterBand>
</VRTDataset>""")
ds = gdal.Open('/vsimem/vrt_read_10.vrt')
if i == 0:
ds.GetRasterBand(1).GetDefaultHistogram()
else:
ds.GetRasterBand(1).GetHistogram()
ds = None
f = gdal.VSIFOpenL('/vsimem/vrt_read_10.vrt', 'rb')
content = gdal.VSIFReadL(1, 10000, f).decode('ascii')
gdal.VSIFCloseL(f)
assert '<Histograms>' in content
gdal.GetDriverByName('GTiff').Delete('/vsimem/vrt_read_10.tif')
gdal.GetDriverByName('VRT').Delete('/vsimem/vrt_read_10.vrt')
def vsicrypt_3():
if not gdaltest.has_vsicrypt:
return 'skip'
for options in [
'sector_size=16', 'alg=AES', 'alg=DES_EDE2', 'alg=DES_EDE3',
'alg=SKIPJACK', 'alg=invalid', 'mode=CBC', 'mode=CFB', 'mode=OFB',
'mode=CTR', 'mode=CBC_CTS', 'mode=invalid',
'freetext=my_free_text', 'add_key_check=yes'
]:
gdal.Unlink('/vsimem/file.bin')
if options == 'alg=invalid' or options == 'mode=invalid':
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,%s,file=/vsimem/file.bin'
% options, 'wb')
else:
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,%s,file=/vsimem/file.bin'
% options, 'wb')
if fp is None:
gdaltest.post_reason('fail')
print(options)
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,file=/vsimem/file.bin',
'r')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
print(options)
return 'fail'
# Some of those algs might be missing
for options in [
'alg=Blowfish', 'alg=Camellia', 'alg=CAST256', 'alg=MARS',
'alg=IDEA', 'alg=RC5', 'alg=RC6', 'alg=Serpent', 'alg=SHACAL2',
'alg=Twofish', 'alg=XTEA'
]:
gdal.Unlink('/vsimem/file.bin')
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,%s,file=/vsimem/file.bin' %
options, 'wb')
if fp is not None:
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
print(options)
return 'fail'
# Test key generation
# Do NOT set VSICRYPT_CRYPTO_RANDOM=NO in production. This is just to speed up tests !
gdal.SetConfigOption("VSICRYPT_CRYPTO_RANDOM", "NO")
fp = gdal.VSIFOpenL(
'/vsicrypt/key=GENERATE_IT,add_key_check=yes,file=/vsimem/file.bin',
'wb')
gdal.SetConfigOption("VSICRYPT_CRYPTO_RANDOM", None)
# Get the generated random key
key_b64 = gdal.GetConfigOption('VSICRYPT_KEY_B64')
if key_b64 is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
print(options)
return 'fail'
gdal.SetConfigOption('VSICRYPT_KEY_B64', None)
fp = gdal.VSIFOpenL('/vsicrypt/key_b64=%s,file=/vsimem/file.bin' % key_b64,
'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
print(options)
return 'fail'
with gdaltest.error_handler():
statRes = gdal.VSIStatL('/vsicrypt//vsimem/file.bin')
if statRes is not None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.Rename('/vsicrypt//vsimem/file.bin',
'/vsicrypt//vsimem/subdir_crypt/file.bin')
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
ret = gdal.Rename('/vsicrypt//vsimem/subdir_crypt/file.bin',
'/vsimem/subdir_crypt/file2.bin')
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
dir_content = gdal.ReadDir('/vsicrypt//vsimem/subdir_crypt')
if dir_content != ['file2.bin']:
gdaltest.post_reason('fail')
print(dir_content)
return 'fail'
gdal.Unlink('/vsimem/subdir_crypt/file2.bin')
return 'success'
def test_vsigs_read_credentials_refresh_token_default_gdal_app():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN',
'http://localhost:%d/accounts.google.com/o/oauth2/token' % gdaltest.webserver_port)
gdal.SetConfigOption('GS_OAUTH2_REFRESH_TOKEN', 'REFRESH_TOKEN')
with gdaltest.error_handler():
assert gdal.GetSignedURL('/vsigs/foo/bar') is None
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(request.headers['Content-Length'])).decode('ascii')
if content != 'refresh_token=REFRESH_TOKEN&client_id=265656308688.apps.googleusercontent.com&client_secret=0IbTUDOYzaL6vnIdWTuQnvLz&grant_type=refresh_token':
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST', '/accounts.google.com/o/oauth2/token', custom_method=method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN', None)
gdal.SetConfigOption('GS_OAUTH2_REFRESH_TOKEN', '')
