def vsizip_11():
content = gdal.ReadDir("/vsizip/data/utf8.zip")
ok = 0
try:
local_vars = {'content': content, 'ok': ok}
exec(
"if content == [u'\u0430\u0431\u0432\u0433\u0434\u0435', u'\u0436\u0437\u0438\u0439\u043a\u043b']: ok = 1",
None, local_vars)
ok = local_vars['ok']
except:
if content == [
'\u0430\u0431\u0432\u0433\u0434\u0435',
'\u0436\u0437\u0438\u0439\u043a\u043b'
]:
ok = 1
if ok == 0:
gdaltest.post_reason('bad content')
print(content)
return 'fail'
return 'success'
def vsizip_10():
gdal.SetConfigOption('CPL_ZIP_ENCODING', 'CP866')
content = gdal.ReadDir("/vsizip/data/cp866.zip")
gdal.SetConfigOption('CPL_ZIP_ENCODING', None)
ok = 0
try:
local_vars = { 'content': content, 'ok': ok }
exec("if content == [u'\u0430\u0431\u0432\u0433\u0434\u0435', u'\u0436\u0437\u0438\u0439\u043a\u043b']: ok = 1", None, local_vars)
ok = local_vars['ok']
except:
if content == ['\u0430\u0431\u0432\u0433\u0434\u0435', '\u0436\u0437\u0438\u0439\u043a\u043b']:
ok = 1
if ok == 0:
if gdal.GetLastErrorMsg().find('Recode from CP866 to UTF-8 not supported') >= 0:
return 'skip'
gdaltest.post_reason('bad content')
print(content)
return 'fail'
return 'success'
def vsizip_10():
gdal.SetConfigOption('CPL_ZIP_ENCODING', 'CP866')
content = gdal.ReadDir("/vsizip/data/cp866.zip")
gdal.SetConfigOption('CPL_ZIP_ENCODING', None)
ok = 0
try:
exec(
"if content == [u'\u0430\u0431\u0432\u0433\u0434\u0435', u'\u0436\u0437\u0438\u0439\u043a\u043b']: ok = 1"
)
except:
if content == [
'\u0430\u0431\u0432\u0433\u0434\u0435',
'\u0436\u0437\u0438\u0439\u043a\u043b'
]:
ok = 1
if ok == 0:
gdaltest.post_reason('bad content')
print(content)
return 'fail'
return 'success'
def vsis3_3():
if gdaltest.webserver_port == 0:
return 'skip'
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket2/a_dir/resource3.bin', 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsis3/s3_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL(
'/vsis3/s3_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsis3/s3_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def test_vsiaz_no_sign_request():
if not gdaltest.built_against_curl():
pytest.skip()
gdal.VSICurlClearCache()
with gdaltest.config_options({ 'AZURE_STORAGE_ACCOUNT': 'naipblobs', 'AZURE_NO_SIGN_REQUEST': 'YES'}):
actual_url = gdal.GetActualURL('/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif')
assert actual_url == 'https://naipblobs.blob.core.windows.net/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif'
assert actual_url == gdal.GetSignedURL('/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif')
f = open_for_read('/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif')
if f is None:
if gdaltest.gdalurlopen('https://naipblobs.blob.core.windows.net/naip/v002/al/2015/al_100cm_2015/30086/m_3008601_ne_16_1_20150804.tif') is None:
pytest.skip('cannot open URL')
pytest.fail()
gdal.VSIFCloseL(f)
assert 'm_3008601_ne_16_1_20150804.tif' in gdal.ReadDir('/vsiaz/naip/v002/al/2015/al_100cm_2015/30086/')
gdal.VSICurlClearCache()
assert stat.S_ISDIR(gdal.VSIStatL('/vsiaz/naip').mode)
gdal.VSICurlClearCache()
assert stat.S_ISDIR(gdal.VSIStatL('/vsiaz/naip/').mode)
gdal.VSICurlClearCache()
assert gdal.VSIStatL('/vsiaz/naip_i_dont_exist') is None
gdal.VSICurlClearCache()
assert stat.S_ISDIR(gdal.VSIStatL('/vsiaz/naip/v002').mode)
def test_vsicurl_test_parse_html_filelist_apache():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/mydir/', 200, {},
"""<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<html>
<head>
<title>Index of /mydir</title>
</head>
<body>
<h1>Index of /mydir</h1>
<table><tr><th><img src="/icons/blank.gif" alt="[ICO]"></th><th><a href="?C=N;O=D">Name</a></th><th><a href="?C=M;O=A">Last modified</a></th><th><a href="?C=S;O=A">Size</a></th><th><a href="?C=D;O=A">Description</a></th></tr><tr><th colspan="5"><hr></th></tr>
<tr><td valign="top"><img src="/icons/back.gif" alt="[DIR]"></td><td><a href="/gdal/data/">Parent Directory</a></td><td> </td><td align="right"> - </td><td> </td></tr>
<tr><td valign="top"><img src="/icons/image2.gif" alt="[IMG]"></td><td><a href="foo.tif">foo.tif</a></td><td align="right">17-May-2010 12:26 </td><td align="right"> 90K</td><td> </td></tr>
<tr><td valign="top"><img src="/icons/image2.gif" alt="[IMG]"></td><td><a href="foo%20with%20space.tif">foo with space.tif</a></td><td align="right">15-Jan-2007 11:02 </td><td align="right">736 </td><td> </td></tr>
<tr><th colspan="5"><hr></th></tr>
</table>
</body></html>""")
with webserver.install_http_handler(handler):
fl = gdal.ReadDir('/vsicurl/http://localhost:%d/mydir' %
gdaltest.webserver_port)
assert fl == ['foo.tif', 'foo%20with%20space.tif']
assert gdal.VSIStatL(
'/vsicurl/http://localhost:%d/mydir/foo%%20with%%20space.tif' %
gdaltest.webserver_port, gdal.VSI_STAT_EXISTS_FLAG) is not None
handler = webserver.SequentialHandler()
handler.add('HEAD', '/mydir/i_dont_exist', 404, {})
with webserver.install_http_handler(handler):
assert gdal.VSIStatL(
'/vsicurl/http://localhost:%d/mydir/i_dont_exist' %
gdaltest.webserver_port, gdal.VSI_STAT_EXISTS_FLAG) is None
def fetchManifest(self):
"""
Fetch manifest.safe
"""
try:
res = gdal.ReadDir(self.srcsafe)
if 'manifest.safe' not in res:
raise Exception("Manifest file not found in "
"{0}".format(self.srcsafe))
except:
raise Exception("Could not fetch manifest from "
"{0}".format(self.srcsafe))
try:
with zipfile.ZipFile(self.dest, mode='a') as myzip:
with myzip.open(os.path.join(self.zip2safe, 'manifest.safe'),
'w') as myfile:
logging.info('Fetching manifest.safe')
self.downloadFile(
os.path.join(self.srcsafe, 'manifest.safe'), myfile)
except:
raise Exception("Could not download manifest.safe from "
"{0} to {1}".format(self.url, self.dest))
def vsicurl_11():
if not gdaltest.run_slow_tests():
return 'skip'
try:
drv = gdal.GetDriverByName( 'HTTP' )
except:
drv = None
if drv is None:
return 'skip'
f = gdal.VSIFOpenL('/vsicurl/http://download.osgeo.org/gdal/data/bmp/Bug2236.bmp', 'rb')
if f is None:
return 'skip'
gdal.VSIFSeekL(f, 1000000, 0)
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
filelist = gdal.ReadDir('/vsicurl/http://download.osgeo.org/gdal/data/gtiff')
if filelist is None or len(filelist) == 0:
return 'fail'
return 'success'
def gdal_cp_recurse(srcdir, targetdir, progress, skip_failure):
if srcdir[-1] == '/':
srcdir = srcdir[0:len(srcdir) - 1]
lst = gdal.ReadDir(srcdir)
if lst is None:
print('%s is not a directory' % srcdir)
return -1
if gdal.VSIStatL(targetdir) is None:
gdal.Mkdir(targetdir, int('0755', 8))
for srcfile in lst:
if srcfile == '.' or srcfile == '..':
continue
fullsrcfile = srcdir + '/' + srcfile
statBuf = gdal.VSIStatL(fullsrcfile, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG)
if statBuf.IsDirectory():
ret = gdal_cp_recurse(fullsrcfile, targetdir + '/' + srcfile, progress, skip_failure)
else:
ret = gdal_cp_single(fullsrcfile, targetdir, progress)
if ret == -2 or (ret == -1 and not skip_failure):
return ret
return 0
def test_vsitar_bug_675():
content = gdal.ReadDir('/vsitar/data/tar_with_star_base256_fields.tar')
assert len(content) == 1
def test_vsifile_10():
gdal.FileFromMemBuffer('/vsimem/vsifile_10.tar',
"""FUZZER_FRIENDLY_ARCHIVE
***NEWFILE***:test.txt
abc***NEWFILE***:huge.txt
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
0123456789012345678901234567890123456789012345678901234567890123456789012345678X
***NEWFILE***:small.txt
a""")
contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar')
if contents is None:
gdal.Unlink('/vsimem/vsifile_10.tar')
pytest.skip()
assert contents == ['test.txt', 'huge.txt', 'small.txt']
assert gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/test.txt').size == 3
assert gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/huge.txt').size == 3888
assert gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/small.txt').size == 1
gdal.FileFromMemBuffer('/vsimem/vsifile_10.tar',
"""FUZZER_FRIENDLY_ARCHIVE
***NEWFILE***:x
abc""")
contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar')
assert contents == ['x']
gdal.FileFromMemBuffer('/vsimem/vsifile_10.tar',
"""FUZZER_FRIENDLY_ARCHIVE
***NEWFILE***:x
abc***NEWFILE***:""")
contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar')
assert contents == ['x']
gdal.Unlink('/vsimem/vsifile_10.tar')
def vsiaz_extra_1():
if not gdaltest.built_against_curl():
return 'skip'
az_resource = gdal.GetConfigOption('/azure/blob/myaccount/az_RESOURCE')
if az_resource is None:
print('Missing AZ_RESOURCE for running gdaltest_list_extra')
return 'skip'
if az_resource.find('/') < 0:
path = '/vsiaz/' + az_resource
statres = gdal.VSIStatL(path)
if statres is None or not stat.S_ISDIR(statres.mode):
gdaltest.post_reason('fail')
print('%s is not a valid bucket' % path)
return 'fail'
readdir = gdal.ReadDir(path)
if readdir is None:
gdaltest.post_reason('fail')
print('ReadDir() should not return empty list')
return 'fail'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
if gdal.VSIStatL(subpath) is None:
gdaltest.post_reason('fail')
print('Stat(%s) should not return an error' % subpath)
return 'fail'
unique_id = 'vsiaz_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
if ret < 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) should not return an error' % subpath)
return 'fail'
readdir = gdal.ReadDir(path)
if unique_id not in readdir:
gdaltest.post_reason('fail')
print('ReadDir(%s) should contain %s' % (path, unique_id))
print(readdir)
return 'fail'
ret = gdal.Mkdir(subpath, 0)
if ret == 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) repeated should return an error' % subpath)
return 'fail'
ret = gdal.Rmdir(subpath)
if ret < 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) should not return an error' % subpath)
return 'fail'
readdir = gdal.ReadDir(path)
if unique_id in readdir:
gdaltest.post_reason('fail')
print('ReadDir(%s) should not contain %s' % (path, unique_id))
print(readdir)
return 'fail'
ret = gdal.Rmdir(subpath)
if ret == 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) repeated should return an error' % subpath)
return 'fail'
ret = gdal.Mkdir(subpath, 0)
if ret < 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) should not return an error' % subpath)
return 'fail'
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
if ret == 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) on non empty directory should return an error' %
subpath)
return 'fail'
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
if data != 'hello':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test.txt')
if ret < 0:
gdaltest.post_reason('fail')
print('Unlink(%s) should not return an error' %
(subpath + '/test.txt'))
return 'fail'
ret = gdal.Rmdir(subpath)
if ret < 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) should not return an error' % subpath)
return 'fail'
return 'success'
f = open_for_read('/vsiaz/' + az_resource)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Same with /vsiaz_streaming/
f = open_for_read('/vsiaz_streaming/' + az_resource)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsiaz/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if gdal.VSIGetLastErrorMsg() == '':
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsiaz_streaming/' + az_resource +
'/invalid_resource.baz')
if f is not None:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def vsiaz_fake_readdir():
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%2F&restype=container',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Blobs>
<Blob>
<Name>a_dir/resource3.bin</Name>
<Properties>
<Last-Modified>01 Jan 1970 00:00:01</Last-Modified>
<Content-Length>123456</Content-Length>
</Properties>
</Blob>
</Blobs>
</EnumerationResults>
""")
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla&prefix=a_dir%2F&restype=container',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<Blobs>
<Blob>
<Name>a_dir/resource4.bin</Name>
<Properties>
<Last-Modified>16 Oct 2016 12:34:56</Last-Modified>
<Content-Length>456789</Content-Length>
</Properties>
</Blob>
<BlobPrefix>
<Name>a_dir/subdir/</Name>
</BlobPrefix>
</Blobs>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL(
'/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
print(gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size)
return 'fail'
if gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
if dir_contents is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=error_test%2F&restype=container',
500)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/error_test/')
if dir_contents is not None:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# List containers (empty result)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/azure/blob/myaccount/?comp=list', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults ServiceEndpoint="https://myaccount.blob.core.windows.net">
<Containers/>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
if dir_contents != ['.']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
gdal.VSICurlClearCache()
# List containers
handler = webserver.SequentialHandler()
handler.add(
'GET', '/azure/blob/myaccount/?comp=list', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer1</Name>
</Container>
</Containers>
<NextMarker>bla</NextMarker>
</EnumerationResults>
""")
handler.add(
'GET', '/azure/blob/myaccount/?comp=list&marker=bla', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer2</Name>
</Container>
</Containers>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
if dir_contents != ['mycontainer1', 'mycontainer2']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
return 'success'
def test_vsicrypt_3():
if not gdaltest.has_vsicrypt:
pytest.skip()
for options in [
'sector_size=16', 'alg=AES', 'alg=DES_EDE2', 'alg=DES_EDE3',
'alg=SKIPJACK', 'alg=invalid', 'mode=CBC', 'mode=CFB', 'mode=OFB',
'mode=CTR', 'mode=CBC_CTS', 'mode=invalid',
'freetext=my_free_text', 'add_key_check=yes'
]:
gdal.Unlink('/vsimem/file.bin')
if options == 'alg=invalid' or options == 'mode=invalid':
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,%s,file=/vsimem/file.bin'
% options, 'wb')
else:
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,%s,file=/vsimem/file.bin'
% options, 'wb')
assert fp is not None, options
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,file=/vsimem/file.bin',
'r')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
assert content == 'hello', options
# Some of those algs might be missing
for options in [
'alg=Blowfish', 'alg=Camellia', 'alg=CAST256', 'alg=MARS',
'alg=IDEA', 'alg=RC5', 'alg=RC6', 'alg=Serpent', 'alg=SHACAL2',
'alg=Twofish', 'alg=XTEA'
]:
gdal.Unlink('/vsimem/file.bin')
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,%s,file=/vsimem/file.bin' %
options, 'wb')
if fp is not None:
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
assert content == 'hello', options
# Test key generation
# Do NOT set VSICRYPT_CRYPTO_RANDOM=NO in production. This is just to speed up tests !
gdal.SetConfigOption("VSICRYPT_CRYPTO_RANDOM", "NO")
fp = gdal.VSIFOpenL(
'/vsicrypt/key=GENERATE_IT,add_key_check=yes,file=/vsimem/file.bin',
'wb')
gdal.SetConfigOption("VSICRYPT_CRYPTO_RANDOM", None)
# Get the generated random key
key_b64 = gdal.GetConfigOption('VSICRYPT_KEY_B64')
assert key_b64 is not None
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
assert content == 'hello', options
gdal.SetConfigOption('VSICRYPT_KEY_B64', None)
fp = gdal.VSIFOpenL('/vsicrypt/key_b64=%s,file=/vsimem/file.bin' % key_b64,
'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
assert content == 'hello', options
with gdaltest.error_handler():
statRes = gdal.VSIStatL('/vsicrypt//vsimem/file.bin')
assert statRes is None
ret = gdal.Rename('/vsicrypt//vsimem/file.bin',
'/vsicrypt//vsimem/subdir_crypt/file.bin')
assert ret == 0
ret = gdal.Rename('/vsicrypt//vsimem/subdir_crypt/file.bin',
'/vsimem/subdir_crypt/file2.bin')
assert ret == 0
dir_content = gdal.ReadDir('/vsicrypt//vsimem/subdir_crypt')
assert dir_content == ['file2.bin']
gdal.Unlink('/vsimem/subdir_crypt/file2.bin')
def get_observations(self,
url: str,
product: str = "NBART",
onlymask: bool = False,
**args) -> tuple:
"""
Get the NRT observation from the S3 or public (HTTP) bucket and load the
data into memory in a numpy array of shape (ysize, xsize, nbands). This is
assuming the DEA package format.
"""
bands = args.pop("bands_required", self.bands)
pkg = parse_pkg(url)
if url.startswith("/vsis3"):
stripped_url = url.replace("/vsis3/dea-public-data",
"https://data.dea.ga.gov.au")
elif url.startswith("/vsicurl"):
stripped_url = url.replace("/vsicurl/", "")
else:
stripped_url = url
fn = [
fn for fn in gdal.ReadDir(f"{url}/QA") if fn.endswith("FMASK.TIF")
][0]
fn = f"{url}/QA/{fn}"
fd = self.openfile(fn)
mask = fd.ReadAsArray()
pnodata = np.count_nonzero(mask == 0) / np.prod(mask.shape)
pclear = np.count_nonzero(mask == 1) / np.prod(mask.shape)
log(f"Package: {pkg}")
log(f"Thumbnail: {stripped_url}/{product}/{product}_THUMBNAIL.JPG")
log(f"Location: {stripped_url}/map.html")
log(f"Pixels: {mask.shape[0]} x {mask.shape[1]}")
log(f"Clear %: {pclear:.4f}")
geo = fd.GetGeoTransform()
prj = fd.GetProjection()
ysize = mask.shape[0]
xsize = mask.shape[1]
if onlymask:
return (geo, prj, mask[:, :, np.newaxis])
log("# Loading data")
hc = checksum_array(mask)
log(f"MASK (sha256:{hc})")
# Handle possible name changes done by DEA between NRT and Archive by creating a mapping
# between bands and filenames.
fns = [
fn for fn in gdal.ReadDir(f"{url}/{product}")
if fn.endswith(".TIF") or fn.endswith(".tif")
]
bfm = {fn.split(f"{product}_")[1].split(".")[0]: fn for fn in fns}
# Load other bands
data = np.empty((ysize, xsize, len(bands)), dtype=np.float32)
for i, band in enumerate(bands):
fn = f"{url}/{product}/{bfm[band]}"
fd = self.openfile(fn)
fd.ReadAsArray(buf_obj=data[:, :, i],
buf_ysize=ysize,
buf_xsize=xsize)
hc = checksum_array(data[:, :, i])
log(f" {band} (sha256:{hc})")
nodata = fd.GetRasterBand(1).GetNoDataValue()
data[data == nodata] = np.nan
log(f"\nShape: {data.shape}")
return (geo, prj, data, mask)
def vsifile_10():
gdal.FileFromMemBuffer('/vsimem/vsifile_10.tar',
"""FUZZER_FRIENDLY_ARCHIVE
***NEWFILE***:test.txt
abc***NEWFILE***:huge.txt
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
01234567890123456789012345678901234567890123456789012345678901234567890123456789
0123456789012345678901234567890123456789012345678901234567890123456789012345678X
***NEWFILE***:small.txt
a""")
contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar')
if contents is None:
gdal.Unlink('/vsimem/vsifile_10.tar')
return 'skip'
if contents != ['test.txt', 'huge.txt', 'small.txt']:
gdaltest.post_reason('fail')
print(contents)
return 'fail'
if gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/test.txt').size != 3:
gdaltest.post_reason('fail')
print(gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/test.txt').size)
return 'fail'
if gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/huge.txt').size != 3888:
gdaltest.post_reason('fail')
print(gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/huge.txt').size)
return 'fail'
if gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/small.txt').size != 1:
gdaltest.post_reason('fail')
print(gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/small.txt').size)
return 'fail'
gdal.FileFromMemBuffer('/vsimem/vsifile_10.tar',
"""FUZZER_FRIENDLY_ARCHIVE
***NEWFILE***:x
abc""")
contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar')
if contents != ['x']:
gdaltest.post_reason('fail')
print(contents)
return 'fail'
gdal.FileFromMemBuffer('/vsimem/vsifile_10.tar',
"""FUZZER_FRIENDLY_ARCHIVE
***NEWFILE***:x
abc***NEWFILE***:""")
contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar')
if contents != ['x']:
gdaltest.post_reason('fail')
print(contents)
return 'fail'
gdal.Unlink('/vsimem/vsifile_10.tar')
return 'success'
def ogr_plscenes_data_v1_nominal():
if gdaltest.plscenes_drv is None:
return 'skip'
gdal.FileFromMemBuffer('/vsimem/data_v1/item-types',
"""{ "item_types": [
{"display_description" : "display_description",
"display_name" : "display_name",
"id": "PSOrthoTile"}
]}""")
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
ds = gdal.OpenEx('PLScenes:', gdal.OF_VECTOR, open_options=['VERSION=data_v1', 'API_KEY=foo'])
gdal.SetConfigOption('PL_URL', None)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
ds = gdal.OpenEx('PLScenes:version=data_v1,api_key=foo,FOLLOW_LINKS=YES', gdal.OF_VECTOR)
gdal.SetConfigOption('PL_URL', None)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
lyr = ds.GetLayer(0)
if lyr.GetName() != 'PSOrthoTile':
gdaltest.post_reason('fail')
return 'fail'
if lyr.TestCapability(ogr.OLCFastFeatureCount) != 1 or \
lyr.TestCapability(ogr.OLCStringsAsUTF8) != 1 or \
lyr.TestCapability(ogr.OLCRandomRead) != 0:
gdaltest.post_reason('fail')
return 'fail'
# Different serialization depending on libjson versions
gdal.FileFromMemBuffer("""/vsimem/data_v1/stats&POSTFIELDS={"interval":"year","item_types":["PSOrthoTile"],"filter":{"type":"AndFilter","config":[{"type":"RangeFilter","field_name":"cloud_cover","config":{"gte":0.000000}}]}}""", """{ "buckets": [ { "count": 1 }, { "count": 1} ] }""")
gdal.FileFromMemBuffer("""/vsimem/data_v1/stats&POSTFIELDS={"interval":"year","item_types":["PSOrthoTile"],"filter":{"type":"AndFilter","config":[{"type":"RangeFilter","field_name":"cloud_cover","config":{"gte":0}}]}}""", """{ "buckets": [ { "count": 1 }, { "count": 1} ] }""")
gdal.FileFromMemBuffer("""/vsimem/data_v1/stats&POSTFIELDS={"interval":"year","item_types":["PSOrthoTile"],"filter":{"type":"AndFilter","config":[{"type":"RangeFilter","field_name":"cloud_cover","config":{"gte":0.0}}]}}""", """{ "buckets": [ { "count": 1 }, { "count": 1} ] }""")
if lyr.GetFeatureCount() != 2:
gdaltest.post_reason('fail')
return 'fail'
if lyr.GetGeomType() != ogr.wkbMultiPolygon:
gdaltest.post_reason('fail')
return 'fail'
ext = lyr.GetExtent()
if ext != (-180.0, 180.0, -90.0, 90.0):
gdaltest.post_reason('fail')
print(ext)
return 'fail'
field_count = lyr.GetLayerDefn().GetFieldCount()
if field_count != 68:
gdaltest.post_reason('fail')
print(field_count)
return 'fail'
# Regular /items/ fetching
gdal.FileFromMemBuffer("""/vsimem/data_v1/quick-search?_page_size=250&POSTFIELDS={"item_types":["PSOrthoTile"],"filter":{"type":"AndFilter","config":[]}}""",
"""{
"_links":
{
"_next": "/vsimem/data_v1/quick-search?page=2"
},
"features" : [
{
"id": "id",
"_links" : {
"_self" : "self",
"assets" : "/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets"
},
"_permissions" : [ "download" ],
"properties": {
"acquired": "2016/02/11 12:34:56.789+00",
"anomalous_pixels": 1.23,
"columns": 1,
"item_type": "foo",
"ground_control": true
},
"geometry":
{
"type": "Polygon",
"coordinates" : [ [ [2,49],[2,49.1],[2.1,49.1],[2.1,49],[2,49] ] ]
}
}
]
}""")
gdal.FileFromMemBuffer('/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets',
"""{
"analytic" : {
"_permissions": ["download"],
"_links": {
"_self": "analytic_links_self",
"activate": "/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/activate",
},
"location": "/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/my.tiff",
"status": "active",
"expires_at": "2016-02-11T12:34:56.789"
}
}""")
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
ds = gdal.OpenEx('PLScenes:', gdal.OF_VECTOR, open_options=['VERSION=data_v1', 'API_KEY=foo', 'FOLLOW_LINKS=YES'])
gdal.SetConfigOption('PL_URL', None)
lyr = ds.GetLayer(0)
f = lyr.GetNextFeature()
if f.GetFID() != 1 or f['id'] != 'id' or f['self_link'] != 'self' or \
f['assets_link'] != '/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets' or \
f['acquired'] != '2016/02/11 12:34:56.789+00' or \
f['anomalous_pixels'] != 1.23 or f['item_type'] != 'foo' or f['columns'] != 1 or \
not f['ground_control'] or \
f['asset_analytic_self_link'] != 'analytic_links_self' or \
f['asset_analytic_activate_link'] != '/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/activate' or \
f['asset_analytic_permissions'] != ['download'] or \
f['asset_analytic_expires_at'] != '2016/02/11 12:34:56.789' or \
f['asset_analytic_location'] != '/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/my.tiff' or \
f['asset_analytic_status'] != 'active' or \
f.GetGeometryRef().ExportToWkt() != 'MULTIPOLYGON (((2 49,2.0 49.1,2.1 49.1,2.1 49.0,2 49)))':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
lyr.ResetReading()
f = lyr.GetNextFeature()
if f.GetFID() != 1:
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
gdal.FileFromMemBuffer('/vsimem/data_v1/quick-search?page=2',
"""{
"features" : [
{
"id": "id2"
}
]
}""")
f = lyr.GetNextFeature()
if f.GetFID() != 2 or f['id'] != 'id2':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
lyr.ResetReading()
f = lyr.GetNextFeature()
if f.GetFID() != 1:
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
f = lyr.GetNextFeature()
if f.GetFID() != 2:
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
f = lyr.GetNextFeature()
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
f = lyr.GetNextFeature()
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
gdal.FileFromMemBuffer("""/vsimem/data_v1/quick-search?_page_size=250&POSTFIELDS={"item_types":["PSOrthoTile"],"filter":{"type":"AndFilter","config":[{"type":"GeometryFilter","field_name":"geometry","config":{"type":"Point","coordinates":[2.0,49.0]}}]}}""",
"""{"features" : [ { "id": "id3", "geometry": { "type": "Point", "coordinates": [2,49]} } ] }""")
# POINT spatial filter
lyr.SetSpatialFilterRect(2, 49, 2, 49)
f = lyr.GetNextFeature()
if f['id'] != 'id3':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
# Cannot find /vsimem/data_v1/stats&POSTFIELDS={"interval":"year","item_types":["PSOrthoTile"],"filter":{"type":"AndFilter","config":[{"type":"GeometryFilter","field_name":"geometry","config":{"type":"Point","coordinates":[2.0,49.0]}}]}}
with gdaltest.error_handler():
if lyr.GetFeatureCount() != 1:
gdaltest.post_reason('fail')
return 'fail'
# Reset spatial filter
lyr.SetSpatialFilter(0, None)
f = lyr.GetNextFeature()
if f['id'] != 'id':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
# Test attribute filter on id
lyr.SetAttributeFilter("id = 'filtered_id'")
gdal.FileFromMemBuffer("""/vsimem/data_v1/quick-search?_page_size=250&POSTFIELDS={"item_types":["PSOrthoTile"],"filter":{"type":"AndFilter","config":[{"type":"StringInFilter","field_name":"id","config":["filtered_id"]}]}}""",
"""{
"id": "filtered_id",
"properties": {}
}""")
f = lyr.GetNextFeature()
if f['id'] != 'filtered_id':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
# Test attribute filter fully evaluated on server side.
lyr.SetAttributeFilter("id != 'a' AND acquired >= '2016/02/11' AND acquired <= '2016/02/12' AND acquired > '1970/01/01 01:23:45' AND acquired < '2100/01/01 01:23:45' AND anomalous_pixels = 1.234567 AND (NOT id = 'b') AND columns > 0 AND columns < 2 AND columns = 1 AND columns IN (1, 2) AND (id IN ('filtered_2') OR id = 'foo') AND permissions = 'download' AND permissions IN ('download')")
content = """{
"features" : [
{
"id": "filtered_2",
"_links" : {
"_self" : "self",
"assets" : "/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets"
},
"_permissions" : [ "download" ],
"properties": {
"acquired": "2016/02/11 12:34:56.789+00",
"anomalous_pixels": 1.23,
"columns": 1,
"item_type": "foo"
},
"geometry":
{
"type": "Polygon",
"coordinates" : [ [ [2,49],[2,49.1],[2.1,49.1],[2.1,49],[2,49] ] ]
}
}
]
}"""
gdal.FileFromMemBuffer("""/vsimem/data_v1/quick-search?_page_size=250&POSTFIELDS={"item_types":["PSOrthoTile"],"filter":{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"NotFilter","config":{"type":"StringInFilter","field_name":"id","config":["a"]}},{"type":"DateRangeFilter","field_name":"acquired","config":{"gte":"2016-02-11T00:00:00Z"}}]},{"type":"DateRangeFilter","field_name":"acquired","config":{"lte":"2016-02-12T00:00:00Z"}}]},{"type":"DateRangeFilter","field_name":"acquired","config":{"gt":"1970-01-01T01:23:45Z"}}]},{"type":"DateRangeFilter","field_name":"acquired","config":{"lt":"2100-01-01T01:23:45Z"}}]},{"type":"RangeFilter","field_name":"anomalous_pixels","config":{"gte":1.234567,"lte":1.234567}}]},{"type":"NotFilter","config":{"type":"StringInFilter","field_name":"id","config":["b"]}}]},{"type":"RangeFilter","field_name":"columns","config":{"gt":0}}]},{"type":"RangeFilter","field_name":"columns","config":{"lt":2}}]},{"type":"NumberInFilter","field_name":"columns","config":[1]}]},{"type":"NumberInFilter","field_name":"columns","config":[1,2]}]},{"type":"OrFilter","config":[{"type":"StringInFilter","field_name":"id","config":["filtered_2"]},{"type":"StringInFilter","field_name":"id","config":["foo"]}]}]},{"type":"PermissionFilter","config":["download"]}]},{"type":"PermissionFilter","config":["download"]}]}]}}""", content)
gdal.FileFromMemBuffer("""/vsimem/data_v1/quick-search?_page_size=250&POSTFIELDS={"item_types":["PSOrthoTile"],"filter":{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"AndFilter","config":[{"type":"NotFilter","config":{"type":"StringInFilter","field_name":"id","config":["a"]}},{"type":"DateRangeFilter","field_name":"acquired","config":{"gte":"2016-02-11T00:00:00Z"}}]},{"type":"DateRangeFilter","field_name":"acquired","config":{"lte":"2016-02-12T00:00:00Z"}}]},{"type":"DateRangeFilter","field_name":"acquired","config":{"gt":"1970-01-01T01:23:45Z"}}]},{"type":"DateRangeFilter","field_name":"acquired","config":{"lt":"2100-01-01T01:23:45Z"}}]},{"type":"RangeFilter","field_name":"anomalous_pixels","config":{"gte":1.23456699,"lte":1.2345670099999999}}]},{"type":"NotFilter","config":{"type":"StringInFilter","field_name":"id","config":["b"]}}]},{"type":"RangeFilter","field_name":"columns","config":{"gt":0}}]},{"type":"RangeFilter","field_name":"columns","config":{"lt":2}}]},{"type":"NumberInFilter","field_name":"columns","config":[1]}]},{"type":"NumberInFilter","field_name":"columns","config":[1,2]}]},{"type":"OrFilter","config":[{"type":"StringInFilter","field_name":"id","config":["filtered_2"]},{"type":"StringInFilter","field_name":"id","config":["foo"]}]}]},{"type":"PermissionFilter","config":["download"]}]},{"type":"PermissionFilter","config":["download"]}]}]}}""", content)
f = lyr.GetNextFeature()
if f['id'] != 'filtered_2':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
# Partly server / partly client
lyr.SetAttributeFilter("id = 'filtered_3' AND id > 'a'")
gdal.FileFromMemBuffer("""/vsimem/data_v1/quick-search?_page_size=250&POSTFIELDS={"item_types":["PSOrthoTile"],"filter":{"type":"AndFilter","config":[{"type":"StringInFilter","field_name":"id","config":["filtered_3"]}]}}""",
"""{
"features" : [
{
"id": "filtered_3",
"properties": {
}
}
]
}""")
f = lyr.GetNextFeature()
if f['id'] != 'filtered_3':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
lyr.SetAttributeFilter("id > 'a' AND id = 'filtered_3'")
f = lyr.GetNextFeature()
if f['id'] != 'filtered_3':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
# Completely client side
lyr.SetAttributeFilter("id > 'a' OR id = 'id'")
f = lyr.GetNextFeature()
if f['id'] != 'id':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
# Completely client side
lyr.SetAttributeFilter("NOT id > 'z'")
f = lyr.GetNextFeature()
if f['id'] != 'id':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
# Reset attribute filter
lyr.SetAttributeFilter(None)
f = lyr.GetNextFeature()
if f['id'] != 'id':
gdaltest.post_reason('fail')
f.DumpReadable()
return 'fail'
# Try raster access
# Missing catalog
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
with gdaltest.error_handler():
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'SCENE=id'])
gdal.SetConfigOption('PL_URL', None)
if ds_raster is not None or gdal.GetLastErrorMsg().find('Missing catalog') < 0:
gdaltest.post_reason('fail')
return 'fail'
# Invalid catalog
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
with gdaltest.error_handler():
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=invalid', 'SCENE=id'])
gdal.SetConfigOption('PL_URL', None)
# visual not an object
gdal.FileFromMemBuffer('/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets',
"""{ "visual": false }""")
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
with gdaltest.error_handler():
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile', 'SCENE=id'])
gdal.SetConfigOption('PL_URL', None)
if ds_raster is not None:
gdaltest.post_reason('fail')
return 'fail'
# Inactive file, and activation link not working
gdal.FileFromMemBuffer('/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets',
"""{
"analytic" : {
"_links": {
"_self": "analytic_links_self",
"activate": "/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/activate",
},
"_permissions": ["download"],
"status": "inactive",
}
}""")
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
with gdaltest.error_handler():
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile', 'SCENE=id', 'ACTIVATION_TIMEOUT=1', 'ASSET=analytic'])
gdal.SetConfigOption('PL_URL', None)
if ds_raster is not None:
gdaltest.post_reason('fail')
return 'fail'
# File in activation
gdal.FileFromMemBuffer('/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets',
"""{
"analytic" : {
"_links": {
"_self": "analytic_links_self",
"activate": "/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/activate",
},
"_permissions": ["download"],
"status": "activating",
}
}""")
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
with gdaltest.error_handler():
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile', 'SCENE=id', 'ACTIVATION_TIMEOUT=1', 'ASSET=analytic'])
gdal.SetConfigOption('PL_URL', None)
if ds_raster is not None:
gdaltest.post_reason('fail')
return 'fail'
gdal.FileFromMemBuffer('/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets',
"""{
"analytic" : {
"_permissions": ["download"],
"_links": {
"_self": "analytic_links_self",
"activate": "/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/activate",
},
"location": "/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/my.tiff",
"status": "active",
"expires_at": "2016-02-11T12:34:56.789"
}
}""")
# Missing /vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/my.tiff
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
with gdaltest.error_handler():
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile', 'SCENE=id', 'ACTIVATION_TIMEOUT=1', 'ASSET=analytic'])
gdal.SetConfigOption('PL_URL', None)
if ds_raster is not None:
gdaltest.post_reason('fail')
return 'fail'
# JSon content for /vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/my.tiff
gdal.FileFromMemBuffer('/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/my.tiff',
"""{}""")
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
with gdaltest.error_handler():
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile', 'SCENE=id', 'ACTIVATION_TIMEOUT=1', 'ASSET=analytic'])
gdal.SetConfigOption('PL_URL', None)
if ds_raster is not None:
gdaltest.post_reason('fail')
return 'fail'
# Missing metadata
gdal.FileFromMemBuffer('/vsimem/data_v1/item-types/PSOrthoTile/items/id/assets/analytic/my.tiff',
open('../gcore/data/byte.tif', 'rb').read())
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
with gdaltest.error_handler():
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile', 'SCENE=id', 'ACTIVATION_TIMEOUT=1', 'ASSET=analytic'])
gdal.SetConfigOption('PL_URL', None)
if ds_raster is None:
gdaltest.post_reason('fail')
return 'fail'
ds_raster = None
# Failed filter by scene id
gdal.FileFromMemBuffer('/vsimem/data_v1/item-types/PSOrthoTile',
"""{"id": "PSOrthoTile"}""")
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile', 'SCENE=id', 'ACTIVATION_TIMEOUT=1', 'ASSET=analytic'])
gdal.SetConfigOption('PL_URL', None)
if ds_raster is None:
gdaltest.post_reason('fail')
return 'fail'
ds_raster = None
# Test metadata items attached to dataset
gdal.FileFromMemBuffer("""/vsimem/data_v1/quick-search?_page_size=250&POSTFIELDS={"item_types":["PSOrthoTile"],"filter":{"type":"AndFilter","config":[{"type":"StringInFilter","field_name":"id","config":["id"]}]}}""",
"""{
"id": "id",
"properties": {
"anomalous_pixels": 1.23
},
}""")
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile', 'SCENE=id', 'ACTIVATION_TIMEOUT=1', 'ASSET=analytic'])
gdal.SetConfigOption('PL_URL', None)
if ds_raster is None:
gdaltest.post_reason('fail')
return 'fail'
if ds_raster.GetMetadataItem('anomalous_pixels') != '1.23':
gdaltest.post_reason('fail')
return 'fail'
ds_raster = None
# Test invalid ASSET
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
with gdaltest.error_handler():
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile', 'SCENE=id', 'ACTIVATION_TIMEOUT=1', 'ASSET=invalid'])
gdal.SetConfigOption('PL_URL', None)
if ds_raster is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test subdatasets
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
ds_raster = gdal.OpenEx('PLScenes:', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile', 'ASSET=list', 'SCENE=id'])
gdal.SetConfigOption('PL_URL', None)
if len(ds_raster.GetSubDatasets()) != 1:
gdaltest.post_reason('fail')
return 'fail'
ds_raster = None
# Unsupported option
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
with gdaltest.error_handler():
ds_raster = gdal.OpenEx('PLScenes:unsupported=yes', gdal.OF_RASTER, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile', 'SCENE=id'])
gdal.SetConfigOption('PL_URL', None)
if ds_raster is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test catalog with vector access
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
ds2 = gdal.OpenEx('PLScenes:', gdal.OF_VECTOR, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=PSOrthoTile'])
gdal.SetConfigOption('PL_URL', None)
if ds2 is None or ds2.GetLayerCount() != 1:
gdaltest.post_reason('fail')
return 'fail'
gdal.SetConfigOption('PL_URL', '/vsimem/data_v1/')
with gdaltest.error_handler():
ds2 = gdal.OpenEx('PLScenes:', gdal.OF_VECTOR, open_options=['VERSION=data_v1', 'API_KEY=foo', 'ITEMTYPES=invalid'])
gdal.SetConfigOption('PL_URL', None)
if ds2 is not None:
gdaltest.post_reason('fail')
return 'fail'
fl = gdal.ReadDir('/vsimem/data_v1')
for filename in fl:
gdal.Unlink(filename)
return 'success'
def vsizip_1():
# We can keep the handle open during all the ZIP writing
hZIP = gdal.VSIFOpenL("/vsizip/vsimem/test.zip", "wb")
if hZIP is None:
gdaltest.post_reason('fail 1')
return 'fail'
# One way to create a directory
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir2/", "wb")
if f is None:
gdaltest.post_reason('fail 2')
return 'fail'
gdal.VSIFCloseL(f)
# A more natural one
gdal.Mkdir("/vsizip/vsimem/test.zip/subdir1", 0)
# Create 1st file
f2 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "wb")
if f2 is None:
gdaltest.post_reason('fail 3')
return 'fail'
gdal.VSIFWriteL("abcd", 1, 4, f2)
gdal.VSIFCloseL(f2)
# Test that we cannot read a zip file being written
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb")
gdal.PopErrorHandler()
if gdal.GetLastErrorMsg() != 'Cannot read a zip file being written':
gdaltest.post_reason('expected error')
print(gdal.GetLastErrorMsg())
return 'fail'
if f is not None:
gdaltest.post_reason('should not have been successful 1')
return 'fail'
# Create 2nd file
f3 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/efghi", "wb")
if f3 is None:
gdaltest.post_reason('fail 4')
return 'fail'
gdal.VSIFWriteL("efghi", 1, 5, f3)
# Try creating a 3d file
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
f4 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/that_wont_work", "wb")
gdal.PopErrorHandler()
if gdal.GetLastErrorMsg(
) != 'Cannot create that_wont_work while another file is being written in the .zip':
gdaltest.post_reason('expected error')
print(gdal.GetLastErrorMsg())
return 'fail'
if f4 is not None:
gdaltest.post_reason('should not have been successful 2')
return 'fail'
gdal.VSIFCloseL(f3)
# Now we can close the main handle
gdal.VSIFCloseL(hZIP)
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb")
if f is None:
gdaltest.post_reason('fail 5')
return 'fail'
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
if data.decode('ASCII') != 'abcd':
gdaltest.post_reason('fail')
print(data)
return 'fail'
# Test alternate uri syntax
gdal.Rename("/vsimem/test.zip", "/vsimem/test.xxx")
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd", "rb")
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
if data.decode('ASCII') != 'abcd':
gdaltest.post_reason('fail')
print(data)
return 'fail'
# With a trailing slash
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd/", "rb")
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
# Test ReadDir()
if len(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}")) != 3:
gdaltest.post_reason('fail')
print(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}"))
return 'fail'
# Unbalanced curls
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Non existing mainfile
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/bla", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Non existing subfile
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.zzz}/bla", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Wrong syntax
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}.aux.xml", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test nested { { } }
hZIP = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}", "wb")
if hZIP is None:
gdaltest.post_reason('fail 1')
return 'fail'
f = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}/test.xxx", "wb")
f_src = gdal.VSIFOpenL("/vsimem/test.xxx", "rb")
data = gdal.VSIFReadL(1, 10000, f_src)
gdal.VSIFCloseL(f_src)
gdal.VSIFWriteL(data, 1, len(data), f)
gdal.VSIFCloseL(f)
gdal.VSIFCloseL(hZIP)
f = gdal.VSIFOpenL(
"/vsizip/{/vsizip/{/vsimem/zipinzip.yyy}/test.xxx}/subdir3/abcd/",
"rb")
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
if data.decode('ASCII') != 'abcd':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.Unlink("/vsimem/test.xxx")
gdal.Unlink("/vsimem/zipinzip.yyy")
return 'success'
def test_cog_byte_to_web_mercator():
tab = [0]
def my_cbk(pct, _, arg):
assert pct >= tab[0]
tab[0] = pct
return 1
directory = '/vsimem/test_cog_byte_to_web_mercator'
gdal.Mkdir(directory, 0o755)
filename = directory + '/cog.tif'
src_ds = gdal.Open('data/byte.tif')
ds = gdal.GetDriverByName('COG').CreateCopy(
filename,
src_ds,
options=['TILING_SCHEME=GoogleMapsCompatible', 'ALIGNED_LEVELS=3'],
callback=my_cbk,
callback_data=tab)
assert tab[0] == 1.0
assert ds
assert len(
gdal.ReadDir(directory)) == 1 # check that the temp file has gone away
ds = None
ds = gdal.Open(filename)
assert ds.RasterCount == 2
assert ds.RasterXSize == 1024
assert ds.RasterYSize == 1024
assert ds.GetRasterBand(
1).GetMaskFlags() == gdal.GMF_ALPHA + gdal.GMF_PER_DATASET
assert ds.GetRasterBand(1).GetBlockSize() == [256, 256]
gt = ds.GetGeoTransform()
assert gt[1] == -gt[5] # yes, checking for strict equality
expected_gt = [
-13149614.849955443, 76.43702828517598, 0.0, 4070118.8821290657, 0.0,
-76.43702828517598
]
for i in range(6):
if gt[i] != pytest.approx(expected_gt[i],
abs=1e-10 * abs(expected_gt[i])):
assert False, gt
assert ds.GetRasterBand(1).Checksum() in (
4363,
4264, # got on Mac at some point
4362, # libjpeg 9d
4569, # libjpeg 9e
)
assert ds.GetRasterBand(1).GetMaskBand().Checksum() == 4356
assert ds.GetRasterBand(1).GetOverviewCount() == 2
ds = None
_check_cog(filename)
# Use our generated COG as the input of the same COG generation: reprojection
# should be skipped
filename2 = directory + '/cog2.tif'
src_ds = gdal.Open(filename)
class my_error_handler(object):
def __init__(self):
self.debug_msg_list = []
self.other_msg_list = []
def handler(self, eErrClass, err_no, msg):
if eErrClass == gdal.CE_Debug:
self.debug_msg_list.append(msg)
else:
self.other_msg_list.append(msg)
handler = my_error_handler()
try:
gdal.PushErrorHandler(handler.handler)
gdal.SetCurrentErrorHandlerCatchDebug(True)
with gdaltest.config_option('CPL_DEBUG', 'COG'):
ds = gdal.GetDriverByName('COG').CreateCopy(
filename2,
src_ds,
options=[
'TILING_SCHEME=GoogleMapsCompatible', 'ALIGNED_LEVELS=3'
])
finally:
gdal.PopErrorHandler()
assert ds
assert 'COG: Skipping reprojection step: source dataset matches reprojection specifications' in handler.debug_msg_list
assert handler.other_msg_list == []
src_ds = None
ds = None
# Cleanup
gdal.GetDriverByName('GTiff').Delete(filename)
gdal.GetDriverByName('GTiff').Delete(filename2)
gdal.Unlink(directory)
def test_vsizip_1():
# We can keep the handle open during all the ZIP writing
hZIP = gdal.VSIFOpenL("/vsizip/vsimem/test.zip", "wb")
assert hZIP is not None, 'fail 1'
# One way to create a directory
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir2/", "wb")
assert f is not None, 'fail 2'
gdal.VSIFCloseL(f)
# A more natural one
gdal.Mkdir("/vsizip/vsimem/test.zip/subdir1", 0)
# Create 1st file
f2 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "wb")
assert f2 is not None, 'fail 3'
gdal.VSIFWriteL("abcd", 1, 4, f2)
gdal.VSIFCloseL(f2)
# Test that we cannot read a zip file being written
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb")
gdal.PopErrorHandler()
assert gdal.GetLastErrorMsg() == 'Cannot read a zip file being written', \
'expected error'
assert f is None, 'should not have been successful 1'
# Create 2nd file
f3 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/efghi", "wb")
assert f3 is not None, 'fail 4'
gdal.VSIFWriteL("efghi", 1, 5, f3)
# Try creating a 3d file
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
f4 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/that_wont_work", "wb")
gdal.PopErrorHandler()
assert gdal.GetLastErrorMsg() == 'Cannot create that_wont_work while another file is being written in the .zip', \
'expected error'
assert f4 is None, 'should not have been successful 2'
gdal.VSIFCloseL(f3)
# Now we can close the main handle
gdal.VSIFCloseL(hZIP)
# ERROR 6: Support only 1 file in archive file /vsimem/test.zip when no explicit in-archive filename is specified
gdal.ErrorReset()
with gdaltest.error_handler():
f = gdal.VSIFOpenL('/vsizip/vsimem/test.zip', 'rb')
if f is not None:
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() != '', 'expected error'
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb")
assert f is not None, 'fail 5'
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
assert data.decode('ASCII') == 'abcd'
# Test alternate uri syntax
gdal.Rename("/vsimem/test.zip", "/vsimem/test.xxx")
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd", "rb")
assert f is not None
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
assert data.decode('ASCII') == 'abcd'
# With a trailing slash
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd/", "rb")
assert f is not None
gdal.VSIFCloseL(f)
# Test ReadDir()
assert len(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}")) == 3
# Unbalanced curls
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx", "rb")
assert f is None
# Non existing mainfile
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/bla", "rb")
assert f is None
# Non existing subfile
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.zzz}/bla", "rb")
assert f is None
# Wrong syntax
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}.aux.xml", "rb")
assert f is None
# Test nested { { } }
hZIP = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}", "wb")
assert hZIP is not None, 'fail 1'
f = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}/test.xxx", "wb")
f_src = gdal.VSIFOpenL("/vsimem/test.xxx", "rb")
data = gdal.VSIFReadL(1, 10000, f_src)
gdal.VSIFCloseL(f_src)
gdal.VSIFWriteL(data, 1, len(data), f)
gdal.VSIFCloseL(f)
gdal.VSIFCloseL(hZIP)
f = gdal.VSIFOpenL("/vsizip/{/vsizip/{/vsimem/zipinzip.yyy}/test.xxx}/subdir3/abcd/", "rb")
assert f is not None
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
assert data.decode('ASCII') == 'abcd'
gdal.Unlink("/vsimem/test.xxx")
gdal.Unlink("/vsimem/zipinzip.yyy")
# Test VSIStatL on a non existing file
assert gdal.VSIStatL('/vsizip//vsimem/foo.zip') is None
# Test ReadDir on a non existing file
assert gdal.ReadDir('/vsizip//vsimem/foo.zip') is None
def vsigs_readdir():
if gdaltest.webserver_port == 0:
return 'skip'
handler = webserver.SequentialHandler()
handler.add(
'GET', '/gs_fake_bucket2/?delimiter=%2F&prefix=a_dir%2F', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Contents>
<Key>a_dir/resource3.bin</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>123456</Size>
</Contents>
</ListBucketResult>
""")
handler.add(
'GET', '/gs_fake_bucket2/?delimiter=%2F&marker=bla&prefix=a_dir%2F',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<Contents>
<Key>a_dir/resource4.bin</Key>
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
</Contents>
<CommonPrefixes>
<Prefix>a_dir/subdir/</Prefix>
</CommonPrefixes>
</ListBucketResult>
""")
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL(
'/vsigs/gs_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir/resource3.bin')
if dir_contents is not None:
gdaltest.post_reason('fail')
return 'fail'
# List buckets
handler = webserver.SequentialHandler()
handler.add(
'GET', '/', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult>
<Buckets>
<Bucket>
<Name>mybucket</Name>
</Bucket>
</Buckets>
</ListAllMyBucketsResult>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsigs/')
if dir_contents != ['mybucket']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
return 'success'
def vsiswift_fake_readdir():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=1', 200,
{ 'Content-type': 'application/json' },
"""[
{
"last_modified": "1970-01-01T00:00:01",
"bytes": 123456,
"name": "bar.baz"
}
]""")
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=1&marker=bar.baz', 200,
{ 'Content-type': 'application/json' },
"""[
{
"subdir": "mysubdir/"
}
]""")
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=1&marker=mysubdir%2F', 200,
{ 'Content-type': 'application/json' },
"""[
]""")
with gdaltest.config_option('SWIFT_MAX_KEYS', '1'):
with webserver.install_http_handler(handler):
f = open_for_read('/vsiswift/foo/bar.baz')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsiswift/foo')
if dir_contents != ['bar.baz', 'mysubdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
stat_res = gdal.VSIStatL('/vsiswift/foo/bar.baz')
if stat_res.size != 123456:
gdaltest.post_reason('fail')
print(stat_res.size)
return 'fail'
if stat_res.mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsiswift/foo/bar.baz')
if dir_contents is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000&prefix=error_test%2F', 500)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/foo/error_test/')
if dir_contents is not None:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# List containers (empty result)
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something', 200, { 'Content-type': 'application/json' },
"""[]
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/')
if dir_contents != [ '.' ]:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# List containers
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something', 200, { 'Content-type': 'application/json' },
"""[ { "name": "mycontainer1", "count": 0, "bytes": 0 },
{ "name": "mycontainer2", "count": 0, "bytes": 0}
] """)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/')
if dir_contents != [ 'mycontainer1', 'mycontainer2' ]:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
# ReadDir() with a file and directory of same names
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something', 200, { 'Content-type': 'application/json' },
"""[ {
"last_modified": "1970-01-01T00:00:01",
"bytes": 123456,
"name": "foo"
},
{ "subdir": "foo/"} ] """)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/')
if dir_contents != [ 'foo', 'foo/' ]:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
handler = webserver.SequentialHandler()
handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200,
{ 'Content-type': 'application/json' }, "[]")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiswift/foo/')
if dir_contents != [ '.' ]:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
return 'success'
def vsizip_2():
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/foo.bar", "wb")
if fmain is None:
gdaltest.post_reason('fail 1')
return 'fail'
gdal.VSIFWriteL("12345", 1, 5, fmain)
gdal.VSIFCloseL(fmain)
content = gdal.ReadDir("/vsizip/vsimem/test2.zip")
if content != ['foo.bar']:
gdaltest.post_reason('bad content 1')
print(content)
return 'fail'
# Now append a second file
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/bar.baz", "wb")
if fmain is None:
gdaltest.post_reason('fail 2')
return 'fail'
gdal.VSIFWriteL("67890", 1, 5, fmain)
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
content = gdal.ReadDir("/vsizip/vsimem/test2.zip")
gdal.PopErrorHandler()
if gdal.GetLastErrorMsg() != 'Cannot read a zip file being written':
gdaltest.post_reason('expected error')
print(gdal.GetLastErrorMsg())
return 'fail'
if content != None:
gdaltest.post_reason('bad content 2')
print(content)
return 'fail'
gdal.VSIFCloseL(fmain)
content = gdal.ReadDir("/vsizip/vsimem/test2.zip")
if content != ['foo.bar', 'bar.baz']:
gdaltest.post_reason('bad content 3')
print(content)
return 'fail'
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/foo.bar", "rb")
if fmain is None:
gdaltest.post_reason('fail 3')
return 'fail'
data = gdal.VSIFReadL(1, 5, fmain)
gdal.VSIFCloseL(fmain)
if data.decode('ASCII') != '12345':
print(data)
return 'fail'
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/bar.baz", "rb")
if fmain is None:
gdaltest.post_reason('fail 4')
return 'fail'
data = gdal.VSIFReadL(1, 5, fmain)
gdal.VSIFCloseL(fmain)
if data.decode('ASCII') != '67890':
print(data)
return 'fail'
gdal.Unlink("/vsimem/test2.zip")
return 'success'
def test_vsiaz_extra_1():
if not gdaltest.built_against_curl():
pytest.skip()
az_resource = gdal.GetConfigOption('AZ_RESOURCE')
if az_resource is None:
pytest.skip('Missing AZ_RESOURCE')
if '/' not in az_resource:
path = '/vsiaz/' + az_resource
statres = gdal.VSIStatL(path)
assert statres is not None and stat.S_ISDIR(statres.mode), \
('%s is not a valid bucket' % path)
readdir = gdal.ReadDir(path)
assert readdir is not None, 'ReadDir() should not return empty list'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
assert gdal.VSIStatL(subpath) is not None, \
('Stat(%s) should not return an error' % subpath)
unique_id = 'vsiaz_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id in readdir, \
('ReadDir(%s) should contain %s' % (path, unique_id))
ret = gdal.Mkdir(subpath, 0)
assert ret != 0, ('Mkdir(%s) repeated should return an error' %
subpath)
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id not in readdir, \
('ReadDir(%s) should not contain %s' % (path, unique_id))
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
assert f is not None
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
assert ret != 0, \
('Rmdir(%s) on non empty directory should return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
assert f is not None
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
assert data == 'hello'
gdal.VSIFCloseL(f)
md = gdal.GetFileMetadata(subpath + '/test.txt', 'HEADERS')
assert 'x-ms-blob-type' in md
md = gdal.GetFileMetadata(subpath + '/test.txt', 'METADATA')
assert 'ETag' in md
assert 'x-ms-blob-type' not in md
md = gdal.GetFileMetadata(subpath + '/test.txt', 'TAGS')
assert md == {}
# Change properties
assert gdal.SetFileMetadata(subpath + '/test.txt',
{'x-ms-blob-content-type': 'foo'},
'PROPERTIES')
md = gdal.GetFileMetadata(subpath + '/test.txt', 'HEADERS')
assert md['Content-Type'] == 'foo'
# Change metadata
assert gdal.SetFileMetadata(subpath + '/test.txt',
{'x-ms-meta-FOO': 'BAR'}, 'METADATA')
md = gdal.GetFileMetadata(subpath + '/test.txt', 'METADATA')
assert md['x-ms-meta-FOO'] == 'BAR'
# Change tags
assert gdal.SetFileMetadata(subpath + '/test.txt', {'BAR': 'BAZ'},
'TAGS')
md = gdal.GetFileMetadata(subpath + '/test.txt', 'TAGS')
assert md['BAR'] == 'BAZ'
assert gdal.Rename(subpath + '/test.txt', subpath + '/test2.txt') == 0
f = gdal.VSIFOpenL(subpath + '/test2.txt', 'rb')
assert f is not None
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
assert data == 'hello'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test2.txt')
assert ret >= 0, \
('Unlink(%s) should not return an error' % (subpath + '/test2.txt'))
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
return
f = open_for_read('/vsiaz/' + az_resource)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
# Same with /vsiaz_streaming/
f = open_for_read('/vsiaz_streaming/' + az_resource)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
if False: # pylint: disable=using-constant-test
# we actually try to read at read() time and bSetError = false
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsiaz/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert gdal.VSIGetLastErrorMsg() != ''
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsiaz_streaming/' + az_resource +
'/invalid_resource.baz')
assert f is None, gdal.VSIGetLastErrorMsg()
# Test GetSignedURL()
signed_url = gdal.GetSignedURL('/vsiaz/' + az_resource)
f = open_for_read('/vsicurl_streaming/' + signed_url)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
def test_ogr_csw_vsimem_cleanup():
gdal.SetConfigOption('CPL_CURL_ENABLE_VSIMEM', None)
for f in gdal.ReadDir('/vsimem/'):
gdal.Unlink('/vsimem/' + f)
os.mkdir('tmppamproxydir')
gdal.SetConfigOption('GDAL_PAM_PROXY_DIR', 'tmppamproxydir')
# Compute statistics. They should be saved in the .aux.xml in the proxyDB
ds = gdal.Open('tmpdirreadonly/byte.tif')
stats = ds.GetRasterBand(1).ComputeStatistics(False)
gdal.ErrorReset()
ds = None
error_msg = gdal.GetLastErrorMsg()
if error_msg != '':
print('did not expected error message')
sys.exit(1)
# Check that the .aux.xml in the proxyDB exists
filelist = gdal.ReadDir('tmppamproxydir')
if not '000000_tmpdirreadonly_byte.tif.aux.xml' in filelist:
print(
'did not get find 000000_tmpdirreadonly_byte.tif.aux.xml on filesystem'
)
sys.exit(1)
# Test altering a value to check that the file will be used
f = open('tmppamproxydir/000000_tmpdirreadonly_byte.tif.aux.xml', 'w')
f.write("""<PAMDataset>
<PAMRasterBand band="1">
<Metadata>
<MDI key="STATISTICS_MAXIMUM">255</MDI>
<MDI key="STATISTICS_MEAN">126.765</MDI>
<MDI key="STATISTICS_MINIMUM">-9999</MDI>
<MDI key="STATISTICS_STDDEV">22.928470838676</MDI>
def ogr_tiger_4():
if ogrtest.tiger_ds is None:
return 'skip'
# load all the files into memory.
for file in gdal.ReadDir('tmp/cache/TGR01001'):
if file[0] == '.':
continue
data = open('tmp/cache/TGR01001/' + file, 'r').read()
f = gdal.VSIFOpenL('/vsimem/tigertest/' + file, 'wb')
gdal.VSIFWriteL(data, 1, len(data), f)
gdal.VSIFCloseL(f)
# Try reading.
ogrtest.tiger_ds = ogr.Open('/vsimem/tigertest/TGR01001.RT1')
if ogrtest.tiger_ds is None:
gdaltest.post_reason('fail to open.')
return 'fail'
ogrtest.tiger_ds = None
# also test opening with a filename (#4443)
ogrtest.tiger_ds = ogr.Open('tmp/cache/TGR01001/TGR01001.RT1')
if ogrtest.tiger_ds is None:
gdaltest.post_reason('fail')
return 'fail'
# Check a few features.
cc_layer = ogrtest.tiger_ds.GetLayerByName('CompleteChain')
if cc_layer.GetFeatureCount() != 19289:
gdaltest.post_reason('wrong cc feature count')
return 'fail'
feat = cc_layer.GetNextFeature()
feat = cc_layer.GetNextFeature()
feat = cc_layer.GetNextFeature()
if feat.TLID != 2833200 or feat.FRIADDL is not None or feat.BLOCKL != 5000:
gdaltest.post_reason('wrong attribute on cc feature.')
return 'fail'
if ogrtest.check_feature_geometry(
feat,
'LINESTRING (-86.4402 32.504137,-86.440313 32.504009,-86.440434 32.503884,-86.440491 32.503805,-86.44053 32.503757,-86.440578 32.503641,-86.440593 32.503515,-86.440588 32.503252,-86.440596 32.50298)',
max_error=0.000001) != 0:
return 'fail'
feat = ogrtest.tiger_ds.GetLayerByName('TLIDRange').GetNextFeature()
if feat.MODULE != 'TGR01001' or feat.TLMINID != 2822718:
gdaltest.post_reason('got wrong TLIDRange attributes')
return 'fail'
# Try to recover memory from /vsimem.
for file in gdal.ReadDir('tmp/cache/TGR01001'):
if file[0] == '.':
continue
gdal.Unlink('/vsimem/tigertest/' + file)
return 'success'
def vsisync():
with gdaltest.error_handler():
if gdal.Sync('/i_do/not/exist', '/vsimem/'):
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
if gdal.Sync('vsifile.py', '/i_do/not/exist'):
gdaltest.post_reason('fail')
return 'fail'
# Test copying a file
for i in range(2):
if not gdal.Sync('vsifile.py', '/vsimem/'):
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsimem/vsifile.py').size != gdal.VSIStatL(
'vsifile.py').size:
gdaltest.post_reason('fail')
return 'fail'
gdal.Unlink('/vsimem/vsifile.py')
# Test copying the content of a directory
gdal.Mkdir('/vsimem/test_sync', 0)
gdal.FileFromMemBuffer('/vsimem/test_sync/foo.txt', 'bar')
gdal.Mkdir('/vsimem/test_sync/subdir', 0)
gdal.FileFromMemBuffer('/vsimem/test_sync/subdir/bar.txt', 'baz')
if sys.platform != 'win32':
with gdaltest.error_handler():
if gdal.Sync('/vsimem/test_sync/', '/i_do_not/exist'):
gdaltest.post_reason('fail')
return 'fail'
if not gdal.Sync('/vsimem/test_sync/', '/vsimem/out'):
gdaltest.post_reason('fail')
return 'fail'
if gdal.ReadDir('/vsimem/out') != ['foo.txt', 'subdir']:
gdaltest.post_reason('fail')
print(gdal.ReadDir('/vsimem/out'))
return 'fail'
if gdal.ReadDir('/vsimem/out/subdir') != ['bar.txt']:
gdaltest.post_reason('fail')
print(gdal.ReadDir('/vsimem/out/subdir'))
return 'fail'
# Again
if not gdal.Sync('/vsimem/test_sync/', '/vsimem/out'):
gdaltest.post_reason('fail')
return 'fail'
gdal.RmdirRecursive('/vsimem/out')
# Test copying a directory
pct_values = []
def my_progress(pct, message, user_data):
pct_values.append(pct)
if not gdal.Sync('/vsimem/test_sync', '/vsimem/out', callback=my_progress):
gdaltest.post_reason('fail')
return 'fail'
if pct_values != [0.5, 1.0]:
gdaltest.post_reason('fail')
print(pct_values)
return 'fail'
if gdal.ReadDir('/vsimem/out') != ['test_sync']:
gdaltest.post_reason('fail')
print(gdal.ReadDir('/vsimem/out'))
return 'fail'
if gdal.ReadDir('/vsimem/out/test_sync') != ['foo.txt', 'subdir']:
gdaltest.post_reason('fail')
print(gdal.ReadDir('/vsimem/out/test_sync'))
return 'fail'
gdal.RmdirRecursive('/vsimem/test_sync')
gdal.RmdirRecursive('/vsimem/out')
return 'success'
def test_vsiadls_fake_readdir():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/azure/blob/myaccount/az_fake_bucket2?directory=a_dir%20with_space&recursive=false&resource=filesystem', 200,
{'Content-type': 'application/json;charset=utf-8', 'x-ms-continuation': 'contmarker'},
"""
{"paths":[{"name":"a_dir with_space/resource3 with_space.bin","contentLength":"123456","lastModified": "Mon, 01 Jan 1970 00:00:01"}]}
""")
handler.add('GET', '/azure/blob/myaccount/az_fake_bucket2?continuation=contmarker&directory=a_dir%20with_space&recursive=false&resource=filesystem', 200,
{'Content-type': 'application/json;charset=utf-8'},
"""
{"paths":[{"name":"a_dir with_space/resource4.bin","contentLength":"456789","lastModified": "16 Oct 2016 12:34:56"},
{"name":"a_dir with_space/subdir","isDirectory":"true"}]}
"""
)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiadls/az_fake_bucket2/a_dir with_space/resource3 with_space.bin')
if f is None:
pytest.fail()
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsiadls/az_fake_bucket2/a_dir with_space')
assert dir_contents == ['resource3 with_space.bin', 'resource4.bin', 'subdir']
assert gdal.VSIStatL('/vsiadls/az_fake_bucket2/a_dir with_space/resource3 with_space.bin').size == 123456
assert gdal.VSIStatL('/vsiadls/az_fake_bucket2/a_dir with_space/resource3 with_space.bin').mtime == 1
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsiadls/az_fake_bucket2/a_dir with_space/resource3 with_space.bin')
assert dir_contents is None
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add('GET', '/azure/blob/myaccount/az_fake_bucket2?directory=error_test&recursive=false&resource=filesystem', 500)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiadls/az_fake_bucket2/error_test/')
assert dir_contents is None
# List containers (empty result)
handler = webserver.SequentialHandler()
handler.add('GET', '/azure/blob/myaccount/?resource=account', 200, {'Content-type': 'application/json'},
"""{ "filesystems": [] }""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiadls/')
assert dir_contents == ['.']
gdal.VSICurlClearCache()
# List containers
handler = webserver.SequentialHandler()
handler.add('GET', '/azure/blob/myaccount/?resource=account', 200, {'Content-type': 'application/json', 'x-ms-continuation': 'contmarker'},
"""{ "filesystems": [{ "name": "mycontainer1"}] }""")
handler.add('GET', '/azure/blob/myaccount/?continuation=contmarker&resource=account', 200, {'Content-type': 'application/json'},
"""{ "filesystems": [{ "name": "mycontainer2"}] }""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiadls/')
assert dir_contents == ['mycontainer1', 'mycontainer2']
def vsis3_3():
if gdaltest.webserver_port == 0:
return 'skip'
f = open_for_read('/vsis3/s3_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsis3/s3_fake_bucket2/a_dir')
if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']:
gdaltest.post_reason('fail')
print(dir_contents)
return 'fail'
if gdal.VSIStatL(
'/vsis3/s3_fake_bucket2/a_dir/resource3.bin').size != 123456:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsis3/s3_fake_bucket2/a_dir/resource3.bin').mtime != 1:
gdaltest.post_reason('fail')
return 'fail'
# Test CPL_VSIL_CURL_NON_CACHED
for config_option_value in [
'/vsis3/s3_non_cached/test.txt', '/vsis3/s3_non_cached',
'/vsis3/s3_non_cached:/vsis3/unrelated',
'/vsis3/unrelated:/vsis3/s3_non_cached',
'/vsis3/unrelated:/vsis3/s3_non_cached:/vsis3/unrelated'
]:
gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', config_option_value)
# So that the server knows we want it to serve initial content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_1' %
gdaltest.webserver_port)
f = open_for_read('/vsis3/s3_non_cached/test.txt')
if f is None:
gdaltest.post_reason('fail')
print(config_option_value)
return 'fail'
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
# So that the server knows we want it to serve other content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_2' %
gdaltest.webserver_port)
size = gdal.VSIStatL('/vsis3/s3_non_cached/test.txt').size
if size != 4:
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
# So that the server knows we want it to serve initial content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_1' %
gdaltest.webserver_port)
size = gdal.VSIStatL('/vsis3/s3_non_cached/test.txt').size
if size != 3:
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
# So that the server knows we want it to serve other content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_2' %
gdaltest.webserver_port)
f = open_for_read('/vsis3/s3_non_cached/test.txt')
if f is None:
gdaltest.post_reason('fail')
print(config_option_value)
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'bar2':
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', None)
# Retry without option
for config_option_value in [None, '/vsis3/s3_non_cached/bar.txt']:
gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', config_option_value)
# So that the server knows we want it to serve initial content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_1' %
gdaltest.webserver_port)
f = open_for_read('/vsis3/s3_non_cached/test.txt')
if f is None:
gdaltest.post_reason('fail')
print(config_option_value)
return 'fail'
data = gdal.VSIFReadL(1, 3, f).decode('ascii')
gdal.VSIFCloseL(f)
if data != 'foo':
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
# So that the server knows we want it to serve other content
gdaltest.gdalurlopen(
'http://127.0.0.1:%d/s3_non_cached_test_use_content_2' %
gdaltest.webserver_port)
f = open_for_read('/vsis3/s3_non_cached/test.txt')
if f is None:
gdaltest.post_reason('fail')
print(config_option_value)
return 'fail'
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
# We should still get foo because of caching
if data != 'foo':
gdaltest.post_reason('fail')
print(config_option_value)
print(data)
return 'fail'
gdal.SetConfigOption('CPL_VSIL_CURL_NON_CACHED', None)
return 'success'
