def test_vsizip_1():
# We can keep the handle open during all the ZIP writing
hZIP = gdal.VSIFOpenL("/vsizip/vsimem/test.zip", "wb")
assert hZIP is not None, 'fail 1'
# One way to create a directory
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir2/", "wb")
assert f is not None, 'fail 2'
gdal.VSIFCloseL(f)
# A more natural one
gdal.Mkdir("/vsizip/vsimem/test.zip/subdir1", 0)
# Create 1st file
f2 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "wb")
assert f2 is not None, 'fail 3'
gdal.VSIFWriteL("abcd", 1, 4, f2)
gdal.VSIFCloseL(f2)
# Test that we cannot read a zip file being written
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb")
gdal.PopErrorHandler()
assert gdal.GetLastErrorMsg() == 'Cannot read a zip file being written', \
'expected error'
assert f is None, 'should not have been successful 1'
# Create 2nd file
f3 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/efghi", "wb")
assert f3 is not None, 'fail 4'
gdal.VSIFWriteL("efghi", 1, 5, f3)
# Try creating a 3d file
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
f4 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/that_wont_work", "wb")
gdal.PopErrorHandler()
assert gdal.GetLastErrorMsg() == 'Cannot create that_wont_work while another file is being written in the .zip', \
'expected error'
assert f4 is None, 'should not have been successful 2'
gdal.VSIFCloseL(f3)
# Now we can close the main handle
gdal.VSIFCloseL(hZIP)
# ERROR 6: Support only 1 file in archive file /vsimem/test.zip when no explicit in-archive filename is specified
gdal.ErrorReset()
with gdaltest.error_handler():
f = gdal.VSIFOpenL('/vsizip/vsimem/test.zip', 'rb')
if f is not None:
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() != '', 'expected error'
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb")
assert f is not None, 'fail 5'
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
assert data.decode('ASCII') == 'abcd'
# Test alternate uri syntax
gdal.Rename("/vsimem/test.zip", "/vsimem/test.xxx")
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd", "rb")
assert f is not None
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
assert data.decode('ASCII') == 'abcd'
# With a trailing slash
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd/", "rb")
assert f is not None
gdal.VSIFCloseL(f)
# Test ReadDir()
assert len(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}")) == 3
# Unbalanced curls
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx", "rb")
assert f is None
# Non existing mainfile
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/bla", "rb")
assert f is None
# Non existing subfile
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.zzz}/bla", "rb")
assert f is None
# Wrong syntax
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}.aux.xml", "rb")
assert f is None
# Test nested { { } }
hZIP = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}", "wb")
assert hZIP is not None, 'fail 1'
f = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}/test.xxx", "wb")
f_src = gdal.VSIFOpenL("/vsimem/test.xxx", "rb")
data = gdal.VSIFReadL(1, 10000, f_src)
gdal.VSIFCloseL(f_src)
gdal.VSIFWriteL(data, 1, len(data), f)
gdal.VSIFCloseL(f)
gdal.VSIFCloseL(hZIP)
f = gdal.VSIFOpenL(
"/vsizip/{/vsizip/{/vsimem/zipinzip.yyy}/test.xxx}/subdir3/abcd/",
"rb")
assert f is not None
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
assert data.decode('ASCII') == 'abcd'
gdal.Unlink("/vsimem/test.xxx")
gdal.Unlink("/vsimem/zipinzip.yyy")
# Test VSIStatL on a non existing file
assert gdal.VSIStatL('/vsizip//vsimem/foo.zip') is None
# Test ReadDir on a non existing file
assert gdal.ReadDir('/vsizip//vsimem/foo.zip') is None
def test_vsiaz_extra_1():
if not gdaltest.built_against_curl():
pytest.skip()
az_resource = gdal.GetConfigOption('AZ_RESOURCE')
if az_resource is None:
pytest.skip('Missing AZ_RESOURCE')
if '/' not in az_resource:
path = '/vsiaz/' + az_resource
statres = gdal.VSIStatL(path)
assert statres is not None and stat.S_ISDIR(statres.mode), \
('%s is not a valid bucket' % path)
readdir = gdal.ReadDir(path)
assert readdir is not None, 'ReadDir() should not return empty list'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
assert gdal.VSIStatL(subpath) is not None, \
('Stat(%s) should not return an error' % subpath)
unique_id = 'vsiaz_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id in readdir, \
('ReadDir(%s) should contain %s' % (path, unique_id))
ret = gdal.Mkdir(subpath, 0)
assert ret != 0, ('Mkdir(%s) repeated should return an error' % subpath)
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id not in readdir, \
('ReadDir(%s) should not contain %s' % (path, unique_id))
ret = gdal.Rmdir(subpath)
assert ret != 0, ('Rmdir(%s) repeated should return an error' % subpath)
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
assert f is not None
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
assert ret != 0, \
('Rmdir(%s) on non empty directory should return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
assert f is not None
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
assert data == 'hello'
gdal.VSIFCloseL(f)
assert gdal.Rename(subpath + '/test.txt', subpath + '/test2.txt') == 0
f = gdal.VSIFOpenL(subpath + '/test2.txt', 'rb')
assert f is not None
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
assert data == 'hello'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test2.txt')
assert ret >= 0, \
('Unlink(%s) should not return an error' % (subpath + '/test2.txt'))
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
return
f = open_for_read('/vsiaz/' + az_resource)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
# Same with /vsiaz_streaming/
f = open_for_read('/vsiaz_streaming/' + az_resource)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
if False: # pylint: disable=using-constant-test
# we actually try to read at read() time and bSetError = false
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsiaz/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert gdal.VSIGetLastErrorMsg() != ''
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsiaz_streaming/' + az_resource + '/invalid_resource.baz')
assert f is None, gdal.VSIGetLastErrorMsg()
# Test GetSignedURL()
signed_url = gdal.GetSignedURL('/vsiaz/' + az_resource)
f = open_for_read('/vsicurl_streaming/' + signed_url)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
def ecrgtoc_1():
toc_xml = """<Table_of_Contents>
<file_header file_status="new">
<file_name>TOC.xml</file_name>
</file_header>
<product product_title="ProductTitle">
<disc id="DiscId">
<frame_list number_of_frames="2">
<scale size="1:500 K">
<frame name="000000009s0013.lf2">
<frame_path>clfc\\2</frame_path>
<frame_version>001</frame_version>
<frame_chart_type>lf</frame_chart_type>
<frame_zone>2</frame_zone>
</frame>
<frame name="000000009t0013.lf2">
<frame_path>clfc\\2</frame_path>
<frame_version>001</frame_version>
<frame_chart_type>lf</frame_chart_type>
<frame_zone>2</frame_zone>
</frame>
</scale>
</frame_list>
</disc>
</product>
<extension_list>
<extension code="LF">
<chart_code>LF</chart_code>
<chart_type>1:500 K (LFC Day)</chart_type>
<chart_scale>1:500 K</chart_scale>
<chart_description>LFC Day</chart_description>
</extension>
</extension_list>
</Table_of_Contents>"""
f = gdal.VSIFOpenL('/vsimem/TOC.xml', 'wb')
gdal.VSIFWriteL(toc_xml, 1, len(toc_xml), f)
gdal.VSIFCloseL(f)
ds = gdal.Open('/vsimem/TOC.xml')
if ds is None:
return 'fail'
expected_gt = [
-85.43147208121826, 0.00059486040609137061, 0.0, 33.166986564299428,
0.0, -0.00044985604606525913
]
gt = ds.GetGeoTransform()
for i in range(6):
if abs(gt[i] - expected_gt[i]) > 1e-10:
gdaltest.post_reason('did not get expected geotransform')
print(gt)
wkt = ds.GetProjectionRef()
if wkt.find('WGS 84') == -1:
gdaltest.post_reason('did not get expected SRS')
print(wkt)
return 'fail'
filelist = ds.GetFileList()
if len(filelist) != 3:
gdaltest.post_reason('did not get expected filelist')
print(filelist)
return 'fail'
ds2 = gdal.GetDriverByName('NITF').Create('/vsimem/clfc/2/000000009s0013.lf2', 2304, 2304, 3, \
options = ['ICORDS=G', 'TRE=GEOLOB=000605184000800256-85.43147208122+33.16698656430'])
ds2.SetGeoTransform([
-85.43147208122, 0.00059486040609137061, 0.0, 33.16698656430, 0.0,
-0.00044985604606525913
])
ds2.SetProjection(wkt)
ds2.GetRasterBand(1).Fill(255)
ds2 = None
ds2 = gdal.GetDriverByName('NITF').Create('/vsimem/clfc/2/000000009t0013.lf2', 2304, 2304, 3, \
options = ['ICORDS=G', 'TRE=GEOLOB=000605184000800256-84.06091370558+33.16698656430'])
ds2.SetGeoTransform([
-84.06091370558, 0.00059486040609137061, 0.0, 33.16698656430, 0.0,
-0.00044985604606525913
])
ds2.SetProjection(wkt)
ds2 = None
cs = ds.GetRasterBand(1).Checksum()
ds = None
if cs != 5966:
gdaltest.post_reason('bad checksum')
print(cs)
return 'fail'
return 'success'
def test_vsigs_write():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = ''
while True:
numchars = int(request.rfile.readline().strip(), 16)
content += request.rfile.read(numchars).decode('ascii')
request.rfile.read(2)
if numchars == 0:
break
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Simulate failure while transmitting
f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if ret != 0:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Simulate failure at end of transfer
f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = ''
while True:
numchars = int(request.rfile.readline().strip(), 16)
content += request.rfile.read(numchars).decode('ascii')
request.rfile.read(2)
if numchars == 0:
break
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if ret != 35000:
gdal.VSIFCloseL(f)
pytest.fail(ret)
with gdaltest.error_handler():
ret = gdal.VSIFCloseL(f)
assert ret != 0
def vsizip_1():
# We can keep the handle open during all the ZIP writing
hZIP = gdal.VSIFOpenL("/vsizip/vsimem/test.zip", "wb")
if hZIP is None:
gdaltest.post_reason('fail 1')
return 'fail'
# One way to create a directory
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir2/", "wb")
if f is None:
gdaltest.post_reason('fail 2')
return 'fail'
gdal.VSIFCloseL(f)
# A more natural one
gdal.Mkdir("/vsizip/vsimem/test.zip/subdir1", 0)
# Create 1st file
f2 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "wb")
if f2 is None:
gdaltest.post_reason('fail 3')
return 'fail'
gdal.VSIFWriteL("abcd", 1, 4, f2)
gdal.VSIFCloseL(f2)
# Test that we cannot read a zip file being written
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb")
gdal.PopErrorHandler()
if gdal.GetLastErrorMsg() != 'Cannot read a zip file being written':
gdaltest.post_reason('expected error')
print(gdal.GetLastErrorMsg())
return 'fail'
if f is not None:
gdaltest.post_reason('should not have been successful 1')
return 'fail'
# Create 2nd file
f3 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/efghi", "wb")
if f3 is None:
gdaltest.post_reason('fail 4')
return 'fail'
gdal.VSIFWriteL("efghi", 1, 5, f3)
# Try creating a 3d file
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
f4 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/that_wont_work", "wb")
gdal.PopErrorHandler()
if gdal.GetLastErrorMsg(
) != 'Cannot create that_wont_work while another file is being written in the .zip':
gdaltest.post_reason('expected error')
print(gdal.GetLastErrorMsg())
return 'fail'
if f4 is not None:
gdaltest.post_reason('should not have been successful 2')
return 'fail'
gdal.VSIFCloseL(f3)
# Now we can close the main handle
gdal.VSIFCloseL(hZIP)
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb")
if f is None:
gdaltest.post_reason('fail 5')
return 'fail'
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
if data.decode('ASCII') != 'abcd':
gdaltest.post_reason('fail')
print(data)
return 'fail'
# Test alternate uri syntax
gdal.Rename("/vsimem/test.zip", "/vsimem/test.xxx")
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd", "rb")
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
if data.decode('ASCII') != 'abcd':
gdaltest.post_reason('fail')
print(data)
return 'fail'
# With a trailing slash
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd/", "rb")
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
# Test ReadDir()
if len(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}")) != 3:
gdaltest.post_reason('fail')
print(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}"))
return 'fail'
# Unbalanced curls
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Non existing mainfile
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/bla", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Non existing subfile
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.zzz}/bla", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Wrong syntax
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}.aux.xml", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test nested { { } }
hZIP = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}", "wb")
if hZIP is None:
gdaltest.post_reason('fail 1')
return 'fail'
f = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}/test.xxx", "wb")
f_src = gdal.VSIFOpenL("/vsimem/test.xxx", "rb")
data = gdal.VSIFReadL(1, 10000, f_src)
gdal.VSIFCloseL(f_src)
gdal.VSIFWriteL(data, 1, len(data), f)
gdal.VSIFCloseL(f)
gdal.VSIFCloseL(hZIP)
f = gdal.VSIFOpenL(
"/vsizip/{/vsizip/{/vsimem/zipinzip.yyy}/test.xxx}/subdir3/abcd/",
"rb")
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
if data.decode('ASCII') != 'abcd':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.Unlink("/vsimem/test.xxx")
gdal.Unlink("/vsimem/zipinzip.yyy")
return 'success'
def gdal_cp_single(srcfile, targetfile, progress):
if targetfile.endswith('/'):
stat_res = gdal.VSIStatL(targetfile)
else:
stat_res = gdal.VSIStatL(targetfile + '/')
if (stat_res is None and targetfile.endswith('/')) or \
(stat_res is not None and stat.S_ISDIR(stat_res.mode)):
(_, tail) = os.path.split(srcfile)
if targetfile.endswith('/'):
targetfile = targetfile + tail
else:
targetfile = targetfile + '/' + tail
fin = gdal.VSIFOpenL(srcfile, "rb")
if fin is None:
print('Cannot open %s' % srcfile)
return -1
fout = gdal.VSIFOpenL(targetfile, "wb")
if fout is None:
print('Cannot create %s' % targetfile)
gdal.VSIFCloseL(fin)
return -1
version_num = int(gdal.VersionInfo('VERSION_NUM'))
total_size = 0
if version_num < 1900 or progress is not None:
gdal.VSIFSeekL(fin, 0, 2)
total_size = gdal.VSIFTellL(fin)
gdal.VSIFSeekL(fin, 0, 0)
buf_max_size = 4096
copied = 0
ret = 0
# print('Copying %s...' % srcfile)
if progress is not None:
if not progress.Progress(0.0, 'Copying %s' % srcfile):
print('Copy stopped by user')
ret = -2
while ret == 0:
if total_size != 0 and copied + buf_max_size > total_size:
to_read = total_size - copied
else:
to_read = buf_max_size
buf = gdal.VSIFReadL(1, to_read, fin)
if buf is None:
if copied == 0:
print('Cannot read %d bytes in %s' % (to_read, srcfile))
ret = -1
break
buf_size = len(buf)
if gdal.VSIFWriteL(buf, 1, buf_size, fout) != buf_size:
print('Error writing %d bytes' % buf_size)
ret = -1
break
copied += buf_size
if progress is not None and total_size != 0:
if not progress.Progress(copied * 1.0 / total_size,
'Copying %s' % srcfile):
print('Copy stopped by user')
ret = -2
break
if to_read < buf_max_size or buf_size != buf_max_size:
break
gdal.VSIFCloseL(fin)
gdal.VSIFCloseL(fout)
return ret
def vsifile_5():
fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'wb')
ref_data = ''.join(['%08X' % i for i in range(5 * 32768)])
gdal.VSIFWriteL(ref_data, 1, len(ref_data), fp)
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSI_CACHE', 'YES')
for i in range(3):
if i == 0:
gdal.SetConfigOption('VSI_CACHE_SIZE', '0')
elif i == 1:
gdal.SetConfigOption('VSI_CACHE_SIZE', '65536')
else:
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'rb')
gdal.VSIFSeekL(fp, 50000, 0)
if gdal.VSIFTellL(fp) != 50000:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 50000, 1)
if gdal.VSIFTellL(fp) != 100000:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 0, 2)
if gdal.VSIFTellL(fp) != 5 * 32768 * 8:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFReadL(1, 1, fp)
gdal.VSIFSeekL(fp, 0, 0)
data = gdal.VSIFReadL(1, 3 * 32768, fp)
if data.decode('ascii') != ref_data[0:3 * 32768]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 16384, 0)
data = gdal.VSIFReadL(1, 5 * 32768, fp)
if data.decode('ascii') != ref_data[16384:16384 + 5 * 32768]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
data = gdal.VSIFReadL(1, 50 * 32768, fp)
if data[0:1130496].decode('ascii') != ref_data[16384 + 5 * 32768:]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
gdal.Unlink('tmp/vsifile_5.bin')
return 'success'
def vsicrypt_2():
if not gdaltest.has_vsicrypt:
return 'skip'
# Missing key
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid file
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing',
'wb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid file
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing',
'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid file
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing',
'ab')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid access
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing',
'foo')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Key to short
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=a,file=/vsimem/file.bin', 'wb+')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Invalid signature
gdal.FileFromMemBuffer('/vsimem/file.bin', 'foo')
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Generate empty file
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin',
'wb')
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin',
'rb')
if fp is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsimem/file.bin', 'rb')
header = gdal.VSIFReadL(1, 1000, fp)
gdal.VSIFCloseL(fp)
if len(header) != 46:
gdaltest.post_reason('fail')
print(len(header))
return 'fail'
# Test shortening header
for i in range(46):
fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb')
gdal.VSIFWriteL(header, 1, 46 - 1 - i, fp)
gdal.VSIFCloseL(fp)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test corrupting all bytes of header
for i in range(46):
for val in (0, 127, 255):
fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb')
try:
new_byte = chr(val).encode('latin1')
except:
new_byte = chr(val)
header_new = header[0:i] + new_byte + header[i + 1:]
gdal.VSIFWriteL(header_new, 1, 46, fp)
gdal.VSIFCloseL(fp)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file='
'/vsimem/file.bin', 'rb')
if fp is not None:
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSICRYPT_IV', 'TOO_SHORT')
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file='
'/vsimem/file.bin', 'wb')
gdal.SetConfigOption('VSICRYPT_IV', None)
if fp is not None:
gdal.VSIFCloseL(fp)
# Inconsistent initial vector.
header = struct.pack(
'B' * 38,
86,
83,
73,
67,
82,
89,
80,
84, # signature
38,
0, # header size
1, # major
0, # minor
0,
2, # sector size
0, # alg
0, # mode
8, #size of IV (should be 16)
32,
13,
169,
71,
154,
208,
22,
32, #IV
0,
0, # size of free text
0, # size of key check
0,
0,
0,
0,
0,
0,
0,
0, # size of unencrypted file
0,
0 # size of extra content
)
fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb')
gdal.VSIFWriteL(header, 1, len(header), fp)
gdal.VSIFCloseL(fp)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Inconsistent initial vector with key check.
header = struct.pack(
'B' * 39,
86,
83,
73,
67,
82,
89,
80,
84, # signature
39,
0, # header size
1, # major
0, # minor
0,
2, # sector size
0, # alg
0, # mode
8, #size of IV (should be 16)
32,
13,
169,
71,
154,
208,
22,
32, #IV
0,
0, # size of free text
1, # size of key check
0, # key check
0,
0,
0,
0,
0,
0,
0,
0, # size of unencrypted file
0,
0 # size of extra content
)
fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb')
gdal.VSIFWriteL(header, 1, len(header), fp)
gdal.VSIFCloseL(fp)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test reading with wrong key
fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin',
'wb')
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin',
'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content == 'hello':
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=short_key,file=/vsimem/file.bin',
'ab')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test reading with wrong key with add_key_check
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,add_key_check=yes,file=/vsimem/file.bin',
'wb')
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt/key=short_key,file=/vsimem/file.bin',
'ab')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin', 'ab')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test creating with potentially not build-in alg:
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/alg=blowfish,key=DONT_USE_IN_PROD,file=/vsimem/file.bin',
'wb')
if fp is not None:
gdal.VSIFCloseL(fp)
# Invalid sector_size
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,sector_size=1,file=/vsimem/file.bin',
'wb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Sector size (16) should be at least twice larger than the block size (16) in CBC_CTS
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,sector_size=16,mode=CBC_CTS,file=/vsimem/file.bin',
'wb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
gdal.Unlink('/vsimem/file.bin')
return 'success'
def vsicrypt_6():
try:
import ctypes
except:
return 'skip'
import testnonboundtoswig
testnonboundtoswig.testnonboundtoswig_init()
if testnonboundtoswig.gdal_handle is None:
return 'skip'
testnonboundtoswig.gdal_handle.VSISetCryptKey.argtypes = [
ctypes.c_char_p, ctypes.c_int
]
testnonboundtoswig.gdal_handle.VSISetCryptKey.restype = None
# Set a valid key
testnonboundtoswig.gdal_handle.VSISetCryptKey(
'DONT_USE_IN_PROD'.encode('ASCII'), 16)
if not gdaltest.has_vsicrypt:
return 'skip'
fp = gdal.VSIFOpenL('/vsicrypt/add_key_check=yes,file=/vsimem/file.bin',
'wb+')
if fp is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
return 'fail'
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+')
if fp is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
return 'fail'
# Set a too short key
testnonboundtoswig.gdal_handle.VSISetCryptKey('bbc'.encode('ASCII'), 3)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
# Erase key
testnonboundtoswig.gdal_handle.VSISetCryptKey(None, 0)
with gdaltest.error_handler():
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+')
if fp is not None:
gdaltest.post_reason('fail')
return 'fail'
gdal.Unlink('/vsimem/file.bin')
return 'success'
def vsicrypt_3():
if not gdaltest.has_vsicrypt:
return 'skip'
for options in [
'sector_size=16', 'alg=AES', 'alg=DES_EDE2', 'alg=DES_EDE3',
'alg=SKIPJACK', 'alg=invalid', 'mode=CBC', 'mode=CFB', 'mode=OFB',
'mode=CTR', 'mode=CBC_CTS', 'mode=invalid',
'freetext=my_free_text', 'add_key_check=yes'
]:
gdal.Unlink('/vsimem/file.bin')
if options == 'alg=invalid' or options == 'mode=invalid':
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,%s,file=/vsimem/file.bin'
% options, 'wb')
else:
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,%s,file=/vsimem/file.bin'
% options, 'wb')
if fp is None:
gdaltest.post_reason('fail')
print(options)
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,file=/vsimem/file.bin',
'r')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
print(options)
return 'fail'
# Some of those algs might be missing
for options in [
'alg=Blowfish', 'alg=Camellia', 'alg=CAST256', 'alg=MARS',
'alg=IDEA', 'alg=RC5', 'alg=RC6', 'alg=Serpent', 'alg=SHACAL2',
'alg=Twofish', 'alg=XTEA'
]:
gdal.Unlink('/vsimem/file.bin')
with gdaltest.error_handler():
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,%s,file=/vsimem/file.bin' %
options, 'wb')
if fp is not None:
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL(
'/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
print(options)
return 'fail'
# Test key generation
# Do NOT set VSICRYPT_CRYPTO_RANDOM=NO in production. This is just to speed up tests !
gdal.SetConfigOption("VSICRYPT_CRYPTO_RANDOM", "NO")
fp = gdal.VSIFOpenL(
'/vsicrypt/key=GENERATE_IT,add_key_check=yes,file=/vsimem/file.bin',
'wb')
gdal.SetConfigOption("VSICRYPT_CRYPTO_RANDOM", None)
# Get the generated random key
key_b64 = gdal.GetConfigOption('VSICRYPT_KEY_B64')
if key_b64 is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, fp)
gdal.VSIFCloseL(fp)
fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
print(options)
return 'fail'
gdal.SetConfigOption('VSICRYPT_KEY_B64', None)
fp = gdal.VSIFOpenL('/vsicrypt/key_b64=%s,file=/vsimem/file.bin' % key_b64,
'rb')
content = gdal.VSIFReadL(1, 5, fp).decode('latin1')
gdal.VSIFCloseL(fp)
if content != 'hello':
gdaltest.post_reason('fail')
print(options)
return 'fail'
with gdaltest.error_handler():
statRes = gdal.VSIStatL('/vsicrypt//vsimem/file.bin')
if statRes is not None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.Rename('/vsicrypt//vsimem/file.bin',
'/vsicrypt//vsimem/subdir_crypt/file.bin')
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
ret = gdal.Rename('/vsicrypt//vsimem/subdir_crypt/file.bin',
'/vsimem/subdir_crypt/file2.bin')
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
dir_content = gdal.ReadDir('/vsicrypt//vsimem/subdir_crypt')
if dir_content != ['file2.bin']:
gdaltest.post_reason('fail')
print(dir_content)
return 'fail'
gdal.Unlink('/vsimem/subdir_crypt/file2.bin')
return 'success'
def vsicrypt_4():
if not gdaltest.has_vsicrypt:
return 'skip'
test_file = '/vsicrypt/key=DONT_USE_IN_PROD,sector_size=32,file=/vsimem/file_enc.bin'
ref_file = '/vsimem/file.bin'
for seed in range(1000):
gdal.Unlink(test_file)
gdal.Unlink(ref_file)
test_f = gdal.VSIFOpenL(test_file, 'wb+')
ref_f = gdal.VSIFOpenL(ref_file, 'wb+')
import random
random.seed(seed)
for i in range(20):
random_offset = random.randint(0, 1000)
gdal.VSIFSeekL(test_f, random_offset, 0)
gdal.VSIFSeekL(ref_f, random_offset, 0)
random_size = random.randint(1, 80)
random_content = ''.join([
chr(40 + int(10 * random.random())) for i in range(random_size)
])
gdal.VSIFWriteL(random_content, 1, random_size, test_f)
gdal.VSIFWriteL(random_content, 1, random_size, ref_f)
if random.randint(0, 1) == 0:
random_offset = random.randint(0, 1500)
gdal.VSIFSeekL(test_f, random_offset, 0)
gdal.VSIFSeekL(ref_f, random_offset, 0)
random_size = random.randint(1, 80)
test_content = gdal.VSIFReadL(1, random_size, test_f)
ref_content = gdal.VSIFReadL(1, random_size, ref_f)
if test_content != ref_content:
print(seed)
print('Test content (%d):' % len(test_content))
print(test_content)
print('')
print('Ref content (%d):' % len(ref_content))
print(ref_content)
return 'fail'
gdal.VSIFSeekL(test_f, 0, 0)
gdal.VSIFSeekL(ref_f, 0, 0)
test_content = gdal.VSIFReadL(1, 100000, test_f)
ref_content = gdal.VSIFReadL(1, 100000, ref_f)
if test_content != ref_content:
print(seed)
print('Test content (%d):' % len(test_content))
print(test_content)
print('')
print('Ref content (%d):' % len(ref_content))
print(ref_content)
return 'fail'
gdal.Unlink(test_file)
gdal.Unlink(ref_file)
return 'success'
def test_vsiadls_real_instance_tests():
adls_resource = gdal.GetConfigOption('ADLS_RESOURCE')
if adls_resource is None:
pytest.skip('Missing ADLS_RESOURCE')
if '/' not in adls_resource:
path = '/vsiadls/' + adls_resource
try:
statres = gdal.VSIStatL(path)
assert statres is not None and stat.S_ISDIR(statres.mode), \
('%s is not a valid bucket' % path)
readdir = gdal.ReadDir(path)
assert readdir is not None, 'ReadDir() should not return empty list'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
assert gdal.VSIStatL(subpath) is not None, \
('Stat(%s) should not return an error' % subpath)
unique_id = 'vsiadls_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id in readdir, \
('ReadDir(%s) should contain %s' % (path, unique_id))
ret = gdal.Mkdir(subpath, 0)
assert ret != 0, ('Mkdir(%s) repeated should return an error' %
subpath)
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id not in readdir, \
('ReadDir(%s) should not contain %s' % (path, unique_id))
ret = gdal.Rmdir(subpath)
assert ret != 0, ('Rmdir(%s) repeated should return an error' %
subpath)
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
assert f is not None
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
assert ret != 0, \
('Rmdir(%s) on non empty directory should return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
assert f is not None
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
assert data == 'hello'
gdal.VSIFCloseL(f)
assert gdal.VSIStatL(subpath + '/test.txt') is not None
md = gdal.GetFileMetadata(subpath + '/test.txt', 'HEADERS')
assert 'x-ms-properties' in md
md = gdal.GetFileMetadata(subpath + '/test.txt', 'STATUS')
assert 'x-ms-resource-type' in md
assert 'x-ms-properties' not in md
md = gdal.GetFileMetadata(subpath + '/test.txt', 'ACL')
assert 'x-ms-acl' in md
assert 'x-ms-permissions' in md
# Change properties
properties_foo_bar = 'foo=' + base64.b64encode('bar')
assert gdal.SetFileMetadata(
subpath + '/test.txt', {'x-ms-properties': properties_foo_bar},
'PROPERTIES')
md = gdal.GetFileMetadata(subpath + '/test.txt', 'HEADERS')
assert 'x-ms-properties' in md
assert md['x-ms-properties'] == properties_foo_bar
# Change ACL
assert gdal.SetFileMetadata(subpath + '/test.txt',
{'x-ms-permissions': '0777'}, 'ACL')
md = gdal.GetFileMetadata(subpath + '/test.txt', 'ACL')
assert 'x-ms-permissions' in md
assert md['x-ms-permissions'] == 'rwxrwxrwx'
# Change ACL recursively
md = gdal.GetFileMetadata(subpath, 'ACL')
assert 'x-ms-acl' in md
assert gdal.SetFileMetadata(subpath + '/test.txt',
{'x-ms-acl': md['x-ms-acl']}, 'ACL',
['RECURSIVE=YES', 'MODE=set'])
assert gdal.Rename(subpath + '/test.txt',
subpath + '/test2.txt') == 0
assert gdal.VSIStatL(subpath + '/test.txt') is None
assert gdal.VSIStatL(subpath + '/test2.txt') is not None
f = gdal.VSIFOpenL(subpath + '/test2.txt', 'rb')
assert f is not None
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
assert data == 'hello'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test2.txt')
assert ret >= 0, \
('Unlink(%s) should not return an error' % (subpath + '/test2.txt'))
assert gdal.VSIStatL(subpath + '/test2.txt') is None
assert gdal.Unlink(
subpath + '/test2.txt'
) != 0, "Unlink on a deleted file should return an error"
f = gdal.VSIFOpenL(subpath + '/test2.txt', 'wb')
assert f is not None
gdal.VSIFCloseL(f)
assert gdal.VSIStatL(subpath + '/test2.txt') is not None
finally:
assert gdal.RmdirRecursive(subpath) == 0
return
f = open_for_read('/vsiadls/' + adls_resource)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
# Test GetSignedURL()
signed_url = gdal.GetSignedURL('/vsiadls/' + adls_resource)
f = open_for_read('/vsicurl_streaming/' + signed_url)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
def test_visoss_6():
if gdaltest.webserver_port == 0:
pytest.skip()
with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB
with webserver.install_http_handler(webserver.SequentialHandler()):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket4/large_file.bin', 'wb')
assert f is not None
size = 1024 * 1024 + 1
big_buffer = 'a' * size
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
response = '<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>'
request.send_response(200)
request.send_header('Content-type', 'application/xml')
request.send_header('Content-Length', len(response))
request.end_headers()
request.wfile.write(response.encode('ascii'))
handler.add('POST',
'/oss_fake_bucket4/large_file.bin?uploads',
custom_method=method)
def method(request):
if request.headers['Content-Length'] != '1048576':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('ETag', '"first_etag"')
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/oss_fake_bucket4/large_file.bin?partNumber=1&uploadId=my_id',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL(big_buffer, 1, size, f)
assert ret == size
handler = webserver.SequentialHandler()
def method(request):
if request.headers['Content-Length'] != '1':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
return
request.send_response(200)
request.send_header('ETag', '"second_etag"')
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/oss_fake_bucket4/large_file.bin?partNumber=2&uploadId=my_id',
custom_method=method)
def method(request):
if request.headers['Content-Length'] != '186':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
content = request.rfile.read(186).decode('ascii')
if content != """<CompleteMultipartUpload>
<Part>
<PartNumber>1</PartNumber><ETag>"first_etag"</ETag></Part>
<Part>
<PartNumber>2</PartNumber><ETag>"second_etag"</ETag></Part>
</CompleteMultipartUpload>
""":
sys.stderr.write('Did not get expected content: %s\n' % content)
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('POST',
'/oss_fake_bucket4/large_file.bin?uploadId=my_id',
custom_method=method)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == ''
handler = webserver.SequentialHandler()
handler.add('POST',
'/oss_fake_bucket4/large_file_initiate_403_error.bin?uploads',
403)
handler.add(
'POST',
'/oss_fake_bucket4/large_file_initiate_empty_result.bin?uploads', 200)
handler.add(
'POST',
'/oss_fake_bucket4/large_file_initiate_invalid_xml_result.bin?uploads',
200, {}, 'foo')
handler.add(
'POST',
'/oss_fake_bucket4/large_file_initiate_no_uploadId.bin?uploads', 200,
{}, '<foo/>')
with webserver.install_http_handler(handler):
for filename in [
'/vsioss/oss_fake_bucket4/large_file_initiate_403_error.bin',
'/vsioss/oss_fake_bucket4/large_file_initiate_empty_result.bin',
'/vsioss/oss_fake_bucket4/large_file_initiate_invalid_xml_result.bin',
'/vsioss/oss_fake_bucket4/large_file_initiate_no_uploadId.bin'
]:
with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB
f = gdal.VSIFOpenL(filename, 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFWriteL(big_buffer, 1, size, f)
assert ret == 0
gdal.ErrorReset()
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == ''
handler = webserver.SequentialHandler()
handler.add(
'POST',
'/oss_fake_bucket4/large_file_upload_part_403_error.bin?uploads', 200,
{},
'<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>'
)
handler.add(
'PUT',
'/oss_fake_bucket4/large_file_upload_part_403_error.bin?partNumber=1&uploadId=my_id',
403)
handler.add(
'DELETE',
'/oss_fake_bucket4/large_file_upload_part_403_error.bin?uploadId=my_id',
204)
handler.add(
'POST', '/oss_fake_bucket4/large_file_upload_part_no_etag.bin?uploads',
200, {},
'<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>'
)
handler.add(
'PUT',
'/oss_fake_bucket4/large_file_upload_part_no_etag.bin?partNumber=1&uploadId=my_id',
200)
handler.add(
'DELETE',
'/oss_fake_bucket4/large_file_upload_part_no_etag.bin?uploadId=my_id',
204)
with webserver.install_http_handler(handler):
for filename in [
'/vsioss/oss_fake_bucket4/large_file_upload_part_403_error.bin',
'/vsioss/oss_fake_bucket4/large_file_upload_part_no_etag.bin'
]:
with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB
f = gdal.VSIFOpenL(filename, 'wb')
assert f is not None, filename
with gdaltest.error_handler():
ret = gdal.VSIFWriteL(big_buffer, 1, size, f)
assert ret == 0, filename
gdal.ErrorReset()
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == '', filename
# Simulate failure in AbortMultipart stage
handler = webserver.SequentialHandler()
handler.add(
'POST',
'/oss_fake_bucket4/large_file_abortmultipart_403_error.bin?uploads',
200, {},
'<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>'
)
handler.add(
'PUT',
'/oss_fake_bucket4/large_file_abortmultipart_403_error.bin?partNumber=1&uploadId=my_id',
403)
handler.add(
'DELETE',
'/oss_fake_bucket4/large_file_abortmultipart_403_error.bin?uploadId=my_id',
403)
filename = '/vsioss/oss_fake_bucket4/large_file_abortmultipart_403_error.bin'
with webserver.install_http_handler(handler):
with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB
f = gdal.VSIFOpenL(filename, 'wb')
assert f is not None, filename
with gdaltest.error_handler():
ret = gdal.VSIFWriteL(big_buffer, 1, size, f)
assert ret == 0, filename
gdal.ErrorReset()
with gdaltest.error_handler():
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() != '', filename
# Simulate failure in CompleteMultipartUpload stage
handler = webserver.SequentialHandler()
handler.add(
'POST',
'/oss_fake_bucket4/large_file_completemultipart_403_error.bin?uploads',
200, {},
'<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>'
)
handler.add(
'PUT',
'/oss_fake_bucket4/large_file_completemultipart_403_error.bin?partNumber=1&uploadId=my_id',
200, {'ETag': 'first_etag'}, '')
handler.add(
'PUT',
'/oss_fake_bucket4/large_file_completemultipart_403_error.bin?partNumber=2&uploadId=my_id',
200, {'ETag': 'second_etag'}, '')
handler.add(
'POST',
'/oss_fake_bucket4/large_file_completemultipart_403_error.bin?uploadId=my_id',
403)
# handler.add('DELETE', '/oss_fake_bucket4/large_file_completemultipart_403_error.bin?uploadId=my_id', 204)
filename = '/vsioss/oss_fake_bucket4/large_file_completemultipart_403_error.bin'
with webserver.install_http_handler(handler):
with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB
f = gdal.VSIFOpenL(filename, 'wb')
assert f is not None, filename
ret = gdal.VSIFWriteL(big_buffer, 1, size, f)
assert ret == size, filename
gdal.ErrorReset()
with gdaltest.error_handler():
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() != '', filename
def test_visoss_4():
if gdaltest.webserver_port == 0:
pytest.skip()
with webserver.install_http_handler(webserver.SequentialHandler()):
with gdaltest.error_handler():
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3', 'wb')
assert f is None
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_fake_bucket3/empty_file.bin', 200,
{'Connection': 'close'}, 'foo')
with webserver.install_http_handler(handler):
assert gdal.VSIStatL(
'/vsioss/oss_fake_bucket3/empty_file.bin').size == 3
# Empty file
handler = webserver.SequentialHandler()
def method(request):
if request.headers['Content-Length'] != '0':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/oss_fake_bucket3/empty_file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb')
assert f is not None
gdal.ErrorReset()
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == ''
handler = webserver.SequentialHandler()
handler.add('GET', '/oss_fake_bucket3/empty_file.bin', 200,
{'Connection': 'close'}, '')
with webserver.install_http_handler(handler):
assert gdal.VSIStatL(
'/vsioss/oss_fake_bucket3/empty_file.bin').size == 0
# Invalid seek
handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
assert ret != 0
gdal.VSIFCloseL(f)
# Invalid read
handler = webserver.SequentialHandler()
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
assert not ret
gdal.VSIFCloseL(f)
# Error case
handler = webserver.SequentialHandler()
handler.add('PUT', '/oss_fake_bucket3/empty_file_error.bin', 403)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file_error.bin',
'wb')
assert f is not None
gdal.ErrorReset()
with gdaltest.error_handler():
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() != ''
# Nominal case
with webserver.install_http_handler(webserver.SequentialHandler()):
f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/another_file.bin', 'wb')
assert f is not None
assert gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) == 0
assert gdal.VSIFSeekL(f, 0, 1) == 0
assert gdal.VSIFSeekL(f, 0, 2) == 0
assert gdal.VSIFWriteL('foo', 1, 3, f) == 3
assert gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) == 0
assert gdal.VSIFWriteL('bar', 1, 3, f) == 3
handler = webserver.SequentialHandler()
def method(request):
if request.headers['Content-Length'] != '6':
sys.stderr.write('Did not get expected headers: %s\n' %
str(request.headers))
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(6).decode('ascii')
if content != 'foobar':
sys.stderr.write('Did not get expected content: %s\n' % content)
request.send_response(400)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/oss_fake_bucket3/another_file.bin',
custom_method=method)
gdal.ErrorReset()
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
assert gdal.GetLastErrorMsg() == ''
def vsis3_6():
if gdaltest.webserver_port == 0:
return 'skip'
gdal.SetConfigOption('VSIS3_CHUNK_SIZE', '1') # 1 MB
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket4/large_file.bin', 'wb')
gdal.SetConfigOption('VSIS3_CHUNK_SIZE', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
size = 1024 * 1024 + 1
ret = gdal.VSIFWriteL(''.join('a' for i in range(size)), 1, size, f)
if ret != size:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
gdal.VSIFCloseL(f)
if gdal.GetLastErrorMsg() != '':
gdaltest.post_reason('fail')
return 'fail'
for filename in [
'/vsis3/s3_fake_bucket4/large_file_initiate_403_error.bin',
'/vsis3/s3_fake_bucket4/large_file_initiate_empty_result.bin',
'/vsis3/s3_fake_bucket4/large_file_initiate_invalid_xml_result.bin',
'/vsis3/s3_fake_bucket4/large_file_initiate_no_uploadId.bin'
]:
gdal.SetConfigOption('VSIS3_CHUNK_SIZE', '1') # 1 MB
f = gdal.VSIFOpenL(filename, 'wb')
gdal.SetConfigOption('VSIS3_CHUNK_SIZE', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
size = 1024 * 1024 + 1
with gdaltest.error_handler():
ret = gdal.VSIFWriteL(''.join('a' for i in range(size)), 1, size,
f)
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
gdal.ErrorReset()
gdal.VSIFCloseL(f)
if gdal.GetLastErrorMsg() != '':
gdaltest.post_reason('fail')
return 'fail'
for filename in [
'/vsis3/s3_fake_bucket4/large_file_upload_part_403_error.bin',
'/vsis3/s3_fake_bucket4/large_file_upload_part_no_etag.bin'
]:
gdal.SetConfigOption('VSIS3_CHUNK_SIZE', '1') # 1 MB
f = gdal.VSIFOpenL(filename, 'wb')
gdal.SetConfigOption('VSIS3_CHUNK_SIZE', None)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
size = 1024 * 1024 + 1
with gdaltest.error_handler():
ret = gdal.VSIFWriteL(''.join('a' for i in range(size)), 1, size,
f)
if ret != 0:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
gdal.ErrorReset()
gdal.VSIFCloseL(f)
if gdal.GetLastErrorMsg() != '':
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def vsiaz_extra_1():
if not gdaltest.built_against_curl():
return 'skip'
az_resource = gdal.GetConfigOption('AZ_RESOURCE')
if az_resource is None:
print('Missing AZ_RESOURCE for running gdaltest_list_extra')
return 'skip'
if az_resource.find('/') < 0:
path = '/vsiaz/' + az_resource
statres = gdal.VSIStatL(path)
if statres is None or not stat.S_ISDIR(statres.mode):
gdaltest.post_reason('fail')
print('%s is not a valid bucket' % path)
return 'fail'
readdir = gdal.ReadDir(path)
if readdir is None:
gdaltest.post_reason('fail')
print('ReadDir() should not return empty list')
return 'fail'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
if gdal.VSIStatL(subpath) is None:
gdaltest.post_reason('fail')
print('Stat(%s) should not return an error' % subpath)
return 'fail'
unique_id = 'vsiaz_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
if ret < 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) should not return an error' % subpath)
return 'fail'
readdir = gdal.ReadDir(path)
if unique_id not in readdir:
gdaltest.post_reason('fail')
print('ReadDir(%s) should contain %s' % (path, unique_id))
print(readdir)
return 'fail'
ret = gdal.Mkdir(subpath, 0)
if ret == 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) repeated should return an error' % subpath)
return 'fail'
ret = gdal.Rmdir(subpath)
if ret < 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) should not return an error' % subpath)
return 'fail'
readdir = gdal.ReadDir(path)
if unique_id in readdir:
gdaltest.post_reason('fail')
print('ReadDir(%s) should not contain %s' % (path, unique_id))
print(readdir)
return 'fail'
ret = gdal.Rmdir(subpath)
if ret == 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) repeated should return an error' % subpath)
return 'fail'
ret = gdal.Mkdir(subpath, 0)
if ret < 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) should not return an error' % subpath)
return 'fail'
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
if ret == 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) on non empty directory should return an error' %
subpath)
return 'fail'
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
if data != 'hello':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test.txt')
if ret < 0:
gdaltest.post_reason('fail')
print('Unlink(%s) should not return an error' %
(subpath + '/test.txt'))
return 'fail'
ret = gdal.Rmdir(subpath)
if ret < 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) should not return an error' % subpath)
return 'fail'
return 'success'
f = open_for_read('/vsiaz/' + az_resource)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Same with /vsiaz_streaming/
f = open_for_read('/vsiaz_streaming/' + az_resource)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
if False: # pylint: disable=using-constant-test
# we actually try to read at read() time and bSetError = false
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsiaz/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if gdal.VSIGetLastErrorMsg() == '':
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsiaz_streaming/' + az_resource +
'/invalid_resource.baz')
if f is not None:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
# Test GetSignedURL()
signed_url = gdal.GetSignedURL('/vsiaz/' + az_resource)
f = open_for_read('/vsicurl_streaming/' + signed_url)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
return 'success'
def _VSIFPrintfL(f, s):
gdal.VSIFWriteL(s, 1, len(s), f)
def ogr_tiger_4():
if ogrtest.tiger_ds is None:
return 'skip'
# load all the files into memory.
for filename in gdal.ReadDir('tmp/cache/TGR01001'):
if filename.startswith('.'):
continue
data = open('tmp/cache/TGR01001/' + filename, 'r').read()
f = gdal.VSIFOpenL('/vsimem/tigertest/' + filename, 'wb')
gdal.VSIFWriteL(data, 1, len(data), f)
gdal.VSIFCloseL(f)
# Try reading.
ogrtest.tiger_ds = ogr.Open('/vsimem/tigertest/TGR01001.RT1')
if ogrtest.tiger_ds is None:
gdaltest.post_reason('fail to open.')
return 'fail'
ogrtest.tiger_ds = None
# also test opening with a filename (#4443)
ogrtest.tiger_ds = ogr.Open('tmp/cache/TGR01001/TGR01001.RT1')
if ogrtest.tiger_ds is None:
gdaltest.post_reason('fail')
return 'fail'
# Check a few features.
cc_layer = ogrtest.tiger_ds.GetLayerByName('CompleteChain')
if cc_layer.GetFeatureCount() != 19289:
gdaltest.post_reason('wrong cc feature count')
return 'fail'
feat = cc_layer.GetNextFeature()
feat = cc_layer.GetNextFeature()
feat = cc_layer.GetNextFeature()
if feat.TLID != 2833200 or feat.FRIADDL is not None or feat.BLOCKL != 5000:
gdaltest.post_reason('wrong attribute on cc feature.')
return 'fail'
if ogrtest.check_feature_geometry(
feat,
'LINESTRING (-86.4402 32.504137,-86.440313 32.504009,-86.440434 32.503884,-86.440491 32.503805,-86.44053 32.503757,-86.440578 32.503641,-86.440593 32.503515,-86.440588 32.503252,-86.440596 32.50298)',
max_error=0.000001) != 0:
return 'fail'
feat = ogrtest.tiger_ds.GetLayerByName('TLIDRange').GetNextFeature()
if feat.MODULE != 'TGR01001' or feat.TLMINID != 2822718:
gdaltest.post_reason('got wrong TLIDRange attributes')
return 'fail'
# Try to recover memory from /vsimem.
for filename in gdal.ReadDir('tmp/cache/TGR01001'):
if filename.startswith('.'):
continue
gdal.Unlink('/vsimem/tigertest/' + filename)
return 'success'
def write(self, data):
gdal.VSIFWriteL(data, 1, len(data), self.f)
def vsiswift_extra_1():
if not gdaltest.built_against_curl():
return 'skip'
swift_resource = gdal.GetConfigOption('SWIFT_RESOURCE')
if swift_resource is None:
print('Missing SWIFT_RESOURCE for running gdaltest_list_extra')
return 'skip'
if swift_resource.find('/') < 0:
path = '/vsiswift/' + swift_resource
statres = gdal.VSIStatL(path)
if statres is None or not stat.S_ISDIR(statres.mode):
gdaltest.post_reason('fail')
print('%s is not a valid bucket' % path)
return 'fail'
readdir = gdal.ReadDir(path)
if readdir is None:
gdaltest.post_reason('fail')
print('ReadDir() should not return empty list')
return 'fail'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
if gdal.VSIStatL(subpath) is None:
gdaltest.post_reason('fail')
print('Stat(%s) should not return an error' % subpath)
return 'fail'
unique_id = 'vsiswift_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
if ret < 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) should not return an error' % subpath)
return 'fail'
readdir = gdal.ReadDir(path)
if unique_id not in readdir:
gdaltest.post_reason('fail')
print('ReadDir(%s) should contain %s' % (path, unique_id))
print(readdir)
return 'fail'
ret = gdal.Mkdir(subpath, 0)
if ret == 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) repeated should return an error' % subpath)
return 'fail'
ret = gdal.Rmdir(subpath)
if ret < 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) should not return an error' % subpath)
return 'fail'
readdir = gdal.ReadDir(path)
if unique_id in readdir:
gdaltest.post_reason('fail')
print('ReadDir(%s) should not contain %s' % (path, unique_id))
print(readdir)
return 'fail'
ret = gdal.Rmdir(subpath)
if ret == 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) repeated should return an error' % subpath)
return 'fail'
ret = gdal.Mkdir(subpath, 0)
if ret < 0:
gdaltest.post_reason('fail')
print('Mkdir(%s) should not return an error' % subpath)
return 'fail'
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
if ret == 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) on non empty directory should return an error' %
subpath)
return 'fail'
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
if data != 'hello':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test.txt')
if ret < 0:
gdaltest.post_reason('fail')
print('Unlink(%s) should not return an error' %
(subpath + '/test.txt'))
return 'fail'
ret = gdal.Rmdir(subpath)
if ret < 0:
gdaltest.post_reason('fail')
print('Rmdir(%s) should not return an error' % subpath)
return 'fail'
return 'success'
f = open_for_read('/vsiswift/' + swift_resource)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Same with /vsiswift_streaming/
f = open_for_read('/vsiswift_streaming/' + swift_resource)
if f is None:
gdaltest.post_reason('fail')
return 'fail'
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
if len(ret) != 1:
gdaltest.post_reason('fail')
print(ret)
return 'fail'
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsiswift_streaming/' + swift_resource +
'/invalid_resource.baz')
if f is not None:
gdaltest.post_reason('fail')
print(gdal.VSIGetLastErrorMsg())
return 'fail'
return 'success'
def vsifile_generic(filename):
start_time = time.time()
fp = gdal.VSIFOpenL(filename, 'wb+')
if fp is None:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFWriteL('0123456789', 1, 10, fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTruncateL(fp, 20) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTruncateL(fp, 5) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFSeekL(fp, 0, 2) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 5:
gdaltest.post_reason('failure')
return 'fail'
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 7:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if abs(start_time - statBuf.mtime) > 2:
gdaltest.post_reason('failure')
print(statBuf.mtime)
return 'fail'
fp = gdal.VSIFOpenL(filename, 'rb')
buf = gdal.VSIFReadL(1, 7, fp)
if gdal.VSIFWriteL('a', 1, 1, fp) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFTruncateL(fp, 0) == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(fp)
if buf.decode('ascii') != '01234XX':
gdaltest.post_reason('failure')
print(buf.decode('ascii'))
return 'fail'
# Test append mode on existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 9:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if gdal.Unlink(filename) != 0:
gdaltest.post_reason('failure')
return 'fail'
statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG)
if statBuf is not None:
gdaltest.post_reason('failure')
return 'fail'
# Test append mode on non existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 2:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if gdal.Unlink(filename) != 0:
gdaltest.post_reason('failure')
return 'fail'
return 'success'
def gdal_pansharpen(argv):
argv = gdal.GeneralCmdLineProcessor(argv)
if argv is None:
return -1
pan_name = None
last_name = None
spectral_ds = []
spectral_bands = []
out_name = None
bands = []
weights = []
frmt = None
creation_options = []
callback = gdal.TermProgress_nocb
resampling = None
spat_adjust = None
verbose_vrt = False
num_threads = None
bitdepth = None
nodata = None
i = 1
argc = len(argv)
while i < argc:
if (argv[i] == '-of' or argv[i] == '-f') and i < len(argv) - 1:
frmt = argv[i + 1]
i = i + 1
elif argv[i] == '-r' and i < len(argv) - 1:
resampling = argv[i + 1]
i = i + 1
elif argv[i] == '-spat_adjust' and i < len(argv) - 1:
spat_adjust = argv[i + 1]
i = i + 1
elif argv[i] == '-b' and i < len(argv) - 1:
bands.append(int(argv[i + 1]))
i = i + 1
elif argv[i] == '-w' and i < len(argv) - 1:
weights.append(float(argv[i + 1]))
i = i + 1
elif argv[i] == '-co' and i < len(argv) - 1:
creation_options.append(argv[i + 1])
i = i + 1
elif argv[i] == '-threads' and i < len(argv) - 1:
num_threads = argv[i + 1]
i = i + 1
elif argv[i] == '-bitdepth' and i < len(argv) - 1:
bitdepth = argv[i + 1]
i = i + 1
elif argv[i] == '-nodata' and i < len(argv) - 1:
nodata = argv[i + 1]
i = i + 1
elif argv[i] == '-q':
callback = None
elif argv[i] == '-verbose_vrt':
verbose_vrt = True
elif argv[i][0] == '-':
sys.stderr.write('Unrecognized option : %s\n' % argv[i])
return Usage()
elif pan_name is None:
pan_name = argv[i]
pan_ds = gdal.Open(pan_name)
if pan_ds is None:
return 1
else:
if last_name is not None:
pos = last_name.find(',band=')
if pos > 0:
spectral_name = last_name[0:pos]
ds = gdal.Open(spectral_name)
if ds is None:
return 1
band_num = int(last_name[pos + len(',band='):])
band = ds.GetRasterBand(band_num)
spectral_ds.append(ds)
spectral_bands.append(band)
else:
spectral_name = last_name
ds = gdal.Open(spectral_name)
if ds is None:
return 1
for j in range(ds.RasterCount):
spectral_ds.append(ds)
spectral_bands.append(ds.GetRasterBand(j + 1))
last_name = argv[i]
i = i + 1
if pan_name is None or not spectral_bands:
return Usage()
out_name = last_name
if frmt is None:
frmt = GetOutputDriverFor(out_name)
if not bands:
bands = [j + 1 for j in range(len(spectral_bands))]
else:
for band in bands:
if band < 0 or band > len(spectral_bands):
print('Invalid band number in -b: %d' % band)
return 1
if weights and len(weights) != len(spectral_bands):
print('There must be as many -w values specified as input spectral bands')
return 1
vrt_xml = """<VRTDataset subClass="VRTPansharpenedDataset">\n"""
if bands != [j + 1 for j in range(len(spectral_bands))]:
for i, band in enumerate(bands):
sband = spectral_bands[band - 1]
datatype = gdal.GetDataTypeName(sband.DataType)
colorname = gdal.GetColorInterpretationName(sband.GetColorInterpretation())
vrt_xml += """ <VRTRasterBand dataType="%s" band="%d" subClass="VRTPansharpenedRasterBand">
<ColorInterp>%s</ColorInterp>
</VRTRasterBand>\n""" % (datatype, i + 1, colorname)
vrt_xml += """ <PansharpeningOptions>\n"""
if weights:
vrt_xml += """ <AlgorithmOptions>\n"""
vrt_xml += """ <Weights>"""
for i, weight in enumerate(weights):
if i > 0:
vrt_xml += ","
vrt_xml += "%.16g" % weight
vrt_xml += "</Weights>\n"
vrt_xml += """ </AlgorithmOptions>\n"""
if resampling is not None:
vrt_xml += ' <Resampling>%s</Resampling>\n' % resampling
if num_threads is not None:
vrt_xml += ' <NumThreads>%s</NumThreads>\n' % num_threads
if bitdepth is not None:
vrt_xml += ' <BitDepth>%s</BitDepth>\n' % bitdepth
if nodata is not None:
vrt_xml += ' <NoData>%s</NoData>\n' % nodata
if spat_adjust is not None:
vrt_xml += ' <SpatialExtentAdjustment>%s</SpatialExtentAdjustment>\n' % spat_adjust
pan_relative = '0'
if frmt.upper() == 'VRT':
if not os.path.isabs(pan_name):
pan_relative = '1'
pan_name = os.path.relpath(pan_name, os.path.dirname(out_name))
vrt_xml += """ <PanchroBand>
<SourceFilename relativeToVRT="%s">%s</SourceFilename>
<SourceBand>1</SourceBand>
</PanchroBand>\n""" % (pan_relative, pan_name)
for i, sband in enumerate(spectral_bands):
dstband = ''
for j, band in enumerate(bands):
if i + 1 == band:
dstband = ' dstBand="%d"' % (j + 1)
break
ms_relative = '0'
ms_name = spectral_ds[i].GetDescription()
if frmt.upper() == 'VRT':
if not os.path.isabs(ms_name):
ms_relative = '1'
ms_name = os.path.relpath(ms_name, os.path.dirname(out_name))
vrt_xml += """ <SpectralBand%s>
<SourceFilename relativeToVRT="%s">%s</SourceFilename>
<SourceBand>%d</SourceBand>
</SpectralBand>\n""" % (dstband, ms_relative, ms_name, sband.GetBand())
vrt_xml += """ </PansharpeningOptions>\n"""
vrt_xml += """</VRTDataset>\n"""
if frmt.upper() == 'VRT':
f = gdal.VSIFOpenL(out_name, 'wb')
if f is None:
print('Cannot create %s' % out_name)
return 1
gdal.VSIFWriteL(vrt_xml, 1, len(vrt_xml), f)
gdal.VSIFCloseL(f)
if verbose_vrt:
vrt_ds = gdal.Open(out_name, gdal.GA_Update)
vrt_ds.SetMetadata(vrt_ds.GetMetadata())
else:
vrt_ds = gdal.Open(out_name)
if vrt_ds is None:
return 1
return 0
vrt_ds = gdal.Open(vrt_xml)
out_ds = gdal.GetDriverByName(frmt).CreateCopy(out_name, vrt_ds, 0, creation_options, callback=callback)
if out_ds is None:
return 1
return 0
def vsizip_2():
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/foo.bar", "wb")
if fmain is None:
gdaltest.post_reason('fail 1')
return 'fail'
gdal.VSIFWriteL("12345", 1, 5, fmain)
gdal.VSIFCloseL(fmain)
content = gdal.ReadDir("/vsizip/vsimem/test2.zip")
if content != ['foo.bar']:
gdaltest.post_reason('bad content 1')
print(content)
return 'fail'
# Now append a second file
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/bar.baz", "wb")
if fmain is None:
gdaltest.post_reason('fail 2')
return 'fail'
gdal.VSIFWriteL("67890", 1, 5, fmain)
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
content = gdal.ReadDir("/vsizip/vsimem/test2.zip")
gdal.PopErrorHandler()
if gdal.GetLastErrorMsg() != 'Cannot read a zip file being written':
gdaltest.post_reason('expected error')
print(gdal.GetLastErrorMsg())
return 'fail'
if content != None:
gdaltest.post_reason('bad content 2')
print(content)
return 'fail'
gdal.VSIFCloseL(fmain)
content = gdal.ReadDir("/vsizip/vsimem/test2.zip")
if content != ['foo.bar', 'bar.baz']:
gdaltest.post_reason('bad content 3')
print(content)
return 'fail'
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/foo.bar", "rb")
if fmain is None:
gdaltest.post_reason('fail 3')
return 'fail'
data = gdal.VSIFReadL(1, 5, fmain)
gdal.VSIFCloseL(fmain)
if data.decode('ASCII') != '12345':
print(data)
return 'fail'
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/bar.baz", "rb")
if fmain is None:
gdaltest.post_reason('fail 4')
return 'fail'
data = gdal.VSIFReadL(1, 5, fmain)
gdal.VSIFCloseL(fmain)
if data.decode('ASCII') != '67890':
print(data)
return 'fail'
gdal.Unlink("/vsimem/test2.zip")
return 'success'
def vsifile_generic(filename):
start_time = time.time()
fp = gdal.VSIFOpenL(filename, 'wb+')
assert fp is not None
assert gdal.VSIFWriteL('0123456789', 1, 10, fp) == 10
assert gdal.VSIFFlushL(fp) == 0
assert gdal.VSIFTruncateL(fp, 20) == 0
assert gdal.VSIFTellL(fp) == 10
assert gdal.VSIFTruncateL(fp, 5) == 0
assert gdal.VSIFTellL(fp) == 10
assert gdal.VSIFSeekL(fp, 0, 2) == 0
assert gdal.VSIFTellL(fp) == 5
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
assert statBuf.size == 7
assert start_time == pytest.approx(statBuf.mtime, abs=2)
fp = gdal.VSIFOpenL(filename, 'rb')
buf = gdal.VSIFReadL(1, 7, fp)
assert gdal.VSIFWriteL('a', 1, 1, fp) == 0
assert gdal.VSIFTruncateL(fp, 0) != 0
gdal.VSIFCloseL(fp)
assert buf.decode('ascii') == '01234XX'
# Test append mode on existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
assert statBuf.size == 9
assert gdal.Unlink(filename) == 0
statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG)
assert statBuf is None
# Test append mode on non existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
assert statBuf.size == 2
assert gdal.Unlink(filename) == 0
def test_vsiaz_fake_write():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
# Test creation of BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:AigkrY7q66WCrx3JRKBte56k7kxV2cxB/ZyGNubxk5I=' or \
'Expect' not in h or h['Expect'] != '100-continue' or \
'Content-Length' not in h or h['Content-Length'] != '40000' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'BlockBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(40000).decode('ascii')
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Simulate illegal read
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
assert not ret
gdal.VSIFCloseL(f)
# Simulate illegal seek
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
assert ret != 0
gdal.VSIFCloseL(f)
# Simulate failure when putting BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method)
if gdal.VSIFSeekL(f, 0, 0) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if gdal.VSIFTellL(f) != 35000:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFSeekL(f, 35000, 0) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFSeekL(f, 0, 1) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFSeekL(f, 0, 2) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFEofL(f) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFCloseL(f)
if ret == 0:
gdal.VSIFCloseL(f)
pytest.fail(ret)
# Simulate creation of BlockBob over an existing blob of incompatible type
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 409)
handler.add('DELETE', '/azure/blob/myaccount/test_copy/file.bin', 202)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
# Test creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:KimVui3ptY9D5ftLlsI7CNOgK36CNAEzsXqcuHskdEY=' or \
'Content-Length' not in h or h['Content-Length'] != '0' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '10' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(10).decode('ascii')
if content != '0123456789':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '6' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(6).decode('ascii')
if content != 'abcdef':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 16:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Test failed creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Test failed writing of a block of an AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
assert f is not None
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 403)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
def extract_tile(ds, src_band_nbr, tile_x, tile_y, jpg_filename):
block_offset = ds.GetRasterBand(src_band_nbr).GetMetadataItem(
'BLOCK_OFFSET_%d_%d' % (tile_x, tile_y), 'TIFF')
block_size = ds.GetRasterBand(src_band_nbr).GetMetadataItem(
'BLOCK_SIZE_%d_%d' % (tile_x, tile_y), 'TIFF')
if block_offset is None or block_size is None:
print('ERROR: Cannot find block (%d,%d)' % (tile_x, tile_y))
return 1
jpegtables = ds.GetRasterBand(src_band_nbr).GetMetadataItem(
'JPEGTABLES', 'TIFF')
if jpegtables is not None:
if (len(jpegtables) % 2
) != 0 or jpegtables[0:4] != 'FFD8' or jpegtables[-2:] != 'D9':
print('ERROR: Invalid JPEG tables')
print(jpegtables)
return 1
# Remove final D9
jpegtables = jpegtables[0:-2]
tiff_f = gdal.VSIFOpenL(ds.GetDescription(), 'rb')
if tiff_f is None:
print('ERROR: Cannot reopen %s' % ds.GetDescription())
return 1
out_f = gdal.VSIFOpenL(jpg_filename, 'wb')
if out_f is None:
print('ERROR: Cannot create %s' % jpg_filename)
gdal.VSIFCloseL(tiff_f)
return 1
# Write JPEG tables
if jpegtables is not None:
for i in range(int(len(jpegtables) / 2)):
c1 = ord(jpegtables[2 * i])
c2 = ord(jpegtables[2 * i + 1])
if c1 >= ord('0') and c1 <= ord('9'):
val = c1 - ord('0')
else:
val = (c1 - ord('A')) + 10
val = val * 16
if c2 >= ord('0') and c2 <= ord('9'):
val = val + (c2 - ord('0'))
else:
val = val + (c2 - ord('A')) + 10
gdal.VSIFWriteL(chr(val), 1, 1, out_f)
else:
gdal.VSIFWriteL(chr(0xFF), 1, 1, out_f)
gdal.VSIFWriteL(chr(0xD8), 1, 1, out_f)
# Write Adobe APP14 marker if necessary
interleave = ds.GetMetadataItem('INTERLEAVE', 'IMAGE_STRUCTURE')
photometric = ds.GetMetadataItem('COMPRESSION', 'IMAGE_STRUCTURE')
if interleave == 'PIXEL' and photometric == 'JPEG' and ds.RasterCount == 3:
adobe_app14 = [
0xFF, 0xEE, 0x00, 0x0E, 0x41, 0x64, 0x6F, 0x62, 0x65, 0x00, 0x64,
0x00, 0x00, 0x00, 0x00, 0x00
]
for c in adobe_app14:
gdal.VSIFWriteL(chr(c), 1, 1, out_f)
# Write JPEG codestream
# skip leading 0xFF 0xD8
gdal.VSIFSeekL(tiff_f, int(block_offset) + 2, 0)
data = gdal.VSIFReadL(1, int(block_size) - 2, tiff_f)
gdal.VSIFCloseL(tiff_f)
gdal.VSIFWriteL(data, 1, len(data), out_f)
gdal.VSIFCloseL(out_f)
aux_xml_filename = '%s.aux.xml' % jpg_filename
gt = ds.GetGeoTransform()
srs = ds.GetProjectionRef()
if srs is not None and srs != '':
sub_gt = [gt[i] for i in range(6)]
(blockxsize, blockysize) = ds.GetRasterBand(1).GetBlockSize()
sub_gt[0] = gt[0] + tile_x * blockxsize * gt[1]
sub_gt[3] = gt[3] + tile_y * blockysize * gt[5]
out_f = gdal.VSIFOpenL(aux_xml_filename, 'wb')
if out_f is None:
print('ERROR: Cannot create %s' % aux_xml_filename)
return 1
content = """<PAMDataset>
<SRS>%s</SRS>
<GeoTransform>%.18g,%.18g,%.18g,%.18g,%.18g,%.18g</GeoTransform>
</PAMDataset>
""" % (srs, sub_gt[0], sub_gt[1], sub_gt[2], sub_gt[3], sub_gt[4],
sub_gt[5])
gdal.VSIFWriteL(content, 1, len(content), out_f)
gdal.VSIFCloseL(out_f)
else:
gdal.Unlink('%s.aux.xml' % jpg_filename)
return 0
def ecrgtoc_4():
toc_xml = """<Table_of_Contents>
<file_header file_status="new">
<file_name>TOC.xml</file_name>
</file_header>
<product product_title="ProductTitle">
<disc id="DiscId">
<frame_list number_of_frames="2">
<scale size="1:500 K">
<frame name="000000009s0013.lf2">
<frame_path>clfc\\2</frame_path>
<frame_version>001</frame_version>
<frame_chart_type>lf</frame_chart_type>
<frame_zone>2</frame_zone>
</frame>
</scale>
<scale size="1:1000 K">
<frame name="000000009s0013.lf2">
<frame_path>clfc\\2</frame_path>
<frame_version>001</frame_version>
<frame_chart_type>lf</frame_chart_type>
<frame_zone>2</frame_zone>
</frame>
</scale>
</frame_list>
</disc>
<disc id="DiscId2">
<frame_list number_of_frames="1">
<scale size="1:500 K">
<frame name="000000009t0013.lf2">
<frame_path>clfc\\2</frame_path>
<frame_version>001</frame_version>
<frame_chart_type>lf</frame_chart_type>
<frame_zone>2</frame_zone>
</frame>
</scale>
</frame_list>
</disc>
</product>
<extension_list>
<extension code="LF">
<chart_code>LF</chart_code>
<chart_type>1:500 K (LFC Day)</chart_type>
<chart_scale>1:500 K</chart_scale>
<chart_description>LFC Day</chart_description>
</extension>
</extension_list>
</Table_of_Contents>"""
f = gdal.VSIFOpenL('/vsimem/TOC.xml', 'wb')
gdal.VSIFWriteL(toc_xml, 1, len(toc_xml), f)
gdal.VSIFCloseL(f)
ds = gdal.Open('/vsimem/TOC.xml')
if ds is None:
return 'fail'
if ds.RasterCount != 0:
gdaltest.post_reason('bad raster count')
return 'fail'
expected_gt = (-85.43147208121826, 0.00059486040609137061, 0.0,
37.241379310344833, 0.0, -0.00044985604606525913)
gt = ds.GetGeoTransform()
for i in range(6):
if abs(gt[i] - expected_gt[i]) > 1e-10:
gdaltest.post_reason('did not get expected geotransform')
print(gt)
return 'fail'
wkt = ds.GetProjectionRef()
if wkt.find('WGS 84') == -1:
gdaltest.post_reason('did not get expected SRS')
print(wkt)
return 'fail'
filelist = ds.GetFileList()
if len(filelist) != 4:
gdaltest.post_reason('did not get expected filelist')
print(filelist)
return 'fail'
subdatasets = ds.GetMetadata('SUBDATASETS')
if len(subdatasets) != 6:
gdaltest.post_reason('did not get expected subdatasets')
print(filelist)
return 'fail'
ds = None
ds = gdal.Open(
'ECRG_TOC_ENTRY:ProductTitle:DiscId:1_500_K:/vsimem/TOC.xml')
if ds is None:
gdaltest.post_reason('did not get subdataset')
return 'fail'
ds = None
ds = gdal.Open(
'ECRG_TOC_ENTRY:ProductTitle:DiscId:1_1000_K:/vsimem/TOC.xml')
if ds is None:
gdaltest.post_reason('did not get subdataset')
return 'fail'
ds = None
ds = gdal.Open(
'ECRG_TOC_ENTRY:ProductTitle:DiscId2:1_500_K:/vsimem/TOC.xml')
if ds is None:
gdaltest.post_reason('did not get subdataset')
return 'fail'
ds = None
gdal.PushErrorHandler('CPLQuietErrorHandler')
ds = gdal.Open('ECRG_TOC_ENTRY:ProductTitle:DiscId:/vsimem/TOC.xml')
gdal.PopErrorHandler()
if ds is not None:
gdaltest.post_reason('should not have got subdataset')
return 'fail'
gdal.Unlink('/vsimem/TOC.xml')
return 'success'
def vsis3_4():
if gdaltest.webserver_port == 0:
return 'skip'
with gdaltest.error_handler():
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3', 'wb')
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsis3/s3_fake_bucket3/empty_file.bin').size != 3:
gdaltest.post_reason('fail')
return 'fail'
# Empty file
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
gdal.VSIFCloseL(f)
if gdal.GetLastErrorMsg() != '':
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIStatL('/vsis3/s3_fake_bucket3/empty_file.bin').size != 0:
gdaltest.post_reason('fail')
return 'fail'
# Invalid seek
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
# Invalid read
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
if len(ret) != 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
# Error case
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file_error.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
with gdaltest.error_handler():
gdal.VSIFCloseL(f)
if gdal.GetLastErrorMsg() == '':
gdaltest.post_reason('fail')
return 'fail'
# Nominal case
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/another_file.bin', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFSeekL(f, 0, 1) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFSeekL(f, 0, 2) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFWriteL('foo', 1, 3, f) != 3:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFWriteL('bar', 1, 3, f) != 3:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
gdal.VSIFCloseL(f)
if gdal.GetLastErrorMsg() != '':
gdaltest.post_reason('fail')
return 'fail'
# Redirect case
f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/redirect', 'wb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFWriteL('foobar', 1, 6, f) != 6:
gdaltest.post_reason('fail')
return 'fail'
gdal.ErrorReset()
gdal.VSIFCloseL(f)
if gdal.GetLastErrorMsg() != '':
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def ogr_osm_10():
if ogrtest.osm_drv is None:
return 'skip'
# A file that does not exist.
ds = ogr.Open('/nonexistent/foo.osm')
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
# Empty .osm file
f = gdal.VSIFOpenL('/vsimem/foo.osm', 'wb')
gdal.VSIFCloseL(f)
ds = ogr.Open('/vsimem/foo.osm')
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
gdal.Unlink('/vsimem/foo.osm')
# Empty .pbf file
f = gdal.VSIFOpenL('/vsimem/foo.pbf', 'wb')
gdal.VSIFCloseL(f)
ds = ogr.Open('/vsimem/foo.pbf')
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
gdal.Unlink('/vsimem/foo.pbf')
if ogrtest.osm_drv_parse_osm:
# Invalid .osm file
f = gdal.VSIFOpenL('/vsimem/foo.osm', 'wb')
data = "<osm>"
gdal.VSIFWriteL(data, 1, len(data), f)
gdal.VSIFCloseL(f)
ds = ogr.Open('/vsimem/foo.osm')
lyr = ds.GetLayer(0)
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
feat = lyr.GetNextFeature()
gdal.PopErrorHandler()
if gdal.GetLastErrorMsg() == '':
gdaltest.post_reason('fail')
return 'fail'
ds = None
gdal.Unlink('/vsimem/foo.osm')
# Invalid .pbf file
f = gdal.VSIFOpenL('/vsimem/foo.pbf', 'wb')
data = "OSMHeader\n"
gdal.VSIFWriteL(data, 1, len(data), f)
gdal.VSIFCloseL(f)
ds = ogr.Open('/vsimem/foo.pbf')
lyr = ds.GetLayer(0)
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
feat = lyr.GetNextFeature()
gdal.PopErrorHandler()
if gdal.GetLastErrorMsg() == '':
gdaltest.post_reason('fail')
return 'fail'
ds = None
gdal.Unlink('/vsimem/foo.pbf')
# Test million laugh pattern
if ogrtest.osm_drv_parse_osm:
ds = ogr.Open('data/billionlaugh.osm')
lyr = ds.GetLayer(0)
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
feat = lyr.GetNextFeature()
gdal.PopErrorHandler()
if feat is not None or gdal.GetLastErrorMsg() == '':
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def test_ogr_osm_10():
if ogrtest.osm_drv is None:
pytest.skip()
# A file that does not exist.
ds = ogr.Open('/nonexistent/foo.osm')
assert ds is None
# Empty .osm file
f = gdal.VSIFOpenL('/vsimem/foo.osm', 'wb')
gdal.VSIFCloseL(f)
ds = ogr.Open('/vsimem/foo.osm')
assert ds is None
gdal.Unlink('/vsimem/foo.osm')
# Empty .pbf file
f = gdal.VSIFOpenL('/vsimem/foo.pbf', 'wb')
gdal.VSIFCloseL(f)
ds = ogr.Open('/vsimem/foo.pbf')
assert ds is None
gdal.Unlink('/vsimem/foo.pbf')
if ogrtest.osm_drv_parse_osm:
# Invalid .osm file
f = gdal.VSIFOpenL('/vsimem/foo.osm', 'wb')
data = "<osm>"
gdal.VSIFWriteL(data, 1, len(data), f)
gdal.VSIFCloseL(f)
ds = ogr.Open('/vsimem/foo.osm')
lyr = ds.GetLayer(0)
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
feat = lyr.GetNextFeature()
gdal.PopErrorHandler()
assert gdal.GetLastErrorMsg() != ''
ds = None
gdal.Unlink('/vsimem/foo.osm')
# Invalid .pbf file
f = gdal.VSIFOpenL('/vsimem/foo.pbf', 'wb')
data = "OSMHeader\n"
gdal.VSIFWriteL(data, 1, len(data), f)
gdal.VSIFCloseL(f)
ds = ogr.Open('/vsimem/foo.pbf')
lyr = ds.GetLayer(0)
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
feat = lyr.GetNextFeature()
gdal.PopErrorHandler()
assert gdal.GetLastErrorMsg() != ''
ds = None
gdal.Unlink('/vsimem/foo.pbf')
# Test million laugh pattern
if ogrtest.osm_drv_parse_osm:
ds = ogr.Open('data/osm/billionlaugh.osm')
lyr = ds.GetLayer(0)
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
feat = lyr.GetNextFeature()
gdal.PopErrorHandler()
assert feat is None and gdal.GetLastErrorMsg() != ''