def test_vsizip_1(): # We can keep the handle open during all the ZIP writing hZIP = gdal.VSIFOpenL("/vsizip/vsimem/test.zip", "wb") assert hZIP is not None, 'fail 1' # One way to create a directory f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir2/", "wb") assert f is not None, 'fail 2' gdal.VSIFCloseL(f) # A more natural one gdal.Mkdir("/vsizip/vsimem/test.zip/subdir1", 0) # Create 1st file f2 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "wb") assert f2 is not None, 'fail 3' gdal.VSIFWriteL("abcd", 1, 4, f2) gdal.VSIFCloseL(f2) # Test that we cannot read a zip file being written gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb") gdal.PopErrorHandler() assert gdal.GetLastErrorMsg() == 'Cannot read a zip file being written', \ 'expected error' assert f is None, 'should not have been successful 1' # Create 2nd file f3 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/efghi", "wb") assert f3 is not None, 'fail 4' gdal.VSIFWriteL("efghi", 1, 5, f3) # Try creating a 3d file gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') f4 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/that_wont_work", "wb") gdal.PopErrorHandler() assert gdal.GetLastErrorMsg() == 'Cannot create that_wont_work while another file is being written in the .zip', \ 'expected error' assert f4 is None, 'should not have been successful 2' gdal.VSIFCloseL(f3) # Now we can close the main handle gdal.VSIFCloseL(hZIP) # ERROR 6: Support only 1 file in archive file /vsimem/test.zip when no explicit in-archive filename is specified gdal.ErrorReset() with gdaltest.error_handler(): f = gdal.VSIFOpenL('/vsizip/vsimem/test.zip', 'rb') if f is not None: gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() != '', 'expected error' f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb") assert f is not None, 'fail 5' data = gdal.VSIFReadL(1, 4, f) gdal.VSIFCloseL(f) assert data.decode('ASCII') == 'abcd' # Test alternate uri syntax gdal.Rename("/vsimem/test.zip", "/vsimem/test.xxx") f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd", "rb") assert f is not None data = gdal.VSIFReadL(1, 4, f) gdal.VSIFCloseL(f) assert data.decode('ASCII') == 'abcd' # With a trailing slash f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd/", "rb") assert f is not None gdal.VSIFCloseL(f) # Test ReadDir() assert len(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}")) == 3 # Unbalanced curls f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx", "rb") assert f is None # Non existing mainfile f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/bla", "rb") assert f is None # Non existing subfile f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.zzz}/bla", "rb") assert f is None # Wrong syntax f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}.aux.xml", "rb") assert f is None # Test nested { { } } hZIP = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}", "wb") assert hZIP is not None, 'fail 1' f = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}/test.xxx", "wb") f_src = gdal.VSIFOpenL("/vsimem/test.xxx", "rb") data = gdal.VSIFReadL(1, 10000, f_src) gdal.VSIFCloseL(f_src) gdal.VSIFWriteL(data, 1, len(data), f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(hZIP) f = gdal.VSIFOpenL( "/vsizip/{/vsizip/{/vsimem/zipinzip.yyy}/test.xxx}/subdir3/abcd/", "rb") assert f is not None data = gdal.VSIFReadL(1, 4, f) gdal.VSIFCloseL(f) assert data.decode('ASCII') == 'abcd' gdal.Unlink("/vsimem/test.xxx") gdal.Unlink("/vsimem/zipinzip.yyy") # Test VSIStatL on a non existing file assert gdal.VSIStatL('/vsizip//vsimem/foo.zip') is None # Test ReadDir on a non existing file assert gdal.ReadDir('/vsizip//vsimem/foo.zip') is None
def test_vsiaz_extra_1(): if not gdaltest.built_against_curl(): pytest.skip() az_resource = gdal.GetConfigOption('AZ_RESOURCE') if az_resource is None: pytest.skip('Missing AZ_RESOURCE') if '/' not in az_resource: path = '/vsiaz/' + az_resource statres = gdal.VSIStatL(path) assert statres is not None and stat.S_ISDIR(statres.mode), \ ('%s is not a valid bucket' % path) readdir = gdal.ReadDir(path) assert readdir is not None, 'ReadDir() should not return empty list' for filename in readdir: if filename != '.': subpath = path + '/' + filename assert gdal.VSIStatL(subpath) is not None, \ ('Stat(%s) should not return an error' % subpath) unique_id = 'vsiaz_test' subpath = path + '/' + unique_id ret = gdal.Mkdir(subpath, 0) assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath) readdir = gdal.ReadDir(path) assert unique_id in readdir, \ ('ReadDir(%s) should contain %s' % (path, unique_id)) ret = gdal.Mkdir(subpath, 0) assert ret != 0, ('Mkdir(%s) repeated should return an error' % subpath) ret = gdal.Rmdir(subpath) assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath) readdir = gdal.ReadDir(path) assert unique_id not in readdir, \ ('ReadDir(%s) should not contain %s' % (path, unique_id)) ret = gdal.Rmdir(subpath) assert ret != 0, ('Rmdir(%s) repeated should return an error' % subpath) ret = gdal.Mkdir(subpath, 0) assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath) f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb') assert f is not None gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) ret = gdal.Rmdir(subpath) assert ret != 0, \ ('Rmdir(%s) on non empty directory should return an error' % subpath) f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb') assert f is not None data = gdal.VSIFReadL(1, 5, f).decode('utf-8') assert data == 'hello' gdal.VSIFCloseL(f) assert gdal.Rename(subpath + '/test.txt', subpath + '/test2.txt') == 0 f = gdal.VSIFOpenL(subpath + '/test2.txt', 'rb') assert f is not None data = gdal.VSIFReadL(1, 5, f).decode('utf-8') assert data == 'hello' gdal.VSIFCloseL(f) ret = gdal.Unlink(subpath + '/test2.txt') assert ret >= 0, \ ('Unlink(%s) should not return an error' % (subpath + '/test2.txt')) ret = gdal.Rmdir(subpath) assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath) return f = open_for_read('/vsiaz/' + az_resource) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1 # Same with /vsiaz_streaming/ f = open_for_read('/vsiaz_streaming/' + az_resource) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1 if False: # pylint: disable=using-constant-test # we actually try to read at read() time and bSetError = false # Invalid bucket : "The specified bucket does not exist" gdal.ErrorReset() f = open_for_read('/vsiaz/not_existing_bucket/foo') with gdaltest.error_handler(): gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert gdal.VSIGetLastErrorMsg() != '' # Invalid resource gdal.ErrorReset() f = open_for_read('/vsiaz_streaming/' + az_resource + '/invalid_resource.baz') assert f is None, gdal.VSIGetLastErrorMsg() # Test GetSignedURL() signed_url = gdal.GetSignedURL('/vsiaz/' + az_resource) f = open_for_read('/vsicurl_streaming/' + signed_url) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1
def ecrgtoc_1(): toc_xml = """<Table_of_Contents> <file_header file_status="new"> <file_name>TOC.xml</file_name> </file_header> <product product_title="ProductTitle"> <disc id="DiscId"> <frame_list number_of_frames="2"> <scale size="1:500 K"> <frame name="000000009s0013.lf2"> <frame_path>clfc\\2</frame_path> <frame_version>001</frame_version> <frame_chart_type>lf</frame_chart_type> <frame_zone>2</frame_zone> </frame> <frame name="000000009t0013.lf2"> <frame_path>clfc\\2</frame_path> <frame_version>001</frame_version> <frame_chart_type>lf</frame_chart_type> <frame_zone>2</frame_zone> </frame> </scale> </frame_list> </disc> </product> <extension_list> <extension code="LF"> <chart_code>LF</chart_code> <chart_type>1:500 K (LFC Day)</chart_type> <chart_scale>1:500 K</chart_scale> <chart_description>LFC Day</chart_description> </extension> </extension_list> </Table_of_Contents>""" f = gdal.VSIFOpenL('/vsimem/TOC.xml', 'wb') gdal.VSIFWriteL(toc_xml, 1, len(toc_xml), f) gdal.VSIFCloseL(f) ds = gdal.Open('/vsimem/TOC.xml') if ds is None: return 'fail' expected_gt = [ -85.43147208121826, 0.00059486040609137061, 0.0, 33.166986564299428, 0.0, -0.00044985604606525913 ] gt = ds.GetGeoTransform() for i in range(6): if abs(gt[i] - expected_gt[i]) > 1e-10: gdaltest.post_reason('did not get expected geotransform') print(gt) wkt = ds.GetProjectionRef() if wkt.find('WGS 84') == -1: gdaltest.post_reason('did not get expected SRS') print(wkt) return 'fail' filelist = ds.GetFileList() if len(filelist) != 3: gdaltest.post_reason('did not get expected filelist') print(filelist) return 'fail' ds2 = gdal.GetDriverByName('NITF').Create('/vsimem/clfc/2/000000009s0013.lf2', 2304, 2304, 3, \ options = ['ICORDS=G', 'TRE=GEOLOB=000605184000800256-85.43147208122+33.16698656430']) ds2.SetGeoTransform([ -85.43147208122, 0.00059486040609137061, 0.0, 33.16698656430, 0.0, -0.00044985604606525913 ]) ds2.SetProjection(wkt) ds2.GetRasterBand(1).Fill(255) ds2 = None ds2 = gdal.GetDriverByName('NITF').Create('/vsimem/clfc/2/000000009t0013.lf2', 2304, 2304, 3, \ options = ['ICORDS=G', 'TRE=GEOLOB=000605184000800256-84.06091370558+33.16698656430']) ds2.SetGeoTransform([ -84.06091370558, 0.00059486040609137061, 0.0, 33.16698656430, 0.0, -0.00044985604606525913 ]) ds2.SetProjection(wkt) ds2 = None cs = ds.GetRasterBand(1).Checksum() ds = None if cs != 5966: gdaltest.post_reason('bad checksum') print(cs) return 'fail' return 'success'
def test_vsigs_write(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii')) content = '' while True: numchars = int(request.rfile.readline().strip(), 16) content += request.rfile.read(numchars).decode('ascii') request.rfile.read(2) if numchars == 0: break if len(content) != 40000: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(200) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/test_copy/file.bin', custom_method=method) with webserver.install_http_handler(handler): ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f) ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f) if ret != 40000: gdal.VSIFCloseL(f) pytest.fail(ret) gdal.VSIFCloseL(f) # Simulate failure while transmitting f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/test_copy/file.bin', custom_method=method) with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f) if ret != 0: gdal.VSIFCloseL(f) pytest.fail(ret) gdal.VSIFCloseL(f) # Simulate failure at end of transfer f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii')) content = '' while True: numchars = int(request.rfile.readline().strip(), 16) content += request.rfile.read(numchars).decode('ascii') request.rfile.read(2) if numchars == 0: break request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/test_copy/file.bin', custom_method=method) with webserver.install_http_handler(handler): ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f) if ret != 35000: gdal.VSIFCloseL(f) pytest.fail(ret) with gdaltest.error_handler(): ret = gdal.VSIFCloseL(f) assert ret != 0
def vsizip_1(): # We can keep the handle open during all the ZIP writing hZIP = gdal.VSIFOpenL("/vsizip/vsimem/test.zip", "wb") if hZIP is None: gdaltest.post_reason('fail 1') return 'fail' # One way to create a directory f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir2/", "wb") if f is None: gdaltest.post_reason('fail 2') return 'fail' gdal.VSIFCloseL(f) # A more natural one gdal.Mkdir("/vsizip/vsimem/test.zip/subdir1", 0) # Create 1st file f2 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "wb") if f2 is None: gdaltest.post_reason('fail 3') return 'fail' gdal.VSIFWriteL("abcd", 1, 4, f2) gdal.VSIFCloseL(f2) # Test that we cannot read a zip file being written gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb") gdal.PopErrorHandler() if gdal.GetLastErrorMsg() != 'Cannot read a zip file being written': gdaltest.post_reason('expected error') print(gdal.GetLastErrorMsg()) return 'fail' if f is not None: gdaltest.post_reason('should not have been successful 1') return 'fail' # Create 2nd file f3 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/efghi", "wb") if f3 is None: gdaltest.post_reason('fail 4') return 'fail' gdal.VSIFWriteL("efghi", 1, 5, f3) # Try creating a 3d file gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') f4 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/that_wont_work", "wb") gdal.PopErrorHandler() if gdal.GetLastErrorMsg( ) != 'Cannot create that_wont_work while another file is being written in the .zip': gdaltest.post_reason('expected error') print(gdal.GetLastErrorMsg()) return 'fail' if f4 is not None: gdaltest.post_reason('should not have been successful 2') return 'fail' gdal.VSIFCloseL(f3) # Now we can close the main handle gdal.VSIFCloseL(hZIP) f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb") if f is None: gdaltest.post_reason('fail 5') return 'fail' data = gdal.VSIFReadL(1, 4, f) gdal.VSIFCloseL(f) if data.decode('ASCII') != 'abcd': gdaltest.post_reason('fail') print(data) return 'fail' # Test alternate uri syntax gdal.Rename("/vsimem/test.zip", "/vsimem/test.xxx") f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd", "rb") if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f) gdal.VSIFCloseL(f) if data.decode('ASCII') != 'abcd': gdaltest.post_reason('fail') print(data) return 'fail' # With a trailing slash f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd/", "rb") if f is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) # Test ReadDir() if len(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}")) != 3: gdaltest.post_reason('fail') print(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}")) return 'fail' # Unbalanced curls f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx", "rb") if f is not None: gdaltest.post_reason('fail') return 'fail' # Non existing mainfile f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/bla", "rb") if f is not None: gdaltest.post_reason('fail') return 'fail' # Non existing subfile f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.zzz}/bla", "rb") if f is not None: gdaltest.post_reason('fail') return 'fail' # Wrong syntax f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}.aux.xml", "rb") if f is not None: gdaltest.post_reason('fail') return 'fail' # Test nested { { } } hZIP = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}", "wb") if hZIP is None: gdaltest.post_reason('fail 1') return 'fail' f = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}/test.xxx", "wb") f_src = gdal.VSIFOpenL("/vsimem/test.xxx", "rb") data = gdal.VSIFReadL(1, 10000, f_src) gdal.VSIFCloseL(f_src) gdal.VSIFWriteL(data, 1, len(data), f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(hZIP) f = gdal.VSIFOpenL( "/vsizip/{/vsizip/{/vsimem/zipinzip.yyy}/test.xxx}/subdir3/abcd/", "rb") if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f) gdal.VSIFCloseL(f) if data.decode('ASCII') != 'abcd': gdaltest.post_reason('fail') print(data) return 'fail' gdal.Unlink("/vsimem/test.xxx") gdal.Unlink("/vsimem/zipinzip.yyy") return 'success'
def gdal_cp_single(srcfile, targetfile, progress): if targetfile.endswith('/'): stat_res = gdal.VSIStatL(targetfile) else: stat_res = gdal.VSIStatL(targetfile + '/') if (stat_res is None and targetfile.endswith('/')) or \ (stat_res is not None and stat.S_ISDIR(stat_res.mode)): (_, tail) = os.path.split(srcfile) if targetfile.endswith('/'): targetfile = targetfile + tail else: targetfile = targetfile + '/' + tail fin = gdal.VSIFOpenL(srcfile, "rb") if fin is None: print('Cannot open %s' % srcfile) return -1 fout = gdal.VSIFOpenL(targetfile, "wb") if fout is None: print('Cannot create %s' % targetfile) gdal.VSIFCloseL(fin) return -1 version_num = int(gdal.VersionInfo('VERSION_NUM')) total_size = 0 if version_num < 1900 or progress is not None: gdal.VSIFSeekL(fin, 0, 2) total_size = gdal.VSIFTellL(fin) gdal.VSIFSeekL(fin, 0, 0) buf_max_size = 4096 copied = 0 ret = 0 # print('Copying %s...' % srcfile) if progress is not None: if not progress.Progress(0.0, 'Copying %s' % srcfile): print('Copy stopped by user') ret = -2 while ret == 0: if total_size != 0 and copied + buf_max_size > total_size: to_read = total_size - copied else: to_read = buf_max_size buf = gdal.VSIFReadL(1, to_read, fin) if buf is None: if copied == 0: print('Cannot read %d bytes in %s' % (to_read, srcfile)) ret = -1 break buf_size = len(buf) if gdal.VSIFWriteL(buf, 1, buf_size, fout) != buf_size: print('Error writing %d bytes' % buf_size) ret = -1 break copied += buf_size if progress is not None and total_size != 0: if not progress.Progress(copied * 1.0 / total_size, 'Copying %s' % srcfile): print('Copy stopped by user') ret = -2 break if to_read < buf_max_size or buf_size != buf_max_size: break gdal.VSIFCloseL(fin) gdal.VSIFCloseL(fout) return ret
def vsifile_5(): fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'wb') ref_data = ''.join(['%08X' % i for i in range(5 * 32768)]) gdal.VSIFWriteL(ref_data, 1, len(ref_data), fp) gdal.VSIFCloseL(fp) gdal.SetConfigOption('VSI_CACHE', 'YES') for i in range(3): if i == 0: gdal.SetConfigOption('VSI_CACHE_SIZE', '0') elif i == 1: gdal.SetConfigOption('VSI_CACHE_SIZE', '65536') else: gdal.SetConfigOption('VSI_CACHE_SIZE', None) fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'rb') gdal.VSIFSeekL(fp, 50000, 0) if gdal.VSIFTellL(fp) != 50000: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFSeekL(fp, 50000, 1) if gdal.VSIFTellL(fp) != 100000: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFSeekL(fp, 0, 2) if gdal.VSIFTellL(fp) != 5 * 32768 * 8: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFReadL(1, 1, fp) gdal.VSIFSeekL(fp, 0, 0) data = gdal.VSIFReadL(1, 3 * 32768, fp) if data.decode('ascii') != ref_data[0:3 * 32768]: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFSeekL(fp, 16384, 0) data = gdal.VSIFReadL(1, 5 * 32768, fp) if data.decode('ascii') != ref_data[16384:16384 + 5 * 32768]: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' data = gdal.VSIFReadL(1, 50 * 32768, fp) if data[0:1130496].decode('ascii') != ref_data[16384 + 5 * 32768:]: gdaltest.post_reason('fail') gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) return 'fail' gdal.VSIFCloseL(fp) gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) gdal.Unlink('tmp/vsifile_5.bin') return 'success'
def vsicrypt_2(): if not gdaltest.has_vsicrypt: return 'skip' # Missing key with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Invalid file with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'wb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Invalid file with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Invalid file with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'ab') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Invalid access with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=/not_existing/not_existing', 'foo') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Key to short with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=a,file=/vsimem/file.bin', 'wb+') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Invalid signature gdal.FileFromMemBuffer('/vsimem/file.bin', 'foo') with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Generate empty file fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'wb') gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') if fp is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsimem/file.bin', 'rb') header = gdal.VSIFReadL(1, 1000, fp) gdal.VSIFCloseL(fp) if len(header) != 46: gdaltest.post_reason('fail') print(len(header)) return 'fail' # Test shortening header for i in range(46): fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb') gdal.VSIFWriteL(header, 1, 46 - 1 - i, fp) gdal.VSIFCloseL(fp) with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Test corrupting all bytes of header for i in range(46): for val in (0, 127, 255): fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb') try: new_byte = chr(val).encode('latin1') except: new_byte = chr(val) header_new = header[0:i] + new_byte + header[i + 1:] gdal.VSIFWriteL(header_new, 1, 46, fp) gdal.VSIFCloseL(fp) with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=' '/vsimem/file.bin', 'rb') if fp is not None: gdal.VSIFCloseL(fp) gdal.SetConfigOption('VSICRYPT_IV', 'TOO_SHORT') with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=' '/vsimem/file.bin', 'wb') gdal.SetConfigOption('VSICRYPT_IV', None) if fp is not None: gdal.VSIFCloseL(fp) # Inconsistent initial vector. header = struct.pack( 'B' * 38, 86, 83, 73, 67, 82, 89, 80, 84, # signature 38, 0, # header size 1, # major 0, # minor 0, 2, # sector size 0, # alg 0, # mode 8, #size of IV (should be 16) 32, 13, 169, 71, 154, 208, 22, 32, #IV 0, 0, # size of free text 0, # size of key check 0, 0, 0, 0, 0, 0, 0, 0, # size of unencrypted file 0, 0 # size of extra content ) fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb') gdal.VSIFWriteL(header, 1, len(header), fp) gdal.VSIFCloseL(fp) with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Inconsistent initial vector with key check. header = struct.pack( 'B' * 39, 86, 83, 73, 67, 82, 89, 80, 84, # signature 39, 0, # header size 1, # major 0, # minor 0, 2, # sector size 0, # alg 0, # mode 8, #size of IV (should be 16) 32, 13, 169, 71, 154, 208, 22, 32, #IV 0, 0, # size of free text 1, # size of key check 0, # key check 0, 0, 0, 0, 0, 0, 0, 0, # size of unencrypted file 0, 0 # size of extra content ) fp = gdal.VSIFOpenL('/vsimem/file.bin', 'wb') gdal.VSIFWriteL(header, 1, len(header), fp) gdal.VSIFCloseL(fp) with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Test reading with wrong key fp = gdal.VSIFOpenL('/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'wb') gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin', 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content == 'hello': gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=short_key,file=/vsimem/file.bin', 'ab') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Test reading with wrong key with add_key_check fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,add_key_check=yes,file=/vsimem/file.bin', 'wb') gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt/key=short_key,file=/vsimem/file.bin', 'ab') if fp is not None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=dont_use_in_prod,file=/vsimem/file.bin', 'ab') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Test creating with potentially not build-in alg: with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/alg=blowfish,key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'wb') if fp is not None: gdal.VSIFCloseL(fp) # Invalid sector_size with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,sector_size=1,file=/vsimem/file.bin', 'wb') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Sector size (16) should be at least twice larger than the block size (16) in CBC_CTS with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,sector_size=16,mode=CBC_CTS,file=/vsimem/file.bin', 'wb') if fp is not None: gdaltest.post_reason('fail') return 'fail' gdal.Unlink('/vsimem/file.bin') return 'success'
def vsicrypt_6(): try: import ctypes except: return 'skip' import testnonboundtoswig testnonboundtoswig.testnonboundtoswig_init() if testnonboundtoswig.gdal_handle is None: return 'skip' testnonboundtoswig.gdal_handle.VSISetCryptKey.argtypes = [ ctypes.c_char_p, ctypes.c_int ] testnonboundtoswig.gdal_handle.VSISetCryptKey.restype = None # Set a valid key testnonboundtoswig.gdal_handle.VSISetCryptKey( 'DONT_USE_IN_PROD'.encode('ASCII'), 16) if not gdaltest.has_vsicrypt: return 'skip' fp = gdal.VSIFOpenL('/vsicrypt/add_key_check=yes,file=/vsimem/file.bin', 'wb+') if fp is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') return 'fail' fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+') if fp is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') return 'fail' # Set a too short key testnonboundtoswig.gdal_handle.VSISetCryptKey('bbc'.encode('ASCII'), 3) with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb') if fp is not None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+') if fp is not None: gdaltest.post_reason('fail') return 'fail' # Erase key testnonboundtoswig.gdal_handle.VSISetCryptKey(None, 0) with gdaltest.error_handler(): fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'wb+') if fp is not None: gdaltest.post_reason('fail') return 'fail' gdal.Unlink('/vsimem/file.bin') return 'success'
def vsicrypt_3(): if not gdaltest.has_vsicrypt: return 'skip' for options in [ 'sector_size=16', 'alg=AES', 'alg=DES_EDE2', 'alg=DES_EDE3', 'alg=SKIPJACK', 'alg=invalid', 'mode=CBC', 'mode=CFB', 'mode=OFB', 'mode=CTR', 'mode=CBC_CTS', 'mode=invalid', 'freetext=my_free_text', 'add_key_check=yes' ]: gdal.Unlink('/vsimem/file.bin') if options == 'alg=invalid' or options == 'mode=invalid': with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,%s,file=/vsimem/file.bin' % options, 'wb') else: fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,%s,file=/vsimem/file.bin' % options, 'wb') if fp is None: gdaltest.post_reason('fail') print(options) return 'fail' gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PRODDONT_USE_IN_PROD,file=/vsimem/file.bin', 'r') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') print(options) return 'fail' # Some of those algs might be missing for options in [ 'alg=Blowfish', 'alg=Camellia', 'alg=CAST256', 'alg=MARS', 'alg=IDEA', 'alg=RC5', 'alg=RC6', 'alg=Serpent', 'alg=SHACAL2', 'alg=Twofish', 'alg=XTEA' ]: gdal.Unlink('/vsimem/file.bin') with gdaltest.error_handler(): fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,%s,file=/vsimem/file.bin' % options, 'wb') if fp is not None: gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL( '/vsicrypt/key=DONT_USE_IN_PROD,file=/vsimem/file.bin', 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') print(options) return 'fail' # Test key generation # Do NOT set VSICRYPT_CRYPTO_RANDOM=NO in production. This is just to speed up tests ! gdal.SetConfigOption("VSICRYPT_CRYPTO_RANDOM", "NO") fp = gdal.VSIFOpenL( '/vsicrypt/key=GENERATE_IT,add_key_check=yes,file=/vsimem/file.bin', 'wb') gdal.SetConfigOption("VSICRYPT_CRYPTO_RANDOM", None) # Get the generated random key key_b64 = gdal.GetConfigOption('VSICRYPT_KEY_B64') if key_b64 is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFWriteL('hello', 1, 5, fp) gdal.VSIFCloseL(fp) fp = gdal.VSIFOpenL('/vsicrypt//vsimem/file.bin', 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') print(options) return 'fail' gdal.SetConfigOption('VSICRYPT_KEY_B64', None) fp = gdal.VSIFOpenL('/vsicrypt/key_b64=%s,file=/vsimem/file.bin' % key_b64, 'rb') content = gdal.VSIFReadL(1, 5, fp).decode('latin1') gdal.VSIFCloseL(fp) if content != 'hello': gdaltest.post_reason('fail') print(options) return 'fail' with gdaltest.error_handler(): statRes = gdal.VSIStatL('/vsicrypt//vsimem/file.bin') if statRes is not None: gdaltest.post_reason('fail') return 'fail' ret = gdal.Rename('/vsicrypt//vsimem/file.bin', '/vsicrypt//vsimem/subdir_crypt/file.bin') if ret != 0: gdaltest.post_reason('fail') print(ret) return 'fail' ret = gdal.Rename('/vsicrypt//vsimem/subdir_crypt/file.bin', '/vsimem/subdir_crypt/file2.bin') if ret != 0: gdaltest.post_reason('fail') print(ret) return 'fail' dir_content = gdal.ReadDir('/vsicrypt//vsimem/subdir_crypt') if dir_content != ['file2.bin']: gdaltest.post_reason('fail') print(dir_content) return 'fail' gdal.Unlink('/vsimem/subdir_crypt/file2.bin') return 'success'
def vsicrypt_4(): if not gdaltest.has_vsicrypt: return 'skip' test_file = '/vsicrypt/key=DONT_USE_IN_PROD,sector_size=32,file=/vsimem/file_enc.bin' ref_file = '/vsimem/file.bin' for seed in range(1000): gdal.Unlink(test_file) gdal.Unlink(ref_file) test_f = gdal.VSIFOpenL(test_file, 'wb+') ref_f = gdal.VSIFOpenL(ref_file, 'wb+') import random random.seed(seed) for i in range(20): random_offset = random.randint(0, 1000) gdal.VSIFSeekL(test_f, random_offset, 0) gdal.VSIFSeekL(ref_f, random_offset, 0) random_size = random.randint(1, 80) random_content = ''.join([ chr(40 + int(10 * random.random())) for i in range(random_size) ]) gdal.VSIFWriteL(random_content, 1, random_size, test_f) gdal.VSIFWriteL(random_content, 1, random_size, ref_f) if random.randint(0, 1) == 0: random_offset = random.randint(0, 1500) gdal.VSIFSeekL(test_f, random_offset, 0) gdal.VSIFSeekL(ref_f, random_offset, 0) random_size = random.randint(1, 80) test_content = gdal.VSIFReadL(1, random_size, test_f) ref_content = gdal.VSIFReadL(1, random_size, ref_f) if test_content != ref_content: print(seed) print('Test content (%d):' % len(test_content)) print(test_content) print('') print('Ref content (%d):' % len(ref_content)) print(ref_content) return 'fail' gdal.VSIFSeekL(test_f, 0, 0) gdal.VSIFSeekL(ref_f, 0, 0) test_content = gdal.VSIFReadL(1, 100000, test_f) ref_content = gdal.VSIFReadL(1, 100000, ref_f) if test_content != ref_content: print(seed) print('Test content (%d):' % len(test_content)) print(test_content) print('') print('Ref content (%d):' % len(ref_content)) print(ref_content) return 'fail' gdal.Unlink(test_file) gdal.Unlink(ref_file) return 'success'
def test_vsiadls_real_instance_tests(): adls_resource = gdal.GetConfigOption('ADLS_RESOURCE') if adls_resource is None: pytest.skip('Missing ADLS_RESOURCE') if '/' not in adls_resource: path = '/vsiadls/' + adls_resource try: statres = gdal.VSIStatL(path) assert statres is not None and stat.S_ISDIR(statres.mode), \ ('%s is not a valid bucket' % path) readdir = gdal.ReadDir(path) assert readdir is not None, 'ReadDir() should not return empty list' for filename in readdir: if filename != '.': subpath = path + '/' + filename assert gdal.VSIStatL(subpath) is not None, \ ('Stat(%s) should not return an error' % subpath) unique_id = 'vsiadls_test' subpath = path + '/' + unique_id ret = gdal.Mkdir(subpath, 0) assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath) readdir = gdal.ReadDir(path) assert unique_id in readdir, \ ('ReadDir(%s) should contain %s' % (path, unique_id)) ret = gdal.Mkdir(subpath, 0) assert ret != 0, ('Mkdir(%s) repeated should return an error' % subpath) ret = gdal.Rmdir(subpath) assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath) readdir = gdal.ReadDir(path) assert unique_id not in readdir, \ ('ReadDir(%s) should not contain %s' % (path, unique_id)) ret = gdal.Rmdir(subpath) assert ret != 0, ('Rmdir(%s) repeated should return an error' % subpath) ret = gdal.Mkdir(subpath, 0) assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath) f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb') assert f is not None gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) ret = gdal.Rmdir(subpath) assert ret != 0, \ ('Rmdir(%s) on non empty directory should return an error' % subpath) f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb') assert f is not None data = gdal.VSIFReadL(1, 5, f).decode('utf-8') assert data == 'hello' gdal.VSIFCloseL(f) assert gdal.VSIStatL(subpath + '/test.txt') is not None md = gdal.GetFileMetadata(subpath + '/test.txt', 'HEADERS') assert 'x-ms-properties' in md md = gdal.GetFileMetadata(subpath + '/test.txt', 'STATUS') assert 'x-ms-resource-type' in md assert 'x-ms-properties' not in md md = gdal.GetFileMetadata(subpath + '/test.txt', 'ACL') assert 'x-ms-acl' in md assert 'x-ms-permissions' in md # Change properties properties_foo_bar = 'foo=' + base64.b64encode('bar') assert gdal.SetFileMetadata( subpath + '/test.txt', {'x-ms-properties': properties_foo_bar}, 'PROPERTIES') md = gdal.GetFileMetadata(subpath + '/test.txt', 'HEADERS') assert 'x-ms-properties' in md assert md['x-ms-properties'] == properties_foo_bar # Change ACL assert gdal.SetFileMetadata(subpath + '/test.txt', {'x-ms-permissions': '0777'}, 'ACL') md = gdal.GetFileMetadata(subpath + '/test.txt', 'ACL') assert 'x-ms-permissions' in md assert md['x-ms-permissions'] == 'rwxrwxrwx' # Change ACL recursively md = gdal.GetFileMetadata(subpath, 'ACL') assert 'x-ms-acl' in md assert gdal.SetFileMetadata(subpath + '/test.txt', {'x-ms-acl': md['x-ms-acl']}, 'ACL', ['RECURSIVE=YES', 'MODE=set']) assert gdal.Rename(subpath + '/test.txt', subpath + '/test2.txt') == 0 assert gdal.VSIStatL(subpath + '/test.txt') is None assert gdal.VSIStatL(subpath + '/test2.txt') is not None f = gdal.VSIFOpenL(subpath + '/test2.txt', 'rb') assert f is not None data = gdal.VSIFReadL(1, 5, f).decode('utf-8') assert data == 'hello' gdal.VSIFCloseL(f) ret = gdal.Unlink(subpath + '/test2.txt') assert ret >= 0, \ ('Unlink(%s) should not return an error' % (subpath + '/test2.txt')) assert gdal.VSIStatL(subpath + '/test2.txt') is None assert gdal.Unlink( subpath + '/test2.txt' ) != 0, "Unlink on a deleted file should return an error" f = gdal.VSIFOpenL(subpath + '/test2.txt', 'wb') assert f is not None gdal.VSIFCloseL(f) assert gdal.VSIStatL(subpath + '/test2.txt') is not None finally: assert gdal.RmdirRecursive(subpath) == 0 return f = open_for_read('/vsiadls/' + adls_resource) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1 # Test GetSignedURL() signed_url = gdal.GetSignedURL('/vsiadls/' + adls_resource) f = open_for_read('/vsicurl_streaming/' + signed_url) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1
def test_visoss_6(): if gdaltest.webserver_port == 0: pytest.skip() with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB with webserver.install_http_handler(webserver.SequentialHandler()): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket4/large_file.bin', 'wb') assert f is not None size = 1024 * 1024 + 1 big_buffer = 'a' * size handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' response = '<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>' request.send_response(200) request.send_header('Content-type', 'application/xml') request.send_header('Content-Length', len(response)) request.end_headers() request.wfile.write(response.encode('ascii')) handler.add('POST', '/oss_fake_bucket4/large_file.bin?uploads', custom_method=method) def method(request): if request.headers['Content-Length'] != '1048576': sys.stderr.write('Did not get expected headers: %s\n' % str(request.headers)) request.send_response(400) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(200) request.send_header('ETag', '"first_etag"') request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/oss_fake_bucket4/large_file.bin?partNumber=1&uploadId=my_id', custom_method=method) with webserver.install_http_handler(handler): ret = gdal.VSIFWriteL(big_buffer, 1, size, f) assert ret == size handler = webserver.SequentialHandler() def method(request): if request.headers['Content-Length'] != '1': sys.stderr.write('Did not get expected headers: %s\n' % str(request.headers)) request.send_response(400) return request.send_response(200) request.send_header('ETag', '"second_etag"') request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/oss_fake_bucket4/large_file.bin?partNumber=2&uploadId=my_id', custom_method=method) def method(request): if request.headers['Content-Length'] != '186': sys.stderr.write('Did not get expected headers: %s\n' % str(request.headers)) request.send_response(400) request.send_header('Content-Length', 0) request.end_headers() return content = request.rfile.read(186).decode('ascii') if content != """<CompleteMultipartUpload> <Part> <PartNumber>1</PartNumber><ETag>"first_etag"</ETag></Part> <Part> <PartNumber>2</PartNumber><ETag>"second_etag"</ETag></Part> </CompleteMultipartUpload> """: sys.stderr.write('Did not get expected content: %s\n' % content) request.send_response(400) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(200) request.send_header('Content-Length', 0) request.end_headers() handler.add('POST', '/oss_fake_bucket4/large_file.bin?uploadId=my_id', custom_method=method) gdal.ErrorReset() with webserver.install_http_handler(handler): gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() == '' handler = webserver.SequentialHandler() handler.add('POST', '/oss_fake_bucket4/large_file_initiate_403_error.bin?uploads', 403) handler.add( 'POST', '/oss_fake_bucket4/large_file_initiate_empty_result.bin?uploads', 200) handler.add( 'POST', '/oss_fake_bucket4/large_file_initiate_invalid_xml_result.bin?uploads', 200, {}, 'foo') handler.add( 'POST', '/oss_fake_bucket4/large_file_initiate_no_uploadId.bin?uploads', 200, {}, '<foo/>') with webserver.install_http_handler(handler): for filename in [ '/vsioss/oss_fake_bucket4/large_file_initiate_403_error.bin', '/vsioss/oss_fake_bucket4/large_file_initiate_empty_result.bin', '/vsioss/oss_fake_bucket4/large_file_initiate_invalid_xml_result.bin', '/vsioss/oss_fake_bucket4/large_file_initiate_no_uploadId.bin' ]: with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB f = gdal.VSIFOpenL(filename, 'wb') assert f is not None with gdaltest.error_handler(): ret = gdal.VSIFWriteL(big_buffer, 1, size, f) assert ret == 0 gdal.ErrorReset() gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() == '' handler = webserver.SequentialHandler() handler.add( 'POST', '/oss_fake_bucket4/large_file_upload_part_403_error.bin?uploads', 200, {}, '<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>' ) handler.add( 'PUT', '/oss_fake_bucket4/large_file_upload_part_403_error.bin?partNumber=1&uploadId=my_id', 403) handler.add( 'DELETE', '/oss_fake_bucket4/large_file_upload_part_403_error.bin?uploadId=my_id', 204) handler.add( 'POST', '/oss_fake_bucket4/large_file_upload_part_no_etag.bin?uploads', 200, {}, '<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>' ) handler.add( 'PUT', '/oss_fake_bucket4/large_file_upload_part_no_etag.bin?partNumber=1&uploadId=my_id', 200) handler.add( 'DELETE', '/oss_fake_bucket4/large_file_upload_part_no_etag.bin?uploadId=my_id', 204) with webserver.install_http_handler(handler): for filename in [ '/vsioss/oss_fake_bucket4/large_file_upload_part_403_error.bin', '/vsioss/oss_fake_bucket4/large_file_upload_part_no_etag.bin' ]: with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB f = gdal.VSIFOpenL(filename, 'wb') assert f is not None, filename with gdaltest.error_handler(): ret = gdal.VSIFWriteL(big_buffer, 1, size, f) assert ret == 0, filename gdal.ErrorReset() gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() == '', filename # Simulate failure in AbortMultipart stage handler = webserver.SequentialHandler() handler.add( 'POST', '/oss_fake_bucket4/large_file_abortmultipart_403_error.bin?uploads', 200, {}, '<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>' ) handler.add( 'PUT', '/oss_fake_bucket4/large_file_abortmultipart_403_error.bin?partNumber=1&uploadId=my_id', 403) handler.add( 'DELETE', '/oss_fake_bucket4/large_file_abortmultipart_403_error.bin?uploadId=my_id', 403) filename = '/vsioss/oss_fake_bucket4/large_file_abortmultipart_403_error.bin' with webserver.install_http_handler(handler): with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB f = gdal.VSIFOpenL(filename, 'wb') assert f is not None, filename with gdaltest.error_handler(): ret = gdal.VSIFWriteL(big_buffer, 1, size, f) assert ret == 0, filename gdal.ErrorReset() with gdaltest.error_handler(): gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() != '', filename # Simulate failure in CompleteMultipartUpload stage handler = webserver.SequentialHandler() handler.add( 'POST', '/oss_fake_bucket4/large_file_completemultipart_403_error.bin?uploads', 200, {}, '<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult><UploadId>my_id</UploadId></InitiateMultipartUploadResult>' ) handler.add( 'PUT', '/oss_fake_bucket4/large_file_completemultipart_403_error.bin?partNumber=1&uploadId=my_id', 200, {'ETag': 'first_etag'}, '') handler.add( 'PUT', '/oss_fake_bucket4/large_file_completemultipart_403_error.bin?partNumber=2&uploadId=my_id', 200, {'ETag': 'second_etag'}, '') handler.add( 'POST', '/oss_fake_bucket4/large_file_completemultipart_403_error.bin?uploadId=my_id', 403) # handler.add('DELETE', '/oss_fake_bucket4/large_file_completemultipart_403_error.bin?uploadId=my_id', 204) filename = '/vsioss/oss_fake_bucket4/large_file_completemultipart_403_error.bin' with webserver.install_http_handler(handler): with gdaltest.config_option('VSIOSS_CHUNK_SIZE', '1'): # 1 MB f = gdal.VSIFOpenL(filename, 'wb') assert f is not None, filename ret = gdal.VSIFWriteL(big_buffer, 1, size, f) assert ret == size, filename gdal.ErrorReset() with gdaltest.error_handler(): gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() != '', filename
def test_visoss_4(): if gdaltest.webserver_port == 0: pytest.skip() with webserver.install_http_handler(webserver.SequentialHandler()): with gdaltest.error_handler(): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3', 'wb') assert f is None handler = webserver.SequentialHandler() handler.add('GET', '/oss_fake_bucket3/empty_file.bin', 200, {'Connection': 'close'}, 'foo') with webserver.install_http_handler(handler): assert gdal.VSIStatL( '/vsioss/oss_fake_bucket3/empty_file.bin').size == 3 # Empty file handler = webserver.SequentialHandler() def method(request): if request.headers['Content-Length'] != '0': sys.stderr.write('Did not get expected headers: %s\n' % str(request.headers)) request.send_response(400) return request.send_response(200) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/oss_fake_bucket3/empty_file.bin', custom_method=method) with webserver.install_http_handler(handler): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb') assert f is not None gdal.ErrorReset() gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() == '' handler = webserver.SequentialHandler() handler.add('GET', '/oss_fake_bucket3/empty_file.bin', 200, {'Connection': 'close'}, '') with webserver.install_http_handler(handler): assert gdal.VSIStatL( '/vsioss/oss_fake_bucket3/empty_file.bin').size == 0 # Invalid seek handler = webserver.SequentialHandler() with webserver.install_http_handler(handler): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb') assert f is not None with gdaltest.error_handler(): ret = gdal.VSIFSeekL(f, 1, 0) assert ret != 0 gdal.VSIFCloseL(f) # Invalid read handler = webserver.SequentialHandler() with webserver.install_http_handler(handler): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb') assert f is not None with gdaltest.error_handler(): ret = gdal.VSIFReadL(1, 1, f) assert not ret gdal.VSIFCloseL(f) # Error case handler = webserver.SequentialHandler() handler.add('PUT', '/oss_fake_bucket3/empty_file_error.bin', 403) with webserver.install_http_handler(handler): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file_error.bin', 'wb') assert f is not None gdal.ErrorReset() with gdaltest.error_handler(): gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() != '' # Nominal case with webserver.install_http_handler(webserver.SequentialHandler()): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/another_file.bin', 'wb') assert f is not None assert gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) == 0 assert gdal.VSIFSeekL(f, 0, 1) == 0 assert gdal.VSIFSeekL(f, 0, 2) == 0 assert gdal.VSIFWriteL('foo', 1, 3, f) == 3 assert gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) == 0 assert gdal.VSIFWriteL('bar', 1, 3, f) == 3 handler = webserver.SequentialHandler() def method(request): if request.headers['Content-Length'] != '6': sys.stderr.write('Did not get expected headers: %s\n' % str(request.headers)) request.send_response(400) request.send_header('Content-Length', 0) request.end_headers() return request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii')) content = request.rfile.read(6).decode('ascii') if content != 'foobar': sys.stderr.write('Did not get expected content: %s\n' % content) request.send_response(400) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(200) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/oss_fake_bucket3/another_file.bin', custom_method=method) gdal.ErrorReset() with webserver.install_http_handler(handler): gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() == ''
def vsis3_6(): if gdaltest.webserver_port == 0: return 'skip' gdal.SetConfigOption('VSIS3_CHUNK_SIZE', '1') # 1 MB f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket4/large_file.bin', 'wb') gdal.SetConfigOption('VSIS3_CHUNK_SIZE', None) if f is None: gdaltest.post_reason('fail') return 'fail' size = 1024 * 1024 + 1 ret = gdal.VSIFWriteL(''.join('a' for i in range(size)), 1, size, f) if ret != size: gdaltest.post_reason('fail') return 'fail' gdal.ErrorReset() gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() != '': gdaltest.post_reason('fail') return 'fail' for filename in [ '/vsis3/s3_fake_bucket4/large_file_initiate_403_error.bin', '/vsis3/s3_fake_bucket4/large_file_initiate_empty_result.bin', '/vsis3/s3_fake_bucket4/large_file_initiate_invalid_xml_result.bin', '/vsis3/s3_fake_bucket4/large_file_initiate_no_uploadId.bin' ]: gdal.SetConfigOption('VSIS3_CHUNK_SIZE', '1') # 1 MB f = gdal.VSIFOpenL(filename, 'wb') gdal.SetConfigOption('VSIS3_CHUNK_SIZE', None) if f is None: gdaltest.post_reason('fail') return 'fail' size = 1024 * 1024 + 1 with gdaltest.error_handler(): ret = gdal.VSIFWriteL(''.join('a' for i in range(size)), 1, size, f) if ret != 0: gdaltest.post_reason('fail') print(ret) return 'fail' gdal.ErrorReset() gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() != '': gdaltest.post_reason('fail') return 'fail' for filename in [ '/vsis3/s3_fake_bucket4/large_file_upload_part_403_error.bin', '/vsis3/s3_fake_bucket4/large_file_upload_part_no_etag.bin' ]: gdal.SetConfigOption('VSIS3_CHUNK_SIZE', '1') # 1 MB f = gdal.VSIFOpenL(filename, 'wb') gdal.SetConfigOption('VSIS3_CHUNK_SIZE', None) if f is None: gdaltest.post_reason('fail') return 'fail' size = 1024 * 1024 + 1 with gdaltest.error_handler(): ret = gdal.VSIFWriteL(''.join('a' for i in range(size)), 1, size, f) if ret != 0: gdaltest.post_reason('fail') print(ret) return 'fail' gdal.ErrorReset() gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() != '': gdaltest.post_reason('fail') return 'fail' return 'success'
def vsiaz_extra_1(): if not gdaltest.built_against_curl(): return 'skip' az_resource = gdal.GetConfigOption('AZ_RESOURCE') if az_resource is None: print('Missing AZ_RESOURCE for running gdaltest_list_extra') return 'skip' if az_resource.find('/') < 0: path = '/vsiaz/' + az_resource statres = gdal.VSIStatL(path) if statres is None or not stat.S_ISDIR(statres.mode): gdaltest.post_reason('fail') print('%s is not a valid bucket' % path) return 'fail' readdir = gdal.ReadDir(path) if readdir is None: gdaltest.post_reason('fail') print('ReadDir() should not return empty list') return 'fail' for filename in readdir: if filename != '.': subpath = path + '/' + filename if gdal.VSIStatL(subpath) is None: gdaltest.post_reason('fail') print('Stat(%s) should not return an error' % subpath) return 'fail' unique_id = 'vsiaz_test' subpath = path + '/' + unique_id ret = gdal.Mkdir(subpath, 0) if ret < 0: gdaltest.post_reason('fail') print('Mkdir(%s) should not return an error' % subpath) return 'fail' readdir = gdal.ReadDir(path) if unique_id not in readdir: gdaltest.post_reason('fail') print('ReadDir(%s) should contain %s' % (path, unique_id)) print(readdir) return 'fail' ret = gdal.Mkdir(subpath, 0) if ret == 0: gdaltest.post_reason('fail') print('Mkdir(%s) repeated should return an error' % subpath) return 'fail' ret = gdal.Rmdir(subpath) if ret < 0: gdaltest.post_reason('fail') print('Rmdir(%s) should not return an error' % subpath) return 'fail' readdir = gdal.ReadDir(path) if unique_id in readdir: gdaltest.post_reason('fail') print('ReadDir(%s) should not contain %s' % (path, unique_id)) print(readdir) return 'fail' ret = gdal.Rmdir(subpath) if ret == 0: gdaltest.post_reason('fail') print('Rmdir(%s) repeated should return an error' % subpath) return 'fail' ret = gdal.Mkdir(subpath, 0) if ret < 0: gdaltest.post_reason('fail') print('Mkdir(%s) should not return an error' % subpath) return 'fail' f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) ret = gdal.Rmdir(subpath) if ret == 0: gdaltest.post_reason('fail') print('Rmdir(%s) on non empty directory should return an error' % subpath) return 'fail' f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 5, f).decode('utf-8') if data != 'hello': gdaltest.post_reason('fail') print(data) return 'fail' gdal.VSIFCloseL(f) ret = gdal.Unlink(subpath + '/test.txt') if ret < 0: gdaltest.post_reason('fail') print('Unlink(%s) should not return an error' % (subpath + '/test.txt')) return 'fail' ret = gdal.Rmdir(subpath) if ret < 0: gdaltest.post_reason('fail') print('Rmdir(%s) should not return an error' % subpath) return 'fail' return 'success' f = open_for_read('/vsiaz/' + az_resource) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' # Same with /vsiaz_streaming/ f = open_for_read('/vsiaz_streaming/' + az_resource) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' if False: # pylint: disable=using-constant-test # we actually try to read at read() time and bSetError = false # Invalid bucket : "The specified bucket does not exist" gdal.ErrorReset() f = open_for_read('/vsiaz/not_existing_bucket/foo') with gdaltest.error_handler(): gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if gdal.VSIGetLastErrorMsg() == '': gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' # Invalid resource gdal.ErrorReset() f = open_for_read('/vsiaz_streaming/' + az_resource + '/invalid_resource.baz') if f is not None: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' # Test GetSignedURL() signed_url = gdal.GetSignedURL('/vsiaz/' + az_resource) f = open_for_read('/vsicurl_streaming/' + signed_url) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' return 'success'
def _VSIFPrintfL(f, s): gdal.VSIFWriteL(s, 1, len(s), f)
def ogr_tiger_4(): if ogrtest.tiger_ds is None: return 'skip' # load all the files into memory. for filename in gdal.ReadDir('tmp/cache/TGR01001'): if filename.startswith('.'): continue data = open('tmp/cache/TGR01001/' + filename, 'r').read() f = gdal.VSIFOpenL('/vsimem/tigertest/' + filename, 'wb') gdal.VSIFWriteL(data, 1, len(data), f) gdal.VSIFCloseL(f) # Try reading. ogrtest.tiger_ds = ogr.Open('/vsimem/tigertest/TGR01001.RT1') if ogrtest.tiger_ds is None: gdaltest.post_reason('fail to open.') return 'fail' ogrtest.tiger_ds = None # also test opening with a filename (#4443) ogrtest.tiger_ds = ogr.Open('tmp/cache/TGR01001/TGR01001.RT1') if ogrtest.tiger_ds is None: gdaltest.post_reason('fail') return 'fail' # Check a few features. cc_layer = ogrtest.tiger_ds.GetLayerByName('CompleteChain') if cc_layer.GetFeatureCount() != 19289: gdaltest.post_reason('wrong cc feature count') return 'fail' feat = cc_layer.GetNextFeature() feat = cc_layer.GetNextFeature() feat = cc_layer.GetNextFeature() if feat.TLID != 2833200 or feat.FRIADDL is not None or feat.BLOCKL != 5000: gdaltest.post_reason('wrong attribute on cc feature.') return 'fail' if ogrtest.check_feature_geometry( feat, 'LINESTRING (-86.4402 32.504137,-86.440313 32.504009,-86.440434 32.503884,-86.440491 32.503805,-86.44053 32.503757,-86.440578 32.503641,-86.440593 32.503515,-86.440588 32.503252,-86.440596 32.50298)', max_error=0.000001) != 0: return 'fail' feat = ogrtest.tiger_ds.GetLayerByName('TLIDRange').GetNextFeature() if feat.MODULE != 'TGR01001' or feat.TLMINID != 2822718: gdaltest.post_reason('got wrong TLIDRange attributes') return 'fail' # Try to recover memory from /vsimem. for filename in gdal.ReadDir('tmp/cache/TGR01001'): if filename.startswith('.'): continue gdal.Unlink('/vsimem/tigertest/' + filename) return 'success'
def write(self, data): gdal.VSIFWriteL(data, 1, len(data), self.f)
def vsiswift_extra_1(): if not gdaltest.built_against_curl(): return 'skip' swift_resource = gdal.GetConfigOption('SWIFT_RESOURCE') if swift_resource is None: print('Missing SWIFT_RESOURCE for running gdaltest_list_extra') return 'skip' if swift_resource.find('/') < 0: path = '/vsiswift/' + swift_resource statres = gdal.VSIStatL(path) if statres is None or not stat.S_ISDIR(statres.mode): gdaltest.post_reason('fail') print('%s is not a valid bucket' % path) return 'fail' readdir = gdal.ReadDir(path) if readdir is None: gdaltest.post_reason('fail') print('ReadDir() should not return empty list') return 'fail' for filename in readdir: if filename != '.': subpath = path + '/' + filename if gdal.VSIStatL(subpath) is None: gdaltest.post_reason('fail') print('Stat(%s) should not return an error' % subpath) return 'fail' unique_id = 'vsiswift_test' subpath = path + '/' + unique_id ret = gdal.Mkdir(subpath, 0) if ret < 0: gdaltest.post_reason('fail') print('Mkdir(%s) should not return an error' % subpath) return 'fail' readdir = gdal.ReadDir(path) if unique_id not in readdir: gdaltest.post_reason('fail') print('ReadDir(%s) should contain %s' % (path, unique_id)) print(readdir) return 'fail' ret = gdal.Mkdir(subpath, 0) if ret == 0: gdaltest.post_reason('fail') print('Mkdir(%s) repeated should return an error' % subpath) return 'fail' ret = gdal.Rmdir(subpath) if ret < 0: gdaltest.post_reason('fail') print('Rmdir(%s) should not return an error' % subpath) return 'fail' readdir = gdal.ReadDir(path) if unique_id in readdir: gdaltest.post_reason('fail') print('ReadDir(%s) should not contain %s' % (path, unique_id)) print(readdir) return 'fail' ret = gdal.Rmdir(subpath) if ret == 0: gdaltest.post_reason('fail') print('Rmdir(%s) repeated should return an error' % subpath) return 'fail' ret = gdal.Mkdir(subpath, 0) if ret < 0: gdaltest.post_reason('fail') print('Mkdir(%s) should not return an error' % subpath) return 'fail' f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) ret = gdal.Rmdir(subpath) if ret == 0: gdaltest.post_reason('fail') print('Rmdir(%s) on non empty directory should return an error' % subpath) return 'fail' f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 5, f).decode('utf-8') if data != 'hello': gdaltest.post_reason('fail') print(data) return 'fail' gdal.VSIFCloseL(f) ret = gdal.Unlink(subpath + '/test.txt') if ret < 0: gdaltest.post_reason('fail') print('Unlink(%s) should not return an error' % (subpath + '/test.txt')) return 'fail' ret = gdal.Rmdir(subpath) if ret < 0: gdaltest.post_reason('fail') print('Rmdir(%s) should not return an error' % subpath) return 'fail' return 'success' f = open_for_read('/vsiswift/' + swift_resource) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' # Same with /vsiswift_streaming/ f = open_for_read('/vsiswift_streaming/' + swift_resource) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' # Invalid resource gdal.ErrorReset() f = open_for_read('/vsiswift_streaming/' + swift_resource + '/invalid_resource.baz') if f is not None: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' return 'success'
def vsifile_generic(filename): start_time = time.time() fp = gdal.VSIFOpenL(filename, 'wb+') if fp is None: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFWriteL('0123456789', 1, 10, fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTruncateL(fp, 20) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTruncateL(fp, 5) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFSeekL(fp, 0, 2) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 5: gdaltest.post_reason('failure') return 'fail' gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 7: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if abs(start_time - statBuf.mtime) > 2: gdaltest.post_reason('failure') print(statBuf.mtime) return 'fail' fp = gdal.VSIFOpenL(filename, 'rb') buf = gdal.VSIFReadL(1, 7, fp) if gdal.VSIFWriteL('a', 1, 1, fp) != 0: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFTruncateL(fp, 0) == 0: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(fp) if buf.decode('ascii') != '01234XX': gdaltest.post_reason('failure') print(buf.decode('ascii')) return 'fail' # Test append mode on existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 9: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if gdal.Unlink(filename) != 0: gdaltest.post_reason('failure') return 'fail' statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG) if statBuf is not None: gdaltest.post_reason('failure') return 'fail' # Test append mode on non existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 2: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if gdal.Unlink(filename) != 0: gdaltest.post_reason('failure') return 'fail' return 'success'
def gdal_pansharpen(argv): argv = gdal.GeneralCmdLineProcessor(argv) if argv is None: return -1 pan_name = None last_name = None spectral_ds = [] spectral_bands = [] out_name = None bands = [] weights = [] frmt = None creation_options = [] callback = gdal.TermProgress_nocb resampling = None spat_adjust = None verbose_vrt = False num_threads = None bitdepth = None nodata = None i = 1 argc = len(argv) while i < argc: if (argv[i] == '-of' or argv[i] == '-f') and i < len(argv) - 1: frmt = argv[i + 1] i = i + 1 elif argv[i] == '-r' and i < len(argv) - 1: resampling = argv[i + 1] i = i + 1 elif argv[i] == '-spat_adjust' and i < len(argv) - 1: spat_adjust = argv[i + 1] i = i + 1 elif argv[i] == '-b' and i < len(argv) - 1: bands.append(int(argv[i + 1])) i = i + 1 elif argv[i] == '-w' and i < len(argv) - 1: weights.append(float(argv[i + 1])) i = i + 1 elif argv[i] == '-co' and i < len(argv) - 1: creation_options.append(argv[i + 1]) i = i + 1 elif argv[i] == '-threads' and i < len(argv) - 1: num_threads = argv[i + 1] i = i + 1 elif argv[i] == '-bitdepth' and i < len(argv) - 1: bitdepth = argv[i + 1] i = i + 1 elif argv[i] == '-nodata' and i < len(argv) - 1: nodata = argv[i + 1] i = i + 1 elif argv[i] == '-q': callback = None elif argv[i] == '-verbose_vrt': verbose_vrt = True elif argv[i][0] == '-': sys.stderr.write('Unrecognized option : %s\n' % argv[i]) return Usage() elif pan_name is None: pan_name = argv[i] pan_ds = gdal.Open(pan_name) if pan_ds is None: return 1 else: if last_name is not None: pos = last_name.find(',band=') if pos > 0: spectral_name = last_name[0:pos] ds = gdal.Open(spectral_name) if ds is None: return 1 band_num = int(last_name[pos + len(',band='):]) band = ds.GetRasterBand(band_num) spectral_ds.append(ds) spectral_bands.append(band) else: spectral_name = last_name ds = gdal.Open(spectral_name) if ds is None: return 1 for j in range(ds.RasterCount): spectral_ds.append(ds) spectral_bands.append(ds.GetRasterBand(j + 1)) last_name = argv[i] i = i + 1 if pan_name is None or not spectral_bands: return Usage() out_name = last_name if frmt is None: frmt = GetOutputDriverFor(out_name) if not bands: bands = [j + 1 for j in range(len(spectral_bands))] else: for band in bands: if band < 0 or band > len(spectral_bands): print('Invalid band number in -b: %d' % band) return 1 if weights and len(weights) != len(spectral_bands): print('There must be as many -w values specified as input spectral bands') return 1 vrt_xml = """<VRTDataset subClass="VRTPansharpenedDataset">\n""" if bands != [j + 1 for j in range(len(spectral_bands))]: for i, band in enumerate(bands): sband = spectral_bands[band - 1] datatype = gdal.GetDataTypeName(sband.DataType) colorname = gdal.GetColorInterpretationName(sband.GetColorInterpretation()) vrt_xml += """ <VRTRasterBand dataType="%s" band="%d" subClass="VRTPansharpenedRasterBand"> <ColorInterp>%s</ColorInterp> </VRTRasterBand>\n""" % (datatype, i + 1, colorname) vrt_xml += """ <PansharpeningOptions>\n""" if weights: vrt_xml += """ <AlgorithmOptions>\n""" vrt_xml += """ <Weights>""" for i, weight in enumerate(weights): if i > 0: vrt_xml += "," vrt_xml += "%.16g" % weight vrt_xml += "</Weights>\n" vrt_xml += """ </AlgorithmOptions>\n""" if resampling is not None: vrt_xml += ' <Resampling>%s</Resampling>\n' % resampling if num_threads is not None: vrt_xml += ' <NumThreads>%s</NumThreads>\n' % num_threads if bitdepth is not None: vrt_xml += ' <BitDepth>%s</BitDepth>\n' % bitdepth if nodata is not None: vrt_xml += ' <NoData>%s</NoData>\n' % nodata if spat_adjust is not None: vrt_xml += ' <SpatialExtentAdjustment>%s</SpatialExtentAdjustment>\n' % spat_adjust pan_relative = '0' if frmt.upper() == 'VRT': if not os.path.isabs(pan_name): pan_relative = '1' pan_name = os.path.relpath(pan_name, os.path.dirname(out_name)) vrt_xml += """ <PanchroBand> <SourceFilename relativeToVRT="%s">%s</SourceFilename> <SourceBand>1</SourceBand> </PanchroBand>\n""" % (pan_relative, pan_name) for i, sband in enumerate(spectral_bands): dstband = '' for j, band in enumerate(bands): if i + 1 == band: dstband = ' dstBand="%d"' % (j + 1) break ms_relative = '0' ms_name = spectral_ds[i].GetDescription() if frmt.upper() == 'VRT': if not os.path.isabs(ms_name): ms_relative = '1' ms_name = os.path.relpath(ms_name, os.path.dirname(out_name)) vrt_xml += """ <SpectralBand%s> <SourceFilename relativeToVRT="%s">%s</SourceFilename> <SourceBand>%d</SourceBand> </SpectralBand>\n""" % (dstband, ms_relative, ms_name, sband.GetBand()) vrt_xml += """ </PansharpeningOptions>\n""" vrt_xml += """</VRTDataset>\n""" if frmt.upper() == 'VRT': f = gdal.VSIFOpenL(out_name, 'wb') if f is None: print('Cannot create %s' % out_name) return 1 gdal.VSIFWriteL(vrt_xml, 1, len(vrt_xml), f) gdal.VSIFCloseL(f) if verbose_vrt: vrt_ds = gdal.Open(out_name, gdal.GA_Update) vrt_ds.SetMetadata(vrt_ds.GetMetadata()) else: vrt_ds = gdal.Open(out_name) if vrt_ds is None: return 1 return 0 vrt_ds = gdal.Open(vrt_xml) out_ds = gdal.GetDriverByName(frmt).CreateCopy(out_name, vrt_ds, 0, creation_options, callback=callback) if out_ds is None: return 1 return 0
def vsizip_2(): fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/foo.bar", "wb") if fmain is None: gdaltest.post_reason('fail 1') return 'fail' gdal.VSIFWriteL("12345", 1, 5, fmain) gdal.VSIFCloseL(fmain) content = gdal.ReadDir("/vsizip/vsimem/test2.zip") if content != ['foo.bar']: gdaltest.post_reason('bad content 1') print(content) return 'fail' # Now append a second file fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/bar.baz", "wb") if fmain is None: gdaltest.post_reason('fail 2') return 'fail' gdal.VSIFWriteL("67890", 1, 5, fmain) gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') content = gdal.ReadDir("/vsizip/vsimem/test2.zip") gdal.PopErrorHandler() if gdal.GetLastErrorMsg() != 'Cannot read a zip file being written': gdaltest.post_reason('expected error') print(gdal.GetLastErrorMsg()) return 'fail' if content != None: gdaltest.post_reason('bad content 2') print(content) return 'fail' gdal.VSIFCloseL(fmain) content = gdal.ReadDir("/vsizip/vsimem/test2.zip") if content != ['foo.bar', 'bar.baz']: gdaltest.post_reason('bad content 3') print(content) return 'fail' fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/foo.bar", "rb") if fmain is None: gdaltest.post_reason('fail 3') return 'fail' data = gdal.VSIFReadL(1, 5, fmain) gdal.VSIFCloseL(fmain) if data.decode('ASCII') != '12345': print(data) return 'fail' fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/bar.baz", "rb") if fmain is None: gdaltest.post_reason('fail 4') return 'fail' data = gdal.VSIFReadL(1, 5, fmain) gdal.VSIFCloseL(fmain) if data.decode('ASCII') != '67890': print(data) return 'fail' gdal.Unlink("/vsimem/test2.zip") return 'success'
def vsifile_generic(filename): start_time = time.time() fp = gdal.VSIFOpenL(filename, 'wb+') assert fp is not None assert gdal.VSIFWriteL('0123456789', 1, 10, fp) == 10 assert gdal.VSIFFlushL(fp) == 0 assert gdal.VSIFTruncateL(fp, 20) == 0 assert gdal.VSIFTellL(fp) == 10 assert gdal.VSIFTruncateL(fp, 5) == 0 assert gdal.VSIFTellL(fp) == 10 assert gdal.VSIFSeekL(fp, 0, 2) == 0 assert gdal.VSIFTellL(fp) == 5 gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) assert statBuf.size == 7 assert start_time == pytest.approx(statBuf.mtime, abs=2) fp = gdal.VSIFOpenL(filename, 'rb') buf = gdal.VSIFReadL(1, 7, fp) assert gdal.VSIFWriteL('a', 1, 1, fp) == 0 assert gdal.VSIFTruncateL(fp, 0) != 0 gdal.VSIFCloseL(fp) assert buf.decode('ascii') == '01234XX' # Test append mode on existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) assert statBuf.size == 9 assert gdal.Unlink(filename) == 0 statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG) assert statBuf is None # Test append mode on non existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) assert statBuf.size == 2 assert gdal.Unlink(filename) == 0
def test_vsiaz_fake_write(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() # Test creation of BlockBob f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() def method(request): h = request.headers if 'Authorization' not in h or \ h['Authorization'] != 'SharedKey myaccount:AigkrY7q66WCrx3JRKBte56k7kxV2cxB/ZyGNubxk5I=' or \ 'Expect' not in h or h['Expect'] != '100-continue' or \ 'Content-Length' not in h or h['Content-Length'] != '40000' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'BlockBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii')) content = request.rfile.read(40000).decode('ascii') if len(content) != 40000: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) with webserver.install_http_handler(handler): ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f) ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f) if ret != 40000: gdal.VSIFCloseL(f) pytest.fail(ret) gdal.VSIFCloseL(f) # Simulate illegal read f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None with gdaltest.error_handler(): ret = gdal.VSIFReadL(1, 1, f) assert not ret gdal.VSIFCloseL(f) # Simulate illegal seek f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None with gdaltest.error_handler(): ret = gdal.VSIFSeekL(f, 1, 0) assert ret != 0 gdal.VSIFCloseL(f) # Simulate failure when putting BlockBob f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) if gdal.VSIFSeekL(f, 0, 0) != 0: gdal.VSIFCloseL(f) pytest.fail() gdal.VSIFWriteL('x' * 35000, 1, 35000, f) if gdal.VSIFTellL(f) != 35000: gdal.VSIFCloseL(f) pytest.fail() if gdal.VSIFSeekL(f, 35000, 0) != 0: gdal.VSIFCloseL(f) pytest.fail() if gdal.VSIFSeekL(f, 0, 1) != 0: gdal.VSIFCloseL(f) pytest.fail() if gdal.VSIFSeekL(f, 0, 2) != 0: gdal.VSIFCloseL(f) pytest.fail() if gdal.VSIFEofL(f) != 0: gdal.VSIFCloseL(f) pytest.fail() with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.VSIFCloseL(f) if ret == 0: gdal.VSIFCloseL(f) pytest.fail(ret) # Simulate creation of BlockBob over an existing blob of incompatible type f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 409) handler.add('DELETE', '/azure/blob/myaccount/test_copy/file.bin', 202) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201) with webserver.install_http_handler(handler): gdal.VSIFCloseL(f) # Test creation of AppendBlob gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10') f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None) assert f is not None handler = webserver.SequentialHandler() def method(request): h = request.headers if 'Authorization' not in h or \ h['Authorization'] != 'SharedKey myaccount:KimVui3ptY9D5ftLlsI7CNOgK36CNAEzsXqcuHskdEY=' or \ 'Content-Length' not in h or h['Content-Length'] != '0' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) def method(request): h = request.headers if 'Content-Length' not in h or h['Content-Length'] != '10' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' content = request.rfile.read(10).decode('ascii') if content != '0123456789': sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', custom_method=method) def method(request): h = request.headers if 'Content-Length' not in h or h['Content-Length'] != '6' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' content = request.rfile.read(6).decode('ascii') if content != 'abcdef': sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', custom_method=method) with webserver.install_http_handler(handler): ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) if ret != 16: gdal.VSIFCloseL(f) pytest.fail(ret) gdal.VSIFCloseL(f) # Test failed creation of AppendBlob gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10') f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None) assert f is not None handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) if ret != 0: gdal.VSIFCloseL(f) pytest.fail(ret) gdal.VSIFCloseL(f) # Test failed writing of a block of an AppendBlob gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10') f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None) assert f is not None handler = webserver.SequentialHandler() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 403) with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) if ret != 0: gdal.VSIFCloseL(f) pytest.fail(ret) gdal.VSIFCloseL(f)
def extract_tile(ds, src_band_nbr, tile_x, tile_y, jpg_filename): block_offset = ds.GetRasterBand(src_band_nbr).GetMetadataItem( 'BLOCK_OFFSET_%d_%d' % (tile_x, tile_y), 'TIFF') block_size = ds.GetRasterBand(src_band_nbr).GetMetadataItem( 'BLOCK_SIZE_%d_%d' % (tile_x, tile_y), 'TIFF') if block_offset is None or block_size is None: print('ERROR: Cannot find block (%d,%d)' % (tile_x, tile_y)) return 1 jpegtables = ds.GetRasterBand(src_band_nbr).GetMetadataItem( 'JPEGTABLES', 'TIFF') if jpegtables is not None: if (len(jpegtables) % 2 ) != 0 or jpegtables[0:4] != 'FFD8' or jpegtables[-2:] != 'D9': print('ERROR: Invalid JPEG tables') print(jpegtables) return 1 # Remove final D9 jpegtables = jpegtables[0:-2] tiff_f = gdal.VSIFOpenL(ds.GetDescription(), 'rb') if tiff_f is None: print('ERROR: Cannot reopen %s' % ds.GetDescription()) return 1 out_f = gdal.VSIFOpenL(jpg_filename, 'wb') if out_f is None: print('ERROR: Cannot create %s' % jpg_filename) gdal.VSIFCloseL(tiff_f) return 1 # Write JPEG tables if jpegtables is not None: for i in range(int(len(jpegtables) / 2)): c1 = ord(jpegtables[2 * i]) c2 = ord(jpegtables[2 * i + 1]) if c1 >= ord('0') and c1 <= ord('9'): val = c1 - ord('0') else: val = (c1 - ord('A')) + 10 val = val * 16 if c2 >= ord('0') and c2 <= ord('9'): val = val + (c2 - ord('0')) else: val = val + (c2 - ord('A')) + 10 gdal.VSIFWriteL(chr(val), 1, 1, out_f) else: gdal.VSIFWriteL(chr(0xFF), 1, 1, out_f) gdal.VSIFWriteL(chr(0xD8), 1, 1, out_f) # Write Adobe APP14 marker if necessary interleave = ds.GetMetadataItem('INTERLEAVE', 'IMAGE_STRUCTURE') photometric = ds.GetMetadataItem('COMPRESSION', 'IMAGE_STRUCTURE') if interleave == 'PIXEL' and photometric == 'JPEG' and ds.RasterCount == 3: adobe_app14 = [ 0xFF, 0xEE, 0x00, 0x0E, 0x41, 0x64, 0x6F, 0x62, 0x65, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00 ] for c in adobe_app14: gdal.VSIFWriteL(chr(c), 1, 1, out_f) # Write JPEG codestream # skip leading 0xFF 0xD8 gdal.VSIFSeekL(tiff_f, int(block_offset) + 2, 0) data = gdal.VSIFReadL(1, int(block_size) - 2, tiff_f) gdal.VSIFCloseL(tiff_f) gdal.VSIFWriteL(data, 1, len(data), out_f) gdal.VSIFCloseL(out_f) aux_xml_filename = '%s.aux.xml' % jpg_filename gt = ds.GetGeoTransform() srs = ds.GetProjectionRef() if srs is not None and srs != '': sub_gt = [gt[i] for i in range(6)] (blockxsize, blockysize) = ds.GetRasterBand(1).GetBlockSize() sub_gt[0] = gt[0] + tile_x * blockxsize * gt[1] sub_gt[3] = gt[3] + tile_y * blockysize * gt[5] out_f = gdal.VSIFOpenL(aux_xml_filename, 'wb') if out_f is None: print('ERROR: Cannot create %s' % aux_xml_filename) return 1 content = """<PAMDataset> <SRS>%s</SRS> <GeoTransform>%.18g,%.18g,%.18g,%.18g,%.18g,%.18g</GeoTransform> </PAMDataset> """ % (srs, sub_gt[0], sub_gt[1], sub_gt[2], sub_gt[3], sub_gt[4], sub_gt[5]) gdal.VSIFWriteL(content, 1, len(content), out_f) gdal.VSIFCloseL(out_f) else: gdal.Unlink('%s.aux.xml' % jpg_filename) return 0
def ecrgtoc_4(): toc_xml = """<Table_of_Contents> <file_header file_status="new"> <file_name>TOC.xml</file_name> </file_header> <product product_title="ProductTitle"> <disc id="DiscId"> <frame_list number_of_frames="2"> <scale size="1:500 K"> <frame name="000000009s0013.lf2"> <frame_path>clfc\\2</frame_path> <frame_version>001</frame_version> <frame_chart_type>lf</frame_chart_type> <frame_zone>2</frame_zone> </frame> </scale> <scale size="1:1000 K"> <frame name="000000009s0013.lf2"> <frame_path>clfc\\2</frame_path> <frame_version>001</frame_version> <frame_chart_type>lf</frame_chart_type> <frame_zone>2</frame_zone> </frame> </scale> </frame_list> </disc> <disc id="DiscId2"> <frame_list number_of_frames="1"> <scale size="1:500 K"> <frame name="000000009t0013.lf2"> <frame_path>clfc\\2</frame_path> <frame_version>001</frame_version> <frame_chart_type>lf</frame_chart_type> <frame_zone>2</frame_zone> </frame> </scale> </frame_list> </disc> </product> <extension_list> <extension code="LF"> <chart_code>LF</chart_code> <chart_type>1:500 K (LFC Day)</chart_type> <chart_scale>1:500 K</chart_scale> <chart_description>LFC Day</chart_description> </extension> </extension_list> </Table_of_Contents>""" f = gdal.VSIFOpenL('/vsimem/TOC.xml', 'wb') gdal.VSIFWriteL(toc_xml, 1, len(toc_xml), f) gdal.VSIFCloseL(f) ds = gdal.Open('/vsimem/TOC.xml') if ds is None: return 'fail' if ds.RasterCount != 0: gdaltest.post_reason('bad raster count') return 'fail' expected_gt = (-85.43147208121826, 0.00059486040609137061, 0.0, 37.241379310344833, 0.0, -0.00044985604606525913) gt = ds.GetGeoTransform() for i in range(6): if abs(gt[i] - expected_gt[i]) > 1e-10: gdaltest.post_reason('did not get expected geotransform') print(gt) return 'fail' wkt = ds.GetProjectionRef() if wkt.find('WGS 84') == -1: gdaltest.post_reason('did not get expected SRS') print(wkt) return 'fail' filelist = ds.GetFileList() if len(filelist) != 4: gdaltest.post_reason('did not get expected filelist') print(filelist) return 'fail' subdatasets = ds.GetMetadata('SUBDATASETS') if len(subdatasets) != 6: gdaltest.post_reason('did not get expected subdatasets') print(filelist) return 'fail' ds = None ds = gdal.Open( 'ECRG_TOC_ENTRY:ProductTitle:DiscId:1_500_K:/vsimem/TOC.xml') if ds is None: gdaltest.post_reason('did not get subdataset') return 'fail' ds = None ds = gdal.Open( 'ECRG_TOC_ENTRY:ProductTitle:DiscId:1_1000_K:/vsimem/TOC.xml') if ds is None: gdaltest.post_reason('did not get subdataset') return 'fail' ds = None ds = gdal.Open( 'ECRG_TOC_ENTRY:ProductTitle:DiscId2:1_500_K:/vsimem/TOC.xml') if ds is None: gdaltest.post_reason('did not get subdataset') return 'fail' ds = None gdal.PushErrorHandler('CPLQuietErrorHandler') ds = gdal.Open('ECRG_TOC_ENTRY:ProductTitle:DiscId:/vsimem/TOC.xml') gdal.PopErrorHandler() if ds is not None: gdaltest.post_reason('should not have got subdataset') return 'fail' gdal.Unlink('/vsimem/TOC.xml') return 'success'
def vsis3_4(): if gdaltest.webserver_port == 0: return 'skip' with gdaltest.error_handler(): f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3', 'wb') if f is not None: gdaltest.post_reason('fail') return 'fail' if gdal.VSIStatL('/vsis3/s3_fake_bucket3/empty_file.bin').size != 3: gdaltest.post_reason('fail') return 'fail' # Empty file f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' gdal.ErrorReset() gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() != '': gdaltest.post_reason('fail') return 'fail' if gdal.VSIStatL('/vsis3/s3_fake_bucket3/empty_file.bin').size != 0: gdaltest.post_reason('fail') return 'fail' # Invalid seek f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): ret = gdal.VSIFSeekL(f, 1, 0) if ret == 0: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) # Invalid read f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): ret = gdal.VSIFReadL(1, 1, f) if len(ret) != 0: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) # Error case f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file_error.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() == '': gdaltest.post_reason('fail') return 'fail' # Nominal case f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/another_file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) != 0: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFSeekL(f, 0, 1) != 0: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFSeekL(f, 0, 2) != 0: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFWriteL('foo', 1, 3, f) != 3: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFWriteL('bar', 1, 3, f) != 3: gdaltest.post_reason('fail') return 'fail' gdal.ErrorReset() gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() != '': gdaltest.post_reason('fail') return 'fail' # Redirect case f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/redirect', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFWriteL('foobar', 1, 6, f) != 6: gdaltest.post_reason('fail') return 'fail' gdal.ErrorReset() gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() != '': gdaltest.post_reason('fail') return 'fail' return 'success'
def ogr_osm_10(): if ogrtest.osm_drv is None: return 'skip' # A file that does not exist. ds = ogr.Open('/nonexistent/foo.osm') if ds is not None: gdaltest.post_reason('fail') return 'fail' # Empty .osm file f = gdal.VSIFOpenL('/vsimem/foo.osm', 'wb') gdal.VSIFCloseL(f) ds = ogr.Open('/vsimem/foo.osm') if ds is not None: gdaltest.post_reason('fail') return 'fail' gdal.Unlink('/vsimem/foo.osm') # Empty .pbf file f = gdal.VSIFOpenL('/vsimem/foo.pbf', 'wb') gdal.VSIFCloseL(f) ds = ogr.Open('/vsimem/foo.pbf') if ds is not None: gdaltest.post_reason('fail') return 'fail' gdal.Unlink('/vsimem/foo.pbf') if ogrtest.osm_drv_parse_osm: # Invalid .osm file f = gdal.VSIFOpenL('/vsimem/foo.osm', 'wb') data = "<osm>" gdal.VSIFWriteL(data, 1, len(data), f) gdal.VSIFCloseL(f) ds = ogr.Open('/vsimem/foo.osm') lyr = ds.GetLayer(0) gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') feat = lyr.GetNextFeature() gdal.PopErrorHandler() if gdal.GetLastErrorMsg() == '': gdaltest.post_reason('fail') return 'fail' ds = None gdal.Unlink('/vsimem/foo.osm') # Invalid .pbf file f = gdal.VSIFOpenL('/vsimem/foo.pbf', 'wb') data = "OSMHeader\n" gdal.VSIFWriteL(data, 1, len(data), f) gdal.VSIFCloseL(f) ds = ogr.Open('/vsimem/foo.pbf') lyr = ds.GetLayer(0) gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') feat = lyr.GetNextFeature() gdal.PopErrorHandler() if gdal.GetLastErrorMsg() == '': gdaltest.post_reason('fail') return 'fail' ds = None gdal.Unlink('/vsimem/foo.pbf') # Test million laugh pattern if ogrtest.osm_drv_parse_osm: ds = ogr.Open('data/billionlaugh.osm') lyr = ds.GetLayer(0) gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') feat = lyr.GetNextFeature() gdal.PopErrorHandler() if feat is not None or gdal.GetLastErrorMsg() == '': gdaltest.post_reason('fail') return 'fail' return 'success'
def test_ogr_osm_10(): if ogrtest.osm_drv is None: pytest.skip() # A file that does not exist. ds = ogr.Open('/nonexistent/foo.osm') assert ds is None # Empty .osm file f = gdal.VSIFOpenL('/vsimem/foo.osm', 'wb') gdal.VSIFCloseL(f) ds = ogr.Open('/vsimem/foo.osm') assert ds is None gdal.Unlink('/vsimem/foo.osm') # Empty .pbf file f = gdal.VSIFOpenL('/vsimem/foo.pbf', 'wb') gdal.VSIFCloseL(f) ds = ogr.Open('/vsimem/foo.pbf') assert ds is None gdal.Unlink('/vsimem/foo.pbf') if ogrtest.osm_drv_parse_osm: # Invalid .osm file f = gdal.VSIFOpenL('/vsimem/foo.osm', 'wb') data = "<osm>" gdal.VSIFWriteL(data, 1, len(data), f) gdal.VSIFCloseL(f) ds = ogr.Open('/vsimem/foo.osm') lyr = ds.GetLayer(0) gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') feat = lyr.GetNextFeature() gdal.PopErrorHandler() assert gdal.GetLastErrorMsg() != '' ds = None gdal.Unlink('/vsimem/foo.osm') # Invalid .pbf file f = gdal.VSIFOpenL('/vsimem/foo.pbf', 'wb') data = "OSMHeader\n" gdal.VSIFWriteL(data, 1, len(data), f) gdal.VSIFCloseL(f) ds = ogr.Open('/vsimem/foo.pbf') lyr = ds.GetLayer(0) gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') feat = lyr.GetNextFeature() gdal.PopErrorHandler() assert gdal.GetLastErrorMsg() != '' ds = None gdal.Unlink('/vsimem/foo.pbf') # Test million laugh pattern if ogrtest.osm_drv_parse_osm: ds = ogr.Open('data/osm/billionlaugh.osm') lyr = ds.GetLayer(0) gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') feat = lyr.GetNextFeature() gdal.PopErrorHandler() assert feat is None and gdal.GetLastErrorMsg() != ''