def test_vsistdin_5(): f = open('tmp/test_vsistdin_5.bin', 'wb') f.write(b'0123456789' * (1024 * 1024)) f.close() with gdaltest.config_options({ 'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin', 'CPL_VSISTDIN_RESET_POSITION': 'YES', 'CPL_VSISTDIN_FILE_CLOSE': 'YES' }): f = gdal.VSIFOpenL('/vsistdin?buffer_limit=10MB', 'rb') assert f is not None assert gdal.VSIFEofL(f) == 0 assert gdal.VSIFReadL(10, 1, f) == b'0123456789' assert gdal.VSIFTellL(f) == 10 assert gdal.VSIFSeekL(f, 5, 0) == 0 assert gdal.VSIFTellL(f) == 5 assert gdal.VSIFReadL(3, 1, f) == b'567' assert gdal.VSIFTellL(f) == 8 assert gdal.VSIFReadL(4, 1, f) == b'8901' assert gdal.VSIFTellL(f) == 12 assert gdal.VSIFSeekL(f, 0, 2) == 0 assert gdal.VSIFTellL(f) == 10 * 1024 * 1024 assert gdal.VSIFReadL(1, 1, f) == b'' assert gdal.VSIFEofL(f) == 1 assert gdal.VSIFTellL(f) == 10 * 1024 * 1024 assert gdal.VSIFSeekL(f, 5, 0) == 0 assert gdal.VSIFTellL(f) == 5 assert gdal.VSIFReadL(3, 1, f) == b'567' assert gdal.VSIFTellL(f) == 8 assert gdal.VSIFSeekL(f, 10 * 1024 * 1024 - 10 + 1, 0) == 0 assert gdal.VSIFReadL(3, 1, f) == b'123' gdal.VSIFCloseL(f) with gdaltest.config_options({ 'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin', 'CPL_VSISTDIN_RESET_POSITION': 'YES', 'CPL_VSISTDIN_FILE_CLOSE': 'YES' }): f = gdal.VSIFOpenL('/vsistdin?buffer_limit=-1', 'rb') assert f is not None assert gdal.VSIFSeekL(f, 0, 2) == 0 assert gdal.VSIFTellL(f) == 10 * 1024 * 1024 gdal.VSIFCloseL(f) assert gdal.VSIStatL( '/vsistdin?buffer_limit=-1').size == 10 * 1024 * 1024 with gdaltest.config_options({ 'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin', 'CPL_VSISTDIN_RESET_POSITION': 'YES', 'CPL_VSISTDIN_FILE_CLOSE': 'YES' }): f = gdal.VSIFOpenL('/vsistdin?buffer_limit=1GB', 'rb') assert f is not None assert gdal.VSIFSeekL(f, 0, 2) == 0 assert gdal.VSIFTellL(f) == 10 * 1024 * 1024 gdal.VSIFCloseL(f) assert gdal.VSIStatL( '/vsistdin?buffer_limit=-1').size == 10 * 1024 * 1024 with gdaltest.config_options({ 'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin', 'CPL_VSISTDIN_RESET_POSITION': 'YES', 'CPL_VSISTDIN_FILE_CLOSE': 'YES' }): f = gdal.VSIFOpenL('/vsistdin?buffer_limit=10', 'rb') assert f is not None assert gdal.VSIFSeekL(f, 0, 2) == 0 assert gdal.VSIFTellL(f) == 10 * 1024 * 1024 assert gdal.VSIFSeekL(f, 0, 0) == 0 assert gdal.VSIFReadL(5, 1, f) == b'01234' assert gdal.VSIFReadL(3, 1, f) == b'567' with gdaltest.error_handler(): assert gdal.VSIFReadL(3, 1, f) == b'' gdal.VSIFCloseL(f) os.unlink('tmp/test_vsistdin_5.bin')
def misc_6_internal(datatype, nBands, setDriversDone): ds = gdal.GetDriverByName('MEM').Create('', 10, 10, nBands, datatype) if nBands > 0: ds.GetRasterBand(1).Fill(255) ds.SetGeoTransform([2, 1.0 / 10, 0, 49, 0, -1.0 / 10]) ds.SetProjection( 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["degree",0.01745329251994328]]' ) ds.SetMetadata(['a']) for i in range(gdal.GetDriverCount()): drv = gdal.GetDriver(i) md = drv.GetMetadata() if ('DCAP_CREATECOPY' in md or 'DCAP_CREATE' in md) and 'DCAP_RASTER' in md: # print ('drv = %s, nBands = %d, datatype = %s' % (drv.ShortName, nBands, gdal.GetDataTypeName(datatype))) skip = False # FIXME: A few cases that crashes and should be investigated if drv.ShortName == 'JPEG2000': if (nBands == 2 or nBands >= 5) or \ not (datatype == gdal.GDT_Byte or datatype == gdal.GDT_Int16 or datatype == gdal.GDT_UInt16): skip = True if skip is False: dirname = 'tmp/tmp/tmp_%s_%d_%s' % ( drv.ShortName, nBands, gdal.GetDataTypeName(datatype)) try: os.mkdir(dirname) except OSError: try: os.stat(dirname) # Hum the directory already exists... Not expected, but let's try to go on except OSError: reason = 'Cannot create %s before drv = %s, nBands = %d, datatype = %s' % ( dirname, drv.ShortName, nBands, gdal.GetDataTypeName(datatype)) gdaltest.post_reason(reason) return 'fail' filename = get_filename(drv, dirname) dst_ds = drv.CreateCopy(filename, ds) has_succeeded = dst_ds is not None dst_ds = None size = 0 stat = gdal.VSIStatL(filename) if stat is not None: size = stat.size try: shutil.rmtree(dirname) except OSError: reason = 'Cannot remove %s after drv = %s, nBands = %d, datatype = %s' % ( dirname, drv.ShortName, nBands, gdal.GetDataTypeName(datatype)) gdaltest.post_reason(reason) return 'fail' if has_succeeded and drv.ShortName not in setDriversDone and nBands > 0: setDriversDone.add(drv.ShortName) # The first list of drivers fail to detect short writing # The second one is because they are verbose in stderr if 'DCAP_VIRTUALIO' in md and size != 0 and \ drv.ShortName not in ['JPEG2000', 'KMLSUPEROVERLAY', 'HF2', 'ZMap', 'DDS'] and \ drv.ShortName not in ['GIF', 'JP2ECW', 'JP2Lura']: for j in range(10): truncated_size = (size * j) / 10 vsimem_filename = ( '/vsimem/test_truncate/||maxlength=%d||' % truncated_size) + get_filename(drv, '')[1:] # print('drv = %s, nBands = %d, datatype = %s, truncated_size = %d' % (drv.ShortName, nBands, gdal.GetDataTypeName(datatype), truncated_size)) dst_ds = drv.CreateCopy(vsimem_filename, ds) error_detected = False if dst_ds is None: error_detected = True else: gdal.ErrorReset() dst_ds = None if gdal.GetLastErrorMsg() != '': error_detected = True if not error_detected: msg = 'write error not decteded with with drv = %s, nBands = %d, datatype = %s, truncated_size = %d' % ( drv.ShortName, nBands, gdal.GetDataTypeName(datatype), truncated_size) print(msg) gdaltest.post_reason(msg) fl = gdal.ReadDirRecursive('/vsimem/test_truncate') if fl is not None: for myf in fl: gdal.Unlink('/vsimem/test_truncate/' + myf) fl = gdal.ReadDirRecursive( '/vsimem/test_truncate') if fl is not None: print(fl) if drv.ShortName not in [ 'ECW', 'JP2ECW', 'VRT', 'XPM', 'JPEG2000', 'FIT', 'RST', 'INGR', 'USGSDEM', 'KMLSUPEROVERLAY', 'GMT' ]: dst_ds = drv.CreateCopy( filename, ds, callback=misc_6_interrupt_callback_class().cbk) if dst_ds is not None: gdaltest.post_reason( 'interruption did not work with drv = %s, nBands = %d, datatype = %s' % (drv.ShortName, nBands, gdal.GetDataTypeName(datatype))) dst_ds = None try: shutil.rmtree(dirname) except OSError: pass return 'fail' dst_ds = None try: shutil.rmtree(dirname) except OSError: pass try: os.mkdir(dirname) except OSError: reason = 'Cannot create %s before drv = %s, nBands = %d, datatype = %s' % ( dirname, drv.ShortName, nBands, gdal.GetDataTypeName(datatype)) gdaltest.post_reason(reason) return 'fail' ds = None return 'success'
def test_vsifile_13(): gdal.VSIFOpenL('/vsigzip', 'rb') gdal.VSIFOpenL('/vsizip', 'rb') gdal.VSIFOpenL('/vsitar', 'rb') gdal.VSIFOpenL('/vsimem', 'rb') gdal.VSIFOpenL('/vsisparse', 'rb') gdal.VSIFOpenL('/vsisubfile', 'rb') gdal.VSIFOpenL('/vsicurl', 'rb') gdal.VSIFOpenL('/vsis3', 'rb') gdal.VSIFOpenL('/vsicurl_streaming', 'rb') gdal.VSIFOpenL('/vsis3_streaming', 'rb') gdal.VSIFOpenL('/vsistdin', 'rb') fp = gdal.VSIFOpenL('/vsistdout', 'wb') if fp is not None: gdal.VSIFCloseL(fp) gdal.VSIStatL('/vsigzip') gdal.VSIStatL('/vsizip') gdal.VSIStatL('/vsitar') gdal.VSIStatL('/vsimem') gdal.VSIStatL('/vsisparse') gdal.VSIStatL('/vsisubfile') gdal.VSIStatL('/vsicurl') gdal.VSIStatL('/vsis3') gdal.VSIStatL('/vsicurl_streaming') gdal.VSIStatL('/vsis3_streaming') gdal.VSIStatL('/vsistdin') gdal.VSIStatL('/vsistdout')
def test_vsiadls_opendir(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() # Unlimited depth from root handler = webserver.SequentialHandler() handler.add('GET', '/azure/blob/myaccount/?resource=account', 200, { 'Content-type': 'application/json', 'x-ms-continuation': 'contmarker_root' }, """{ "filesystems": [{ "name": "fs1"}, { "name": "fs2"} ]}""") with webserver.install_http_handler(handler): d = gdal.OpenDir('/vsiadls/') assert d is not None handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/fs1?recursive=true&resource=filesystem', 200, { 'Content-type': 'application/json;charset=utf-8', 'x-ms-continuation': 'contmarker_within_fs' }, """ {"paths":[{"name":"foo.txt","contentLength":"123456","lastModified": "Mon, 01 Jan 1970 00:00:01"}]} """) with webserver.install_http_handler(handler): entry = gdal.GetNextDirEntry(d) assert entry.name == 'fs1' assert entry.mode == 16384 handler = webserver.SequentialHandler() with webserver.install_http_handler(handler): entry = gdal.GetNextDirEntry(d) assert entry.name == 'fs1/foo.txt' handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/fs1?continuation=contmarker_within_fs&recursive=true&resource=filesystem', 200, { 'Content-type': 'application/json;charset=utf-8', 'x-ms-continuation': '' }, """ {"paths":[{"name":"bar.txt","contentLength":"123456","lastModified": "Mon, 01 Jan 1970 00:00:01"}]} """) with webserver.install_http_handler(handler): entry = gdal.GetNextDirEntry(d) assert entry.name == 'fs1/bar.txt' handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/fs2?recursive=true&resource=filesystem', 200, { 'Content-type': 'application/json;charset=utf-8', 'x-ms-continuation': '' }, """ {"paths":[]} """) with webserver.install_http_handler(handler): entry = gdal.GetNextDirEntry(d) assert entry.name == 'fs2' handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/?continuation=contmarker_root&resource=account', 200, {'Content-type': 'application/json'}, """{ "filesystems": [{ "name": "fs3"}] }""") handler.add( 'GET', '/azure/blob/myaccount/fs3?recursive=true&resource=filesystem', 200, { 'Content-type': 'application/json;charset=utf-8', 'x-ms-continuation': '' }, """ {"paths":[{"name":"baz.txt","contentLength":"123456","lastModified": "Mon, 01 Jan 1970 00:00:01"}]} """) with webserver.install_http_handler(handler): entry = gdal.GetNextDirEntry(d) assert entry.name == 'fs3' handler = webserver.SequentialHandler() with webserver.install_http_handler(handler): entry = gdal.GetNextDirEntry(d) assert entry.name == 'fs3/baz.txt' entry = gdal.GetNextDirEntry(d) assert entry is None gdal.CloseDir(d) # Prefix filtering on subdir handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/fs1?directory=sub_dir&recursive=true&resource=filesystem', 200, {'Content-type': 'application/json;charset=utf-8'}, """ {"paths":[{"name":"sub_dir/foo.txt","contentLength":"123456","lastModified": "Mon, 01 Jan 1970 00:00:01"}, {"name":"sub_dir/my_prefix_test.txt","contentLength":"40","lastModified": "Mon, 01 Jan 1970 00:00:01"}]} """) with webserver.install_http_handler(handler): d = gdal.OpenDir('/vsiadls/fs1/sub_dir', -1, ['PREFIX=my_prefix']) assert d is not None handler = webserver.SequentialHandler() with webserver.install_http_handler(handler): entry = gdal.GetNextDirEntry(d) assert entry.name == 'my_prefix_test.txt' assert entry.size == 40 assert entry.mode == 32768 assert entry.mtime == 1 entry = gdal.GetNextDirEntry(d) assert entry is None gdal.CloseDir(d) # No network access done s = gdal.VSIStatL( '/vsiadls/fs1/sub_dir/my_prefix_test.txt', gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG | gdal.VSI_STAT_CACHE_ONLY) assert s assert (s.mode & 32768) != 0 assert s.size == 40 assert s.mtime == 1 # No network access done assert gdal.VSIStatL( '/vsiadls/fs1/sub_dir/i_do_not_exist.txt', gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG | gdal.VSI_STAT_CACHE_ONLY) is None
def test_vsitar_longfilename_ustar(): assert gdal.VSIStatL('/vsitar/data/longfilename_ustar.tar/zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz/bbbbbbbbbbbbbbbbbbbbbbb/ccccccccccccccccccccccccc/dddddddddd/e/byte.tif') is not None
def test_vsiswift_extra_1(): if not gdaltest.built_against_curl(): pytest.skip() swift_resource = gdal.GetConfigOption('SWIFT_RESOURCE') if swift_resource is None: pytest.skip('Missing SWIFT_RESOURCE') if '/' not in swift_resource: path = '/vsiswift/' + swift_resource statres = gdal.VSIStatL(path) assert statres is not None and stat.S_ISDIR(statres.mode), \ ('%s is not a valid bucket' % path) readdir = gdal.ReadDir(path) assert readdir is not None, 'ReadDir() should not return empty list' for filename in readdir: if filename != '.': subpath = path + '/' + filename assert gdal.VSIStatL(subpath) is not None, \ ('Stat(%s) should not return an error' % subpath) unique_id = 'vsiswift_test' subpath = path + '/' + unique_id ret = gdal.Mkdir(subpath, 0) assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath) readdir = gdal.ReadDir(path) assert unique_id in readdir, \ ('ReadDir(%s) should contain %s' % (path, unique_id)) ret = gdal.Mkdir(subpath, 0) assert ret != 0, ('Mkdir(%s) repeated should return an error' % subpath) ret = gdal.Rmdir(subpath) assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath) readdir = gdal.ReadDir(path) assert unique_id not in readdir, \ ('ReadDir(%s) should not contain %s' % (path, unique_id)) ret = gdal.Rmdir(subpath) assert ret != 0, ('Rmdir(%s) repeated should return an error' % subpath) ret = gdal.Mkdir(subpath, 0) assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath) f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb') assert f is not None gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) ret = gdal.Rmdir(subpath) assert ret != 0, \ ('Rmdir(%s) on non empty directory should return an error' % subpath) f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb') assert f is not None data = gdal.VSIFReadL(1, 5, f).decode('utf-8') assert data == 'hello' gdal.VSIFCloseL(f) ret = gdal.Unlink(subpath + '/test.txt') assert ret >= 0, \ ('Unlink(%s) should not return an error' % (subpath + '/test.txt')) ret = gdal.Rmdir(subpath) assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath) return f = open_for_read('/vsiswift/' + swift_resource) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1 # Same with /vsiswift_streaming/ f = open_for_read('/vsiswift_streaming/' + swift_resource) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1 # Invalid resource gdal.ErrorReset() f = open_for_read('/vsiswift_streaming/' + swift_resource + '/invalid_resource.baz') assert f is None, gdal.VSIGetLastErrorMsg()
def process(argv, progress=None, progress_arg=None): if not argv: return Usage() dst_filename = None output_format = None src_datasets = [] overwrite_ds = False overwrite_layer = False update = False append = False single_layer = False layer_name_template = None skip_failures = False src_geom_types = [] field_strategy = None src_layer_field_name = None src_layer_field_content = None a_srs = None s_srs = None t_srs = None dsco = [] lco = [] i = 0 while i < len(argv): arg = argv[i] if (arg == '-f' or arg == '-of') and i + 1 < len(argv): i = i + 1 output_format = argv[i] elif arg == '-o' and i + 1 < len(argv): i = i + 1 dst_filename = argv[i] elif arg == '-progress': progress = ogr.TermProgress_nocb progress_arg = None elif arg == '-q' or arg == '-quiet': pass elif arg[0:5] == '-skip': skip_failures = True elif arg == '-update': update = True elif arg == '-overwrite_ds': overwrite_ds = True elif arg == '-overwrite_layer': overwrite_layer = True update = True elif arg == '-append': append = True update = True elif arg == '-single': single_layer = True elif arg == '-a_srs' and i + 1 < len(argv): i = i + 1 a_srs = argv[i] elif arg == '-s_srs' and i + 1 < len(argv): i = i + 1 s_srs = argv[i] elif arg == '-t_srs' and i + 1 < len(argv): i = i + 1 t_srs = argv[i] elif arg == '-nln' and i + 1 < len(argv): i = i + 1 layer_name_template = argv[i] elif arg == '-field_strategy' and i + 1 < len(argv): i = i + 1 field_strategy = argv[i] elif arg == '-src_layer_field_name' and i + 1 < len(argv): i = i + 1 src_layer_field_name = argv[i] elif arg == '-src_layer_field_content' and i + 1 < len(argv): i = i + 1 src_layer_field_content = argv[i] elif arg == '-dsco' and i + 1 < len(argv): i = i + 1 dsco.append(argv[i]) elif arg == '-lco' and i + 1 < len(argv): i = i + 1 lco.append(argv[i]) elif arg == '-src_geom_type' and i + 1 < len(argv): i = i + 1 src_geom_type_names = argv[i].split(',') for src_geom_type_name in src_geom_type_names: src_geom_type = _GetGeomType(src_geom_type_name) if src_geom_type is None: print('ERROR: Unrecognized geometry type: %s' % src_geom_type_name) return 1 src_geom_types.append(src_geom_type) elif arg[0] == '-': print('ERROR: Unrecognized argument : %s' % arg) return Usage() else: if '*' in arg: if sys.version_info < (3,0,0): src_datasets += [fn.decode(sys.getfilesystemencoding()) for fn in glob.glob(arg)] else: src_datasets += glob.glob(arg) else: src_datasets.append(arg) i = i + 1 if dst_filename is None: print('Missing -o') return 1 if update: if output_format is not None: print('ERROR: -f incompatible with -update') return 1 if dsco: print('ERROR: -dsco incompatible with -update') return 1 output_format = '' else: if output_format is None: output_format = GetOutputDriverFor(dst_filename) if src_layer_field_content is None: src_layer_field_content = '{AUTO_NAME}' elif src_layer_field_name is None: src_layer_field_name = 'source_ds_lyr' if not single_layer and output_format == 'ESRI Shapefile' and \ dst_filename.lower().endswith('.shp'): print('ERROR: Non-single layer mode incompatible with non-directory ' 'shapefile output') return 1 if not src_datasets: print('ERROR: No source datasets') return 1 if layer_name_template is None: if single_layer: layer_name_template = 'merged' else: layer_name_template = '{AUTO_NAME}' vrt_filename = None if not EQUAL(output_format, 'VRT'): dst_ds = gdal.OpenEx(dst_filename, gdal.OF_VECTOR | gdal.OF_UPDATE) if dst_ds is not None: if not update and not overwrite_ds: print('ERROR: Destination dataset already exists, ' + 'but -update nor -overwrite_ds are specified') return 1 if overwrite_ds: drv = dst_ds.GetDriver() dst_ds = None if drv.GetDescription() == 'OGR_VRT': # We don't want to destroy the sources of the VRT gdal.Unlink(dst_filename) else: drv.Delete(dst_filename) elif update: print('ERROR: Destination dataset does not exist') return 1 if dst_ds is None: drv = gdal.GetDriverByName(output_format) if drv is None: print('ERROR: Invalid driver: %s' % output_format) return 1 dst_ds = drv.Create( dst_filename, 0, 0, 0, gdal.GDT_Unknown, dsco) if dst_ds is None: return 1 vrt_filename = '/vsimem/_ogrmerge_.vrt' else: if gdal.VSIStatL(dst_filename) and not overwrite_ds: print('ERROR: Destination dataset already exists, ' + 'but -overwrite_ds are specified') return 1 vrt_filename = dst_filename f = gdal.VSIFOpenL(vrt_filename, 'wb') if f is None: print('ERROR: Cannot create %s' % vrt_filename) return 1 writer = XMLWriter(f) writer.open_element('OGRVRTDataSource') if single_layer: ogr_vrt_union_layer_written = False for src_ds_idx, src_dsname in enumerate(src_datasets): src_ds = ogr.Open(src_dsname) if src_ds is None: print('ERROR: Cannot open %s' % src_dsname) if skip_failures: continue gdal.VSIFCloseL(f) gdal.Unlink(vrt_filename) return 1 for src_lyr_idx, src_lyr in enumerate(src_ds): if src_geom_types: gt = ogr.GT_Flatten(src_lyr.GetGeomType()) if gt not in src_geom_types: continue if not ogr_vrt_union_layer_written: ogr_vrt_union_layer_written = True writer.open_element('OGRVRTUnionLayer', attrs={'name': layer_name_template}) if src_layer_field_name is not None: writer.write_element_value('SourceLayerFieldName', src_layer_field_name) if field_strategy is not None: writer.write_element_value('FieldStrategy', field_strategy) layer_name = src_layer_field_content src_lyr_name = src_lyr.GetName() try: src_lyr_name = src_lyr_name.decode('utf-8') except AttributeError: pass basename = None if os.path.exists(src_dsname): basename = os.path.basename(src_dsname) if '.' in basename: basename = '.'.join(basename.split(".")[0:-1]) if basename == src_lyr_name: layer_name = layer_name.replace('{AUTO_NAME}', basename) elif basename is None: layer_name = layer_name.replace( '{AUTO_NAME}', 'Dataset%d_%s' % (src_ds_idx, src_lyr_name)) else: layer_name = layer_name.replace( '{AUTO_NAME}', basename + '_' + src_lyr_name) if basename is not None: layer_name = layer_name.replace('{DS_BASENAME}', basename) else: layer_name = layer_name.replace('{DS_BASENAME}', src_dsname) layer_name = layer_name.replace('{DS_NAME}', '%s' % src_dsname) layer_name = layer_name.replace('{DS_INDEX}', '%d' % src_ds_idx) layer_name = layer_name.replace('{LAYER_NAME}', src_lyr_name) layer_name = layer_name.replace('{LAYER_INDEX}', '%d' % src_lyr_idx) if t_srs is not None: writer.open_element('OGRVRTWarpedLayer') writer.open_element('OGRVRTLayer', attrs={'name': layer_name}) attrs = {} if EQUAL(output_format, 'VRT') and \ os.path.exists(src_dsname) and \ not os.path.isabs(src_dsname) and \ '/' not in vrt_filename and \ '\\' not in vrt_filename: attrs['relativeToVRT'] = '1' if single_layer: attrs['shared'] = '1' writer.write_element_value('SrcDataSource', src_dsname, attrs=attrs) writer.write_element_value('SrcLayer', src_lyr.GetName()) if a_srs is not None: writer.write_element_value('LayerSRS', a_srs) writer.close_element('OGRVRTLayer') if t_srs is not None: if s_srs is not None: writer.write_element_value('SrcSRS', s_srs) writer.write_element_value('TargetSRS', t_srs) writer.close_element('OGRVRTWarpedLayer') if ogr_vrt_union_layer_written: writer.close_element('OGRVRTUnionLayer') else: for src_ds_idx, src_dsname in enumerate(src_datasets): src_ds = ogr.Open(src_dsname) if src_ds is None: print('ERROR: Cannot open %s' % src_dsname) if skip_failures: continue gdal.VSIFCloseL(f) gdal.Unlink(vrt_filename) return 1 for src_lyr_idx, src_lyr in enumerate(src_ds): if src_geom_types: gt = ogr.GT_Flatten(src_lyr.GetGeomType()) if gt not in src_geom_types: continue src_lyr_name = src_lyr.GetName() try: src_lyr_name = src_lyr_name.decode('utf-8') except AttributeError: pass layer_name = layer_name_template basename = None if os.path.exists(src_dsname): basename = os.path.basename(src_dsname) if '.' in basename: basename = '.'.join(basename.split(".")[0:-1]) if basename == src_lyr_name: layer_name = layer_name.replace('{AUTO_NAME}', basename) elif basename is None: layer_name = layer_name.replace( '{AUTO_NAME}', 'Dataset%d_%s' % (src_ds_idx, src_lyr_name)) else: layer_name = layer_name.replace( '{AUTO_NAME}', basename + '_' + src_lyr_name) if basename is not None: layer_name = layer_name.replace('{DS_BASENAME}', basename) elif '{DS_BASENAME}' in layer_name: if skip_failures: if '{DS_INDEX}' not in layer_name: layer_name = layer_name.replace( '{DS_BASENAME}', 'Dataset%d' % src_ds_idx) else: print('ERROR: Layer name template %s ' 'includes {DS_BASENAME} ' 'but %s is not a file' % (layer_name_template, src_dsname)) gdal.VSIFCloseL(f) gdal.Unlink(vrt_filename) return 1 layer_name = layer_name.replace('{DS_NAME}', '%s' % src_dsname) layer_name = layer_name.replace('{DS_INDEX}', '%d' % src_ds_idx) layer_name = layer_name.replace('{LAYER_NAME}', src_lyr_name) layer_name = layer_name.replace('{LAYER_INDEX}', '%d' % src_lyr_idx) if t_srs is not None: writer.open_element('OGRVRTWarpedLayer') writer.open_element('OGRVRTLayer', attrs={'name': layer_name}) attrs = {} if EQUAL(output_format, 'VRT') and \ os.path.exists(src_dsname) and \ not os.path.isabs(src_dsname) and \ '/' not in vrt_filename and \ '\\' not in vrt_filename: attrs['relativeToVRT'] = '1' if single_layer: attrs['shared'] = '1' writer.write_element_value('SrcDataSource', src_dsname, attrs=attrs) writer.write_element_value('SrcLayer', src_lyr_name) if a_srs is not None: writer.write_element_value('LayerSRS', a_srs) writer.close_element('OGRVRTLayer') if t_srs is not None: if s_srs is not None: writer.write_element_value('SrcSRS', s_srs) writer.write_element_value('TargetSRS', t_srs) writer.close_element('OGRVRTWarpedLayer') writer.close_element('OGRVRTDataSource') gdal.VSIFCloseL(f) ret = 0 if not EQUAL(output_format, 'VRT'): accessMode = None if append: accessMode = 'append' elif overwrite_layer: accessMode = 'overwrite' ret = gdal.VectorTranslate(dst_ds, vrt_filename, accessMode=accessMode, layerCreationOptions=lco, skipFailures=skip_failures, callback=progress, callback_data=progress_arg) if ret == 1: ret = 0 else: ret = 1 gdal.Unlink(vrt_filename) return ret
def vsiaz_fake_readdir(): if gdaltest.webserver_port == 0: return 'skip' handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%2F&restype=container', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Prefix>a_dir/</Prefix> <NextMarker>bla</NextMarker> <Blobs> <Blob> <Name>a_dir/resource3.bin</Name> <Properties> <Last-Modified>01 Jan 1970 00:00:01</Last-Modified> <Content-Length>123456</Content-Length> </Properties> </Blob> </Blobs> </EnumerationResults> """) handler.add( 'GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla&prefix=a_dir%2F&restype=container', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Prefix>a_dir/</Prefix> <Blobs> <Blob> <Name>a_dir/resource4.bin</Name> <Properties> <Last-Modified>16 Oct 2016 12:34:56</Last-Modified> <Content-Length>456789</Content-Length> </Properties> </Blob> <BlobPrefix> <Name>a_dir/subdir/</Name> </BlobPrefix> </Blobs> </EnumerationResults> """) with webserver.install_http_handler(handler): f = open_for_read('/vsiaz/az_fake_bucket2/a_dir/resource3.bin') if f is None: if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir') if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' if gdal.VSIStatL( '/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size != 123456: gdaltest.post_reason('fail') print(gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size) return 'fail' if gdal.VSIStatL('/vsiaz/az_fake_bucket2/a_dir/resource3.bin').mtime != 1: gdaltest.post_reason('fail') return 'fail' # ReadDir on something known to be a file shouldn't cause network access dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir/resource3.bin') if dir_contents is not None: gdaltest.post_reason('fail') return 'fail' # Test error on ReadDir() handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=error_test%2F&restype=container', 500) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/error_test/') if dir_contents is not None: gdaltest.post_reason('fail') print(dir_contents) return 'fail' # List containers (empty result) handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/?comp=list', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults ServiceEndpoint="https://myaccount.blob.core.windows.net"> <Containers/> </EnumerationResults> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiaz/') if dir_contents != ['.']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' gdal.VSICurlClearCache() # List containers handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/?comp=list', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Containers> <Container> <Name>mycontainer1</Name> </Container> </Containers> <NextMarker>bla</NextMarker> </EnumerationResults> """) handler.add( 'GET', '/azure/blob/myaccount/?comp=list&marker=bla', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <EnumerationResults> <Containers> <Container> <Name>mycontainer2</Name> </Container> </Containers> </EnumerationResults> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiaz/') if dir_contents != ['mycontainer1', 'mycontainer2']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' return 'success'
def vsiaz_extra_1(): if not gdaltest.built_against_curl(): return 'skip' az_resource = gdal.GetConfigOption('AZ_RESOURCE') if az_resource is None: print('Missing AZ_RESOURCE for running gdaltest_list_extra') return 'skip' if az_resource.find('/') < 0: path = '/vsiaz/' + az_resource statres = gdal.VSIStatL(path) if statres is None or not stat.S_ISDIR(statres.mode): gdaltest.post_reason('fail') print('%s is not a valid bucket' % path) return 'fail' readdir = gdal.ReadDir(path) if readdir is None: gdaltest.post_reason('fail') print('ReadDir() should not return empty list') return 'fail' for filename in readdir: if filename != '.': subpath = path + '/' + filename if gdal.VSIStatL(subpath) is None: gdaltest.post_reason('fail') print('Stat(%s) should not return an error' % subpath) return 'fail' unique_id = 'vsiaz_test' subpath = path + '/' + unique_id ret = gdal.Mkdir(subpath, 0) if ret < 0: gdaltest.post_reason('fail') print('Mkdir(%s) should not return an error' % subpath) return 'fail' readdir = gdal.ReadDir(path) if unique_id not in readdir: gdaltest.post_reason('fail') print('ReadDir(%s) should contain %s' % (path, unique_id)) print(readdir) return 'fail' ret = gdal.Mkdir(subpath, 0) if ret == 0: gdaltest.post_reason('fail') print('Mkdir(%s) repeated should return an error' % subpath) return 'fail' ret = gdal.Rmdir(subpath) if ret < 0: gdaltest.post_reason('fail') print('Rmdir(%s) should not return an error' % subpath) return 'fail' readdir = gdal.ReadDir(path) if unique_id in readdir: gdaltest.post_reason('fail') print('ReadDir(%s) should not contain %s' % (path, unique_id)) print(readdir) return 'fail' ret = gdal.Rmdir(subpath) if ret == 0: gdaltest.post_reason('fail') print('Rmdir(%s) repeated should return an error' % subpath) return 'fail' ret = gdal.Mkdir(subpath, 0) if ret < 0: gdaltest.post_reason('fail') print('Mkdir(%s) should not return an error' % subpath) return 'fail' f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) ret = gdal.Rmdir(subpath) if ret == 0: gdaltest.post_reason('fail') print('Rmdir(%s) on non empty directory should return an error' % subpath) return 'fail' f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 5, f).decode('utf-8') if data != 'hello': gdaltest.post_reason('fail') print(data) return 'fail' gdal.VSIFCloseL(f) ret = gdal.Unlink(subpath + '/test.txt') if ret < 0: gdaltest.post_reason('fail') print('Unlink(%s) should not return an error' % (subpath + '/test.txt')) return 'fail' ret = gdal.Rmdir(subpath) if ret < 0: gdaltest.post_reason('fail') print('Rmdir(%s) should not return an error' % subpath) return 'fail' return 'success' f = open_for_read('/vsiaz/' + az_resource) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' # Same with /vsiaz_streaming/ f = open_for_read('/vsiaz_streaming/' + az_resource) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' if False: # we actually try to read at read() time and bSetError = false # Invalid bucket : "The specified bucket does not exist" gdal.ErrorReset() f = open_for_read('/vsiaz/not_existing_bucket/foo') with gdaltest.error_handler(): gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if gdal.VSIGetLastErrorMsg() == '': gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' # Invalid resource gdal.ErrorReset() f = open_for_read('/vsiaz_streaming/' + az_resource + '/invalid_resource.baz') if f is not None: gdaltest.post_reason('fail') print(gdal.VSIGetLastErrorMsg()) return 'fail' # Test GetSignedURL() signed_url = gdal.GetSignedURL('/vsiaz/' + az_resource) f = open_for_read('/vsicurl_streaming/' + signed_url) if f is None: gdaltest.post_reason('fail') return 'fail' ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(ret) != 1: gdaltest.post_reason('fail') print(ret) return 'fail' return 'success'
def test_vsiaz_extra_1(): if not gdaltest.built_against_curl(): pytest.skip() az_resource = gdal.GetConfigOption('AZ_RESOURCE') if az_resource is None: pytest.skip('Missing AZ_RESOURCE') if '/' not in az_resource: path = '/vsiaz/' + az_resource statres = gdal.VSIStatL(path) assert statres is not None and stat.S_ISDIR(statres.mode), \ ('%s is not a valid bucket' % path) readdir = gdal.ReadDir(path) assert readdir is not None, 'ReadDir() should not return empty list' for filename in readdir: if filename != '.': subpath = path + '/' + filename assert gdal.VSIStatL(subpath) is not None, \ ('Stat(%s) should not return an error' % subpath) unique_id = 'vsiaz_test' subpath = path + '/' + unique_id ret = gdal.Mkdir(subpath, 0) assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath) readdir = gdal.ReadDir(path) assert unique_id in readdir, \ ('ReadDir(%s) should contain %s' % (path, unique_id)) ret = gdal.Mkdir(subpath, 0) assert ret != 0, ('Mkdir(%s) repeated should return an error' % subpath) ret = gdal.Rmdir(subpath) assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath) readdir = gdal.ReadDir(path) assert unique_id not in readdir, \ ('ReadDir(%s) should not contain %s' % (path, unique_id)) ret = gdal.Rmdir(subpath) assert ret != 0, ('Rmdir(%s) repeated should return an error' % subpath) ret = gdal.Mkdir(subpath, 0) assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath) f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb') assert f is not None gdal.VSIFWriteL('hello', 1, 5, f) gdal.VSIFCloseL(f) ret = gdal.Rmdir(subpath) assert ret != 0, \ ('Rmdir(%s) on non empty directory should return an error' % subpath) f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb') assert f is not None data = gdal.VSIFReadL(1, 5, f).decode('utf-8') assert data == 'hello' gdal.VSIFCloseL(f) ret = gdal.Unlink(subpath + '/test.txt') assert ret >= 0, \ ('Unlink(%s) should not return an error' % (subpath + '/test.txt')) ret = gdal.Rmdir(subpath) assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath) return f = open_for_read('/vsiaz/' + az_resource) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1 # Same with /vsiaz_streaming/ f = open_for_read('/vsiaz_streaming/' + az_resource) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1 if False: # pylint: disable=using-constant-test # we actually try to read at read() time and bSetError = false # Invalid bucket : "The specified bucket does not exist" gdal.ErrorReset() f = open_for_read('/vsiaz/not_existing_bucket/foo') with gdaltest.error_handler(): gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert gdal.VSIGetLastErrorMsg() != '' # Invalid resource gdal.ErrorReset() f = open_for_read('/vsiaz_streaming/' + az_resource + '/invalid_resource.baz') assert f is None, gdal.VSIGetLastErrorMsg() # Test GetSignedURL() signed_url = gdal.GetSignedURL('/vsiaz/' + az_resource) f = open_for_read('/vsicurl_streaming/' + signed_url) assert f is not None ret = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(ret) == 1
def vsiaz_fake_basic(): if gdaltest.webserver_port == 0: return 'skip' signed_url = gdal.GetSignedURL('/vsiaz/az_fake_bucket/resource', ['START_DATE=20180213T123456']) if signed_url not in ( 'http://127.0.0.1:8080/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12', 'http://127.0.0.1:8081/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12' ): gdaltest.post_reason('fail') print(signed_url) return 'fail' def method(request): request.protocol_version = 'HTTP/1.1' h = request.headers if 'Authorization' not in h or \ h['Authorization'] != 'SharedKey myaccount:zKb0EXnM/RinBjcUE9EU+qfRIGaIItoUElSWc+FE24E=' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.send_header('Connection', 'close') request.end_headers() request.wfile.write("""foo""".encode('ascii')) handler = webserver.SequentialHandler() handler.add('GET', '/azure/blob/myaccount/az_fake_bucket/resource', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsiaz/az_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' def method(request): request.protocol_version = 'HTTP/1.1' h = request.headers if 'Authorization' not in h or \ h['Authorization'] != 'SharedKey myaccount:8d6IEeOsl7qGpKAxaTTxx2xMNpvqWq8DGlFE67lsmQ4=' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'Accept-Encoding' not in h or h['Accept-Encoding'] != 'gzip': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.send_header('Connection', 'close') request.end_headers() request.wfile.write("""foo""".encode('ascii')) handler = webserver.SequentialHandler() handler.add('GET', '/azure/blob/myaccount/az_fake_bucket/resource', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsiaz_streaming/az_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' handler = webserver.SequentialHandler() handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin', 200, {'Content-Length': '1000000'}) with webserver.install_http_handler(handler): stat_res = gdal.VSIStatL('/vsiaz/az_fake_bucket/resource2.bin') if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' handler = webserver.SequentialHandler() handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin', 200, {'Content-Length': 1000000}) with webserver.install_http_handler(handler): stat_res = gdal.VSIStatL( '/vsiaz_streaming/az_fake_bucket/resource2.bin') if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' return 'success'
def vsigs_readdir(): if gdaltest.webserver_port == 0: return 'skip' handler = webserver.SequentialHandler() handler.add( 'GET', '/gs_fake_bucket2/?delimiter=/&prefix=a_dir/', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>a_dir/</Prefix> <NextMarker>bla</NextMarker> <Contents> <Key>a_dir/resource3.bin</Key> <LastModified>1970-01-01T00:00:01.000Z</LastModified> <Size>123456</Size> </Contents> </ListBucketResult> """) handler.add( 'GET', '/gs_fake_bucket2/?delimiter=/&marker=bla&prefix=a_dir/', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListBucketResult> <Prefix>a_dir/</Prefix> <Contents> <Key>a_dir/resource4.bin</Key> <LastModified>2015-10-16T12:34:56.000Z</LastModified> <Size>456789</Size> </Contents> <CommonPrefixes> <Prefix>a_dir/subdir/</Prefix> </CommonPrefixes> </ListBucketResult> """) with webserver.install_http_handler(handler): f = open_for_read('/vsigs/gs_fake_bucket2/a_dir/resource3.bin') if f is None: if gdaltest.is_travis_branch('trusty'): print('Skipped on trusty branch, but should be investigated') return 'skip' gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir') if dir_contents != ['resource3.bin', 'resource4.bin', 'subdir']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' if gdal.VSIStatL( '/vsigs/gs_fake_bucket2/a_dir/resource3.bin').size != 123456: gdaltest.post_reason('fail') return 'fail' if gdal.VSIStatL('/vsigs/gs_fake_bucket2/a_dir/resource3.bin').mtime != 1: gdaltest.post_reason('fail') return 'fail' # ReadDir on something known to be a file shouldn't cause network access dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir/resource3.bin') if dir_contents is not None: gdaltest.post_reason('fail') return 'fail' # List buckets handler = webserver.SequentialHandler() handler.add( 'GET', '/', 200, {'Content-type': 'application/xml'}, """<?xml version="1.0" encoding="UTF-8"?> <ListAllMyBucketsResult> <Buckets> <Bucket> <Name>mybucket</Name> </Bucket> </Buckets> </ListAllMyBucketsResult> """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsigs/') if dir_contents != ['mybucket']: gdaltest.post_reason('fail') print(dir_contents) return 'fail' return 'success'
def vsigs_2(): if gdaltest.webserver_port == 0: return 'skip' # header file gdal.FileFromMemBuffer('/vsimem/my_headers.txt', 'foo: bar') def method(request): if 'foo' not in request.headers or request.headers['foo'] != 'bar': sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 1) request.end_headers() request.wfile.write("""Y""".encode('ascii')) handler = webserver.SequentialHandler() handler.add('GET', '/gs_fake_bucket_http_header_file/resource', custom_method=method) with webserver.install_http_handler(handler): with gdaltest.config_option('GDAL_HTTP_HEADER_FILE', '/vsimem/my_headers.txt'): f = open_for_read( '/vsigs/gs_fake_bucket_http_header_file/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) if len(data) != 1: gdaltest.post_reason('fail') return 'fail' gdal.Unlink('/vsimem/my_headers.txt') gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY') gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID') gdal.SetConfigOption('CPL_GS_TIMESTAMP', 'my_timestamp') def method(request): request.protocol_version = 'HTTP/1.1' if 'Authorization' not in request.headers: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) return expected_authorization = 'GOOG1 GS_ACCESS_KEY_ID:8tndu9//BfmN+Kg4AFLdUMZMBDQ=' if request.headers['Authorization'] != expected_authorization: sys.stderr.write("Bad Authorization: '%s'\n" % str(request.headers['Authorization'])) request.send_response(403) return request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Length', 3) request.send_header('Connection', 'close') request.end_headers() request.wfile.write("""foo""".encode('ascii')) handler = webserver.SequentialHandler() handler.add('GET', '/gs_fake_bucket/resource', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsigs/gs_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' handler = webserver.SequentialHandler() handler.add('GET', '/gs_fake_bucket/resource', custom_method=method) with webserver.install_http_handler(handler): f = open_for_read('/vsigs_streaming/gs_fake_bucket/resource') if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) if data != 'foo': gdaltest.post_reason('fail') print(data) return 'fail' handler = webserver.SequentialHandler() handler.add('GET', '/gs_fake_bucket/resource2.bin', 206, {'Content-Range': 'bytes 0-0/1000000'}, 'x') with webserver.install_http_handler(handler): stat_res = gdal.VSIStatL('/vsigs/gs_fake_bucket/resource2.bin') if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' handler = webserver.SequentialHandler() handler.add('HEAD', '/gs_fake_bucket/resource2.bin', 200, {'Content-Length': 1000000}) with webserver.install_http_handler(handler): stat_res = gdal.VSIStatL( '/vsigs_streaming/gs_fake_bucket/resource2.bin') if stat_res is None or stat_res.size != 1000000: gdaltest.post_reason('fail') if stat_res is not None: print(stat_res.size) else: print(stat_res) return 'fail' return 'success'
def test_vicar_create(): filename = '/vsimem/test.vic' md = { 'LBLSIZE': 1234, 'BLTYPE': 'foo', 'PROPERTY': { 'MYPROP': { 'INT': 1, 'INT64': 1234567890123, 'REAL': 1.25, 'ARRAY_INT': [1, 2], 'ARRAY_STRING': ['a', 'b'], 'STRING': "eh'eh" }, 'APPROX': { 'NULL': None, 'BOOL_TRUE': True, 'BOOL_FALSE': False, 'OBJ': { "a": "b" } } }, 'TASK': { 'GEN': { 'USER': '******', 'DAT_TIM': 'Thu Sep 24 17:31:50 1992', 'OTHER_PROP': 'foo' } } } src_ds = gdal.Open('data/rgbsmall.tif') ds = gdal.GetDriverByName('VICAR').Create(filename, src_ds.RasterXSize, src_ds.RasterYSize, src_ds.RasterCount, gdal.GDT_Byte) ds.SetMetadata([json.dumps(md)], "json:VICAR") ds.WriteRaster(0, 0, src_ds.RasterXSize, src_ds.RasterYSize, src_ds.ReadRaster()) ds = None assert not gdal.VSIStatL(filename + '.aux.xml') ds = gdal.Open(filename) assert [ds.GetRasterBand(i+1).Checksum() for i in range(ds.RasterCount)] == \ [src_ds.GetRasterBand(i+1).Checksum() for i in range(src_ds.RasterCount)] lbl = ds.GetMetadata_List('json:VICAR')[0] lbl = json.loads(lbl) assert lbl['LBLSIZE'] == 600 assert lbl['BLTYPE'] == 'foo' assert lbl['PROPERTY']['MYPROP'] == md['PROPERTY']['MYPROP'] assert lbl['PROPERTY']['APPROX'] == { 'NULL': 'NULL', 'BOOL_TRUE': 1, 'BOOL_FALSE': 0, 'OBJ': '{"a":"b"}' } assert lbl['TASK'] == md['TASK'] filename2 = '/vsimem/test2.vic' assert gdal.GetDriverByName('VICAR').CreateCopy(filename2, ds) assert not gdal.VSIStatL(filename2 + '.aux.xml') ds = None ds = gdal.Open(filename2) assert [ds.GetRasterBand(i+1).Checksum() for i in range(ds.RasterCount)] == \ [src_ds.GetRasterBand(i+1).Checksum() for i in range(src_ds.RasterCount)] lbl = ds.GetMetadata_List('json:VICAR')[0] lbl = json.loads(lbl) assert lbl['LBLSIZE'] == 600 assert lbl['BLTYPE'] == 'foo' assert lbl['PROPERTY']['MYPROP'] == md['PROPERTY']['MYPROP'] assert lbl['PROPERTY']['APPROX'] == { 'NULL': 'NULL', 'BOOL_TRUE': 1, 'BOOL_FALSE': 0, 'OBJ': '{"a":"b"}' } assert lbl['TASK'] == md['TASK'] ds = None gdal.GetDriverByName('VICAR').Delete(filename) gdal.GetDriverByName('VICAR').Delete(filename2)
def test_ogr_pds4_create_table_character(line_ending): options = ['VAR_LOGICAL_IDENTIFIER=urn:foo:bar:baz:logical_identifier', 'VAR_TITLE=title', 'VAR_INVESTIGATION_AREA_NAME=ian', 'VAR_INVESTIGATION_AREA_LID_REFERENCE=urn:foo:bar:baz:ialr', 'VAR_OBSERVING_SYSTEM_NAME=osn', 'VAR_TARGET=target', 'VAR_TARGET_TYPE=target'] ds = ogr.GetDriverByName('PDS4').CreateDataSource('/vsimem/test.xml', options=options) layer_creation_options = ['TABLE_TYPE=CHARACTER'] if line_ending: layer_creation_options.append('LINE_ENDING=' + line_ending) if line_ending == 'error': with gdaltest.error_handler(): lyr = ds.CreateLayer('0f:oo', options=layer_creation_options) else: lyr = ds.CreateLayer('0f:oo', options=layer_creation_options) fld = ogr.FieldDefn('bool', ogr.OFTInteger) fld.SetSubType(ogr.OFSTBoolean) lyr.CreateField(fld) lyr.CreateField(ogr.FieldDefn('int', ogr.OFTInteger)) lyr.CreateField(ogr.FieldDefn('int64', ogr.OFTInteger64)) lyr.CreateField(ogr.FieldDefn('real', ogr.OFTReal)) lyr.CreateField(ogr.FieldDefn('str', ogr.OFTString)) lyr.CreateField(ogr.FieldDefn('datetime', ogr.OFTDateTime)) lyr.CreateField(ogr.FieldDefn('date', ogr.OFTDate)) lyr.CreateField(ogr.FieldDefn('time', ogr.OFTTime)) f = ogr.Feature(lyr.GetLayerDefn()) f['bool'] = 1 f['int'] = -123456789 f['int64'] = -1234567890123 f['real'] = 1.25 f['str'] = 'foo' f['datetime'] = '2019/01/24 12:34:56.789+00' f['date'] = '2019-01-24' f['time'] = '12:34:56.789' lyr.CreateFeature(f) ds = None f = gdal.VSIFOpenL('/vsimem/test.xml', 'rb') data = gdal.VSIFReadL(1, 100000, f).decode('ascii') gdal.VSIFCloseL(f) assert '_Character' in data assert '_Binary' not in data if line_ending == 'LF': assert '<record_delimiter>Line-Feed</record_delimiter>' in data else: assert '<record_delimiter>Carriage-Return Line-Feed</record_delimiter>' in data assert 'LSB' not in data assert 'MSB' not in data assert '<local_identifier>_0f_oo</local_identifier>' in data if line_ending is None: # Only do that check in that configuration for faster test execution assert validate_xml('/vsimem/test.xml') assert gdal.VSIStatL('/vsimem/test/0f_oo.dat') f = gdal.VSIFOpenL('/vsimem/test/0f_oo.dat', 'rb') data = gdal.VSIFReadL(1, 100000, f).decode('ascii') gdal.VSIFCloseL(f) if line_ending == 'LF': assert '\n' in data assert '\r\n' not in data else: assert '\r\n' in data ds = ogr.Open('/vsimem/test.xml') lyr = ds.GetLayer(0) assert lyr.GetLayerDefn().GetFieldCount() == 8 f = lyr.GetNextFeature() assert f['bool'] assert f['int'] == -123456789 assert f['int64'] == -1234567890123 assert f['real'] == 1.25 assert f['str'] == 'foo' assert f['datetime'] == '2019/01/24 12:34:56.789+00' assert f['date'] == '2019/01/24' assert f['time'] == '12:34:56.789' ds = None if line_ending is None: # Only do that part in that configuration for faster test execution # Add new layer ds = ogr.Open('/vsimem/test.xml', update = 1) lyr = ds.CreateLayer('bar', options=['TABLE_TYPE=CHARACTER']) lyr.CreateField(ogr.FieldDefn('int', ogr.OFTInteger)) f = ogr.Feature(lyr.GetLayerDefn()) f['int'] = 123 lyr.CreateFeature(f) ds = None assert validate_xml('/vsimem/test.xml') ds = ogr.Open('/vsimem/test.xml') lyr = ds.GetLayerByName('bar') f = lyr.GetNextFeature() assert f['int'] == 123 lyr = ds.GetLayer(0) f = lyr.GetNextFeature() assert f['int'] == -123456789 ds = None ogr.GetDriverByName('PDS4').DeleteDataSource('/vsimem/test.xml') gdal.Rmdir('/vsimem/test')
def vsizip_8(): if gdal.VSIStatL('/vsizip/vsizip/data/zero.bin.zip.zip/zero.bin.zip').size != 5000 * 1000 * 1000 + 1: return 'fail' return 'success'
def test_vsiswift_fake_readdir(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add( 'GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=1', 200, {'Content-type': 'application/json'}, """[ { "last_modified": "1970-01-01T00:00:01", "bytes": 123456, "name": "bar.baz" } ]""") handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=1&marker=bar.baz', 200, {'Content-type': 'application/json'}, """[ { "subdir": "mysubdir/" } ]""") handler.add( 'GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=1&marker=mysubdir%2F', 200, {'Content-type': 'application/json'}, """[ ]""") with gdaltest.config_option('SWIFT_MAX_KEYS', '1'): with webserver.install_http_handler(handler): f = open_for_read('/vsiswift/foo/bar.baz') assert f is not None gdal.VSIFCloseL(f) dir_contents = gdal.ReadDir('/vsiswift/foo') assert dir_contents == ['bar.baz', 'mysubdir'] stat_res = gdal.VSIStatL('/vsiswift/foo/bar.baz') assert stat_res.size == 123456 assert stat_res.mtime == 1 # ReadDir on something known to be a file shouldn't cause network access dir_contents = gdal.ReadDir('/vsiswift/foo/bar.baz') assert dir_contents is None # Test error on ReadDir() handler = webserver.SequentialHandler() handler.add( 'GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000&prefix=error_test%2F', 500) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiswift/foo/error_test/') assert dir_contents is None # List containers (empty result) handler = webserver.SequentialHandler() handler.add('GET', '/v1/AUTH_something', 200, {'Content-type': 'application/json'}, """[] """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiswift/') assert dir_contents == ['.'] # List containers gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add( 'GET', '/v1/AUTH_something', 200, {'Content-type': 'application/json'}, """[ { "name": "mycontainer1", "count": 0, "bytes": 0 }, { "name": "mycontainer2", "count": 0, "bytes": 0} ] """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiswift/') assert dir_contents == ['mycontainer1', 'mycontainer2'] # ReadDir() with a file and directory of same names gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add( 'GET', '/v1/AUTH_something', 200, {'Content-type': 'application/json'}, """[ { "last_modified": "1970-01-01T00:00:01", "bytes": 123456, "name": "foo" }, { "subdir": "foo/"} ] """) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiswift/') assert dir_contents == ['foo', 'foo/'] handler = webserver.SequentialHandler() handler.add('GET', '/v1/AUTH_something/foo?delimiter=%2F&limit=10000', 200, {'Content-type': 'application/json'}, "[]") with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiswift/foo/') assert dir_contents == ['.']
def vsifile_generic(filename): start_time = time.time() fp = gdal.VSIFOpenL(filename, 'wb+') if fp is None: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFWriteL('0123456789', 1, 10, fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTruncateL(fp, 20) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTruncateL(fp, 5) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFSeekL(fp, 0, 2) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 5: gdaltest.post_reason('failure') return 'fail' gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 7: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if abs(start_time - statBuf.mtime) > 2: gdaltest.post_reason('failure') print(statBuf.mtime) return 'fail' fp = gdal.VSIFOpenL(filename, 'rb') buf = gdal.VSIFReadL(1, 7, fp) if gdal.VSIFWriteL('a', 1, 1, fp) != 0: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFTruncateL(fp, 0) == 0: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(fp) if buf.decode('ascii') != '01234XX': gdaltest.post_reason('failure') print(buf.decode('ascii')) return 'fail' # Test append mode on existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 9: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if gdal.Unlink(filename) != 0: gdaltest.post_reason('failure') return 'fail' statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG) if statBuf is not None: gdaltest.post_reason('failure') return 'fail' # Test append mode on non existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 2: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if gdal.Unlink(filename) != 0: gdaltest.post_reason('failure') return 'fail' return 'success'
def vsizip_1(): # We can keep the handle open during all the ZIP writing hZIP = gdal.VSIFOpenL("/vsizip/vsimem/test.zip", "wb") if hZIP is None: gdaltest.post_reason('fail 1') return 'fail' # One way to create a directory f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir2/", "wb") if f is None: gdaltest.post_reason('fail 2') return 'fail' gdal.VSIFCloseL(f) # A more natural one gdal.Mkdir("/vsizip/vsimem/test.zip/subdir1", 0) # Create 1st file f2 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "wb") if f2 is None: gdaltest.post_reason('fail 3') return 'fail' gdal.VSIFWriteL("abcd", 1, 4, f2) gdal.VSIFCloseL(f2) # Test that we cannot read a zip file being written gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb") gdal.PopErrorHandler() if gdal.GetLastErrorMsg() != 'Cannot read a zip file being written': gdaltest.post_reason('expected error') print(gdal.GetLastErrorMsg()) return 'fail' if f is not None: gdaltest.post_reason('should not have been successful 1') return 'fail' # Create 2nd file f3 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/efghi", "wb") if f3 is None: gdaltest.post_reason('fail 4') return 'fail' gdal.VSIFWriteL("efghi", 1, 5, f3) # Try creating a 3d file gdal.ErrorReset() gdal.PushErrorHandler('CPLQuietErrorHandler') f4 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/that_wont_work", "wb") gdal.PopErrorHandler() if gdal.GetLastErrorMsg( ) != 'Cannot create that_wont_work while another file is being written in the .zip': gdaltest.post_reason('expected error') print(gdal.GetLastErrorMsg()) return 'fail' if f4 is not None: gdaltest.post_reason('should not have been successful 2') return 'fail' gdal.VSIFCloseL(f3) # Now we can close the main handle gdal.VSIFCloseL(hZIP) # ERROR 6: Support only 1 file in archive file /vsimem/test.zip when no explicit in-archive filename is specified gdal.ErrorReset() with gdaltest.error_handler(): f = gdal.VSIFOpenL('/vsizip/vsimem/test.zip', 'rb') if f is not None: gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() == '': gdaltest.post_reason('expected error') print(gdal.GetLastErrorMsg()) return 'fail' f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb") if f is None: gdaltest.post_reason('fail 5') return 'fail' data = gdal.VSIFReadL(1, 4, f) gdal.VSIFCloseL(f) if data.decode('ASCII') != 'abcd': gdaltest.post_reason('fail') print(data) return 'fail' # Test alternate uri syntax gdal.Rename("/vsimem/test.zip", "/vsimem/test.xxx") f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd", "rb") if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f) gdal.VSIFCloseL(f) if data.decode('ASCII') != 'abcd': gdaltest.post_reason('fail') print(data) return 'fail' # With a trailing slash f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd/", "rb") if f is None: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) # Test ReadDir() if len(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}")) != 3: gdaltest.post_reason('fail') print(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}")) return 'fail' # Unbalanced curls f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx", "rb") if f is not None: gdaltest.post_reason('fail') return 'fail' # Non existing mainfile f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/bla", "rb") if f is not None: gdaltest.post_reason('fail') return 'fail' # Non existing subfile f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.zzz}/bla", "rb") if f is not None: gdaltest.post_reason('fail') return 'fail' # Wrong syntax f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}.aux.xml", "rb") if f is not None: gdaltest.post_reason('fail') return 'fail' # Test nested { { } } hZIP = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}", "wb") if hZIP is None: gdaltest.post_reason('fail 1') return 'fail' f = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}/test.xxx", "wb") f_src = gdal.VSIFOpenL("/vsimem/test.xxx", "rb") data = gdal.VSIFReadL(1, 10000, f_src) gdal.VSIFCloseL(f_src) gdal.VSIFWriteL(data, 1, len(data), f) gdal.VSIFCloseL(f) gdal.VSIFCloseL(hZIP) f = gdal.VSIFOpenL( "/vsizip/{/vsizip/{/vsimem/zipinzip.yyy}/test.xxx}/subdir3/abcd/", "rb") if f is None: gdaltest.post_reason('fail') return 'fail' data = gdal.VSIFReadL(1, 4, f) gdal.VSIFCloseL(f) if data.decode('ASCII') != 'abcd': gdaltest.post_reason('fail') print(data) return 'fail' gdal.Unlink("/vsimem/test.xxx") gdal.Unlink("/vsimem/zipinzip.yyy") # Test VSIStatL on a non existing file if gdal.VSIStatL('/vsizip//vsimem/foo.zip') is not None: gdaltest.post_reason('fail') return 'fail' # Test ReadDir on a non existing file if gdal.ReadDir('/vsizip//vsimem/foo.zip') is not None: gdaltest.post_reason('fail') return 'fail' return 'success'
def vsifile_10(): gdal.FileFromMemBuffer( '/vsimem/vsifile_10.tar', """FUZZER_FRIENDLY_ARCHIVE ***NEWFILE***:test.txt abc***NEWFILE***:huge.txt 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 0123456789012345678901234567890123456789012345678901234567890123456789012345678X ***NEWFILE***:small.txt a""") contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar') if contents is None: gdal.Unlink('/vsimem/vsifile_10.tar') return 'skip' if contents != ['test.txt', 'huge.txt', 'small.txt']: gdaltest.post_reason('fail') print(contents) return 'fail' if gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/test.txt').size != 3: gdaltest.post_reason('fail') print(gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/test.txt').size) return 'fail' if gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/huge.txt').size != 3888: gdaltest.post_reason('fail') print(gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/huge.txt').size) return 'fail' if gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/small.txt').size != 1: gdaltest.post_reason('fail') print(gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/small.txt').size) return 'fail' gdal.FileFromMemBuffer('/vsimem/vsifile_10.tar', """FUZZER_FRIENDLY_ARCHIVE ***NEWFILE***:x abc""") contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar') if contents != ['x']: gdaltest.post_reason('fail') print(contents) return 'fail' gdal.FileFromMemBuffer( '/vsimem/vsifile_10.tar', """FUZZER_FRIENDLY_ARCHIVE ***NEWFILE***:x abc***NEWFILE***:""") contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar') if contents != ['x']: gdaltest.post_reason('fail') print(contents) return 'fail' gdal.Unlink('/vsimem/vsifile_10.tar') return 'success'
def test_vsiadls_fake_readdir(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/az_fake_bucket2?directory=a_dir%20with_space&recursive=false&resource=filesystem', 200, { 'Content-type': 'application/json;charset=utf-8', 'x-ms-continuation': 'contmarker' }, """ {"paths":[{"name":"a_dir with_space/resource3 with_space.bin","contentLength":"123456","lastModified": "Mon, 01 Jan 1970 00:00:01"}]} """) handler.add( 'GET', '/azure/blob/myaccount/az_fake_bucket2?continuation=contmarker&directory=a_dir%20with_space&recursive=false&resource=filesystem', 200, {'Content-type': 'application/json;charset=utf-8'}, """ {"paths":[{"name":"a_dir with_space/resource4.bin","contentLength":"456789","lastModified": "16 Oct 2016 12:34:56"}, {"name":"a_dir with_space/subdir","isDirectory":"true"}]} """) with webserver.install_http_handler(handler): f = open_for_read( '/vsiadls/az_fake_bucket2/a_dir with_space/resource3 with_space.bin' ) if f is None: pytest.fail() gdal.VSIFCloseL(f) dir_contents = gdal.ReadDir('/vsiadls/az_fake_bucket2/a_dir with_space') assert dir_contents == [ 'resource3 with_space.bin', 'resource4.bin', 'subdir' ] assert gdal.VSIStatL( '/vsiadls/az_fake_bucket2/a_dir with_space/resource3 with_space.bin' ).size == 123456 assert gdal.VSIStatL( '/vsiadls/az_fake_bucket2/a_dir with_space/resource3 with_space.bin' ).mtime == 1 # ReadDir on something known to be a file shouldn't cause network access dir_contents = gdal.ReadDir( '/vsiadls/az_fake_bucket2/a_dir with_space/resource3 with_space.bin') assert dir_contents is None # Test error on ReadDir() handler = webserver.SequentialHandler() handler.add( 'GET', '/azure/blob/myaccount/az_fake_bucket2?directory=error_test&recursive=false&resource=filesystem', 500) with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiadls/az_fake_bucket2/error_test/') assert dir_contents is None # List containers (empty result) handler = webserver.SequentialHandler() handler.add('GET', '/azure/blob/myaccount/?resource=account', 200, {'Content-type': 'application/json'}, """{ "filesystems": [] }""") with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiadls/') assert dir_contents == ['.'] gdal.VSICurlClearCache() # List containers handler = webserver.SequentialHandler() handler.add('GET', '/azure/blob/myaccount/?resource=account', 200, { 'Content-type': 'application/json', 'x-ms-continuation': 'contmarker' }, """{ "filesystems": [{ "name": "mycontainer1"}] }""") handler.add( 'GET', '/azure/blob/myaccount/?continuation=contmarker&resource=account', 200, {'Content-type': 'application/json'}, """{ "filesystems": [{ "name": "mycontainer2"}] }""") with webserver.install_http_handler(handler): dir_contents = gdal.ReadDir('/vsiadls/') assert dir_contents == ['mycontainer1', 'mycontainer2']
def test_cog_creation_options(): filename = '/vsimem/cog.tif' src_ds = gdal.Open('data/byte.tif') ds = gdal.GetDriverByName('COG').CreateCopy( filename, src_ds, options=['COMPRESS=DEFLATE', 'LEVEL=1', 'NUM_THREADS=2']) assert ds ds = None ds = gdal.Open(filename) assert ds.GetRasterBand(1).Checksum() == 4672 assert ds.GetMetadataItem('COMPRESSION', 'IMAGE_STRUCTURE') == 'DEFLATE' ds = None filesize = gdal.VSIStatL(filename).size _check_cog(filename) gdal.GetDriverByName('COG').CreateCopy( filename, src_ds, options=['COMPRESS=DEFLATE', 'BIGTIFF=YES', 'LEVEL=1']) assert gdal.VSIStatL(filename).size != filesize gdal.GetDriverByName('COG').CreateCopy( filename, src_ds, options=['COMPRESS=DEFLATE', 'PREDICTOR=YES', 'LEVEL=1']) assert gdal.VSIStatL(filename).size != filesize gdal.GetDriverByName('COG').CreateCopy( filename, src_ds, options=['COMPRESS=DEFLATE', 'LEVEL=9']) assert gdal.VSIStatL(filename).size < filesize colist = gdal.GetDriverByName('COG').GetMetadataItem( 'DMD_CREATIONOPTIONLIST') if '<Value>ZSTD' in colist: gdal.GetDriverByName('COG').CreateCopy(filename, src_ds, options=['COMPRESS=ZSTD']) ds = gdal.Open(filename) assert ds.GetMetadataItem('COMPRESSION', 'IMAGE_STRUCTURE') == 'ZSTD' ds = None if '<Value>WEBP' in colist: with gdaltest.error_handler(): assert not gdal.GetDriverByName('COG').CreateCopy( filename, src_ds, options=['COMPRESS=WEBP']) if '<Value>LERC' in colist: assert gdal.GetDriverByName('COG').CreateCopy( filename, src_ds, options=['COMPRESS=LERC']) filesize_no_z_error = gdal.VSIStatL(filename).size assert gdal.VSIStatL(filename).size != filesize assert gdal.GetDriverByName('COG').CreateCopy( filename, src_ds, options=['COMPRESS=LERC', 'MAX_Z_ERROR=10']) filesize_with_z_error = gdal.VSIStatL(filename).size assert filesize_with_z_error < filesize_no_z_error assert gdal.GetDriverByName('COG').CreateCopy( filename, src_ds, options=['COMPRESS=LERC_DEFLATE']) filesize_lerc_deflate = gdal.VSIStatL(filename).size assert filesize_lerc_deflate < filesize_no_z_error assert gdal.GetDriverByName('COG').CreateCopy( filename, src_ds, options=['COMPRESS=LERC_DEFLATE', 'LEVEL=1']) filesize_lerc_deflate_level_1 = gdal.VSIStatL(filename).size assert filesize_lerc_deflate_level_1 > filesize_lerc_deflate if '<Value>ZSTD' in colist: assert gdal.GetDriverByName('COG').CreateCopy( filename, src_ds, options=['COMPRESS=LERC_ZSTD']) filesize_lerc_zstd = gdal.VSIStatL(filename).size assert filesize_lerc_zstd < filesize_no_z_error assert gdal.GetDriverByName('COG').CreateCopy( filename, src_ds, options=['COMPRESS=LERC_ZSTD', 'LEVEL=1']) filesize_lerc_zstd_level_1 = gdal.VSIStatL(filename).size assert filesize_lerc_zstd_level_1 > filesize_lerc_zstd src_ds = None gdal.GetDriverByName('GTiff').Delete(filename)
def vsifile_generic(filename): start_time = time.time() fp = gdal.VSIFOpenL(filename, 'wb+') assert fp is not None assert gdal.VSIFWriteL('0123456789', 1, 10, fp) == 10 assert gdal.VSIFFlushL(fp) == 0 assert gdal.VSIFTruncateL(fp, 20) == 0 assert gdal.VSIFTellL(fp) == 10 assert gdal.VSIFTruncateL(fp, 5) == 0 assert gdal.VSIFTellL(fp) == 10 assert gdal.VSIFSeekL(fp, 0, 2) == 0 assert gdal.VSIFTellL(fp) == 5 gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) assert statBuf.size == 7 assert start_time == pytest.approx(statBuf.mtime, abs=2) fp = gdal.VSIFOpenL(filename, 'rb') assert gdal.VSIFReadL(1, 0, fp) is None assert gdal.VSIFReadL(0, 1, fp) is None buf = gdal.VSIFReadL(1, 7, fp) assert gdal.VSIFWriteL('a', 1, 1, fp) == 0 assert gdal.VSIFTruncateL(fp, 0) != 0 gdal.VSIFCloseL(fp) assert buf.decode('ascii') == '01234XX' # Test append mode on existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) assert statBuf.size == 9 assert gdal.Unlink(filename) == 0 statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG) assert statBuf is None # Test append mode on non existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) assert statBuf.size == 2 assert gdal.Unlink(filename) == 0
def jpeg_19(): for width, height, iX in [(32, 32, 12), (25, 25, 8), (24, 25, 8)]: src_ds = gdal.GetDriverByName('GTiff').Create('/vsimem/jpeg_19.tif', width, height, 1) src_ds.CreateMaskBand(gdal.GMF_PER_DATASET) src_ds.GetRasterBand(1).GetMaskBand().WriteRaster( 0, 0, iX, height, struct.pack('B' * 1, 255), 1, 1) src_ds.GetRasterBand(1).GetMaskBand().WriteRaster( iX, 0, width - iX, height, struct.pack('B' * 1, 0), 1, 1) tiff_mask_data = src_ds.GetRasterBand(1).GetMaskBand().ReadRaster( 0, 0, width, height) # Generate a JPEG file with a (default) LSB bit mask order out_ds = gdal.GetDriverByName('JPEG').CreateCopy( '/vsimem/jpeg_19.jpg', src_ds) out_ds = None # Generate a JPEG file with a MSB bit mask order gdal.SetConfigOption('JPEG_WRITE_MASK_BIT_ORDER', 'MSB') out_ds = gdal.GetDriverByName('JPEG').CreateCopy( '/vsimem/jpeg_19_msb.jpg', src_ds) del out_ds gdal.SetConfigOption('JPEG_WRITE_MASK_BIT_ORDER', None) src_ds = None # Check that the file are indeed different statBuf = gdal.VSIStatL('/vsimem/jpeg_19.jpg') f = gdal.VSIFOpenL('/vsimem/jpeg_19.jpg', 'rb') data1 = gdal.VSIFReadL(1, statBuf.size, f) gdal.VSIFCloseL(f) statBuf = gdal.VSIStatL('/vsimem/jpeg_19_msb.jpg') f = gdal.VSIFOpenL('/vsimem/jpeg_19_msb.jpg', 'rb') data2 = gdal.VSIFReadL(1, statBuf.size, f) gdal.VSIFCloseL(f) if (width, height, iX) == (24, 25, 8): if data1 != data2: gdaltest.post_reason('fail') return 'fail' else: if data1 == data2: gdaltest.post_reason('fail') return 'fail' # Check the file with the LSB bit mask order ds = gdal.Open('/vsimem/jpeg_19.jpg') jpg_mask_data = ds.GetRasterBand(1).GetMaskBand().ReadRaster( 0, 0, width, height) ds = None if tiff_mask_data != jpg_mask_data: gdaltest.post_reason('fail') return 'fail' # Check the file with the MSB bit mask order ds = gdal.Open('/vsimem/jpeg_19_msb.jpg') jpg_mask_data = ds.GetRasterBand(1).GetMaskBand().ReadRaster( 0, 0, width, height) ds = None if tiff_mask_data != jpg_mask_data: gdaltest.post_reason('fail') return 'fail' gdal.GetDriverByName('GTiff').Delete('/vsimem/jpeg_19.tif') gdal.GetDriverByName('JPEG').Delete('/vsimem/jpeg_19.jpg') gdal.GetDriverByName('JPEG').Delete('/vsimem/jpeg_19_msb.jpg') return 'success'
def main(argv): """Return 0 in case of success, 1 for failure.""" i = 1 filename = None quiet = False full_check = None while i < len(argv): if argv[i] == '-q': quiet = True elif argv[i] == '--full-check=yes': full_check = True elif argv[i] == '--full-check=no': full_check = False elif argv[i] == '--full-check=auto': full_check = None elif argv[i][0] == '-': return Usage() elif filename is None: filename = argv[i] else: return Usage() i += 1 if filename is None: return Usage() if full_check is None: full_check = filename.startswith('/vsimem/') or os.path.exists( filename) try: ret = 0 warnings, errors, details = validate(filename, full_check=full_check) if warnings: if not quiet: print('The following warnings were found:') for warning in warnings: print(' - ' + warning) print('') if errors: if not quiet: print('%s is NOT a valid cloud optimized GeoTIFF.' % filename) print('The following errors were found:') for error in errors: print(' - ' + error) print('') ret = 1 else: if not quiet: print('%s is a valid cloud optimized GeoTIFF' % filename) if not quiet and not warnings and not errors: headers_size = min(details['data_offsets'][k] for k in details['data_offsets']) if headers_size == 0: headers_size = gdal.VSIStatL(filename).size print('\nThe size of all IFD headers is %d bytes' % headers_size) except ValidateCloudOptimizedGeoTIFFException as e: if not quiet: print('%s is NOT a valid cloud optimized GeoTIFF : %s' % (filename, str(e))) ret = 1 return ret
def gdal_cp(argv, progress=None): srcfile = None targetfile = None recurse = False skip_failure = False argv = gdal.GeneralCmdLineProcessor(argv) if argv is None: return -1 for i in range(1, len(argv)): if argv[i] == '-progress': progress = TermProgress() elif argv[i] == '-r': version_num = int(gdal.VersionInfo('VERSION_NUM')) if version_num < 1900: print('ERROR: Python bindings of GDAL 1.9.0 or later required for -r option') return -1 recurse = True elif len(argv[i]) >= 5 and argv[i][0:5] == '-skip': skip_failure = True elif argv[i][0] == '-': print('Unrecognized option : %s' % argv[i]) return Usage() elif srcfile is None: srcfile = argv[i] elif targetfile is None: targetfile = argv[i] else: print('Unexpected option : %s' % argv[i]) return Usage() if srcfile is None or targetfile is None: return Usage() if needsVSICurl(srcfile): srcfile = '/vsicurl/' + srcfile if recurse: # Make sure that 'gdal_cp.py -r [srcdir/]lastsubdir targetdir' creates # targetdir/lastsubdir if targetdir already exists (like cp -r does). if srcfile[-1] == '/': srcfile = srcfile[0:len(srcfile) - 1] statBufSrc = gdal.VSIStatL(srcfile, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG) if statBufSrc is None: statBufSrc = gdal.VSIStatL(srcfile + '/', gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG) statBufDst = gdal.VSIStatL(targetfile, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG) if statBufSrc is not None and statBufSrc.IsDirectory() and statBufDst is not None and statBufDst.IsDirectory(): if targetfile[-1] == '/': targetfile = targetfile[0:-1] if srcfile.rfind('/') != -1: targetfile = targetfile + srcfile[srcfile.rfind('/'):] else: targetfile = targetfile + '/' + srcfile if gdal.VSIStatL(targetfile) is None: gdal.Mkdir(targetfile, int('0755', 8)) return gdal_cp_recurse(srcfile, targetfile, progress, skip_failure) (srcdir, pattern) = os.path.split(srcfile) if not srcdir.startswith('/vsi') and ('*' in pattern or '?' in pattern): return gdal_cp_pattern_match(srcdir, pattern, targetfile, progress, skip_failure) return gdal_cp_single(srcfile, targetfile, progress)
def test_vsifile_10(): gdal.FileFromMemBuffer('/vsimem/vsifile_10.tar', """FUZZER_FRIENDLY_ARCHIVE ***NEWFILE***:test.txt abc***NEWFILE***:huge.txt 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 01234567890123456789012345678901234567890123456789012345678901234567890123456789 0123456789012345678901234567890123456789012345678901234567890123456789012345678X ***NEWFILE***:small.txt a""") contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar') if contents is None: gdal.Unlink('/vsimem/vsifile_10.tar') pytest.skip() assert contents == ['test.txt', 'huge.txt', 'small.txt'] assert gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/test.txt').size == 3 assert gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/huge.txt').size == 3888 assert gdal.VSIStatL('/vsitar//vsimem/vsifile_10.tar/small.txt').size == 1 gdal.FileFromMemBuffer('/vsimem/vsifile_10.tar', """FUZZER_FRIENDLY_ARCHIVE ***NEWFILE***:x abc""") contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar') assert contents == ['x'] gdal.FileFromMemBuffer('/vsimem/vsifile_10.tar', """FUZZER_FRIENDLY_ARCHIVE ***NEWFILE***:x abc***NEWFILE***:""") contents = gdal.ReadDir('/vsitar//vsimem/vsifile_10.tar') assert contents == ['x'] gdal.Unlink('/vsimem/vsifile_10.tar')
def gdal_cp_single(srcfile, targetfile, progress): if targetfile.endswith('/'): stat_res = gdal.VSIStatL(targetfile) else: stat_res = gdal.VSIStatL(targetfile + '/') if (stat_res is None and targetfile.endswith('/')) or \ (stat_res is not None and stat.S_ISDIR(stat_res.mode)): (_, tail) = os.path.split(srcfile) if targetfile.endswith('/'): targetfile = targetfile + tail else: targetfile = targetfile + '/' + tail fin = gdal.VSIFOpenL(srcfile, "rb") if fin is None: print('Cannot open %s' % srcfile) return -1 fout = gdal.VSIFOpenL(targetfile, "wb") if fout is None: print('Cannot create %s' % targetfile) gdal.VSIFCloseL(fin) return -1 version_num = int(gdal.VersionInfo('VERSION_NUM')) total_size = 0 if version_num < 1900 or progress is not None: gdal.VSIFSeekL(fin, 0, 2) total_size = gdal.VSIFTellL(fin) gdal.VSIFSeekL(fin, 0, 0) buf_max_size = 4096 copied = 0 ret = 0 # print('Copying %s...' % srcfile) if progress is not None: if not progress.Progress(0.0, 'Copying %s' % srcfile): print('Copy stopped by user') ret = -2 while ret == 0: if total_size != 0 and copied + buf_max_size > total_size: to_read = total_size - copied else: to_read = buf_max_size buf = gdal.VSIFReadL(1, to_read, fin) if buf is None: if copied == 0: print('Cannot read %d bytes in %s' % (to_read, srcfile)) ret = -1 break buf_size = len(buf) if gdal.VSIFWriteL(buf, 1, buf_size, fout) != buf_size: print('Error writing %d bytes' % buf_size) ret = -1 break copied += buf_size if progress is not None and total_size != 0: if not progress.Progress(copied * 1.0 / total_size, 'Copying %s' % srcfile): print('Copy stopped by user') ret = -2 break if to_read < buf_max_size or buf_size != buf_max_size: break gdal.VSIFCloseL(fin) gdal.VSIFCloseL(fout) return ret
def jpeg_28(): tmpfilename = '/vsimem/jpeg_28.jpg' # Nothing src_ds = gdal.GetDriverByName('MEM').Create('', 1, 1) ds = gdal.GetDriverByName('JPEG').CreateCopy(tmpfilename, src_ds) src_ds = None ds = gdal.Open(tmpfilename) if len(ds.GetMetadata()) != 0: gdaltest.post_reason('fail') return 'fail' # EXIF tags only src_ds = gdal.GetDriverByName('MEM').Create('', 1, 1) src_ds.SetMetadataItem('EXIF_DateTime', 'dt') # not enough values ASCII src_ds.SetMetadataItem('EXIF_DateTimeOriginal', '01234567890123456789') # truncated ASCII src_ds.SetMetadataItem( 'EXIF_DateTimeDigitized', '0123456789012345678') # right number of items ASCII src_ds.SetMetadataItem('EXIF_Make', 'make') # variable ASCII src_ds.SetMetadataItem('EXIF_ExifVersion', '01234') # truncated UNDEFINED src_ds.SetMetadataItem('EXIF_ComponentsConfiguration', '0x1F') # not enough values UNDEFINED src_ds.SetMetadataItem('EXIF_FlashpixVersion', 'ABCD') # right number of items UNDEFINED src_ds.SetMetadataItem('EXIF_SpatialFrequencyResponse', '0xab 0xCD') # variable UNDEFINED src_ds.SetMetadataItem('EXIF_Orientation', '10') # right number of items SHORT src_ds.SetMetadataItem('EXIF_ResolutionUnit', '2 4') # truncated SHORT src_ds.SetMetadataItem('EXIF_TransferFunction', '0 1') # not enough values SHORT src_ds.SetMetadataItem('EXIF_ISOSpeedRatings', '1 2 3') # variable SHORT src_ds.SetMetadataItem('EXIF_StandardOutputSensitivity', '123456789') # right number of items LONG src_ds.SetMetadataItem('EXIF_XResolution', '96') # right number of items RATIONAL src_ds.SetMetadataItem('EXIF_YResolution', '96 0') # truncated RATIONAL src_ds.SetMetadataItem('EXIF_CompressedBitsPerPixel', 'nan') # invalid RATIONAL src_ds.SetMetadataItem('EXIF_ApertureValue', '-1') # invalid RATIONAL with gdaltest.error_handler(): gdal.GetDriverByName('JPEG').CreateCopy(tmpfilename, src_ds) src_ds = None if gdal.VSIStatL(tmpfilename + '.aux.xml') is not None: gdaltest.post_reason('fail') return 'fail' ds = gdal.Open(tmpfilename) got_md = ds.GetMetadata() expected_md = { 'EXIF_DateTimeDigitized': '0123456789012345678', 'EXIF_DateTimeOriginal': '0123456789012345678', 'EXIF_Orientation': '10', 'EXIF_ApertureValue': '(0)', 'EXIF_YResolution': '(96)', 'EXIF_XResolution': '(96)', 'EXIF_TransferFunction': '0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0', 'EXIF_ExifVersion': '0123', 'EXIF_DateTime': 'dt ', 'EXIF_FlashpixVersion': 'ABCD', 'EXIF_ComponentsConfiguration': '0x1f 0x00 0x00 0x00', 'EXIF_Make': 'make', 'EXIF_StandardOutputSensitivity': '123456789', 'EXIF_ResolutionUnit': '2', 'EXIF_CompressedBitsPerPixel': '(0)', 'EXIF_SpatialFrequencyResponse': '0xab 0xcd', 'EXIF_ISOSpeedRatings': '1 2 3' } if got_md != expected_md: gdaltest.post_reason('fail') print(got_md) return 'fail' # Test SRATIONAL for val in (-1.5, -1, -0.5, 0, 0.5, 1, 1.5): src_ds = gdal.GetDriverByName('MEM').Create('', 1, 1) src_ds.SetMetadataItem('EXIF_ShutterSpeedValue', str(val)) gdal.GetDriverByName('JPEG').CreateCopy(tmpfilename, src_ds) src_ds = None if gdal.VSIStatL(tmpfilename + '.aux.xml') is not None: gdaltest.post_reason('fail') return 'fail' ds = gdal.Open(tmpfilename) got_val = ds.GetMetadataItem('EXIF_ShutterSpeedValue') got_val = got_val.replace('(', '').replace(')', '') if float(got_val) != val: gdaltest.post_reason('fail') print(val) print(ds.GetMetadataItem('EXIF_ShutterSpeedValue')) return 'fail' # Test RATIONAL for val in (0, 0.5, 1, 1.5): src_ds = gdal.GetDriverByName('MEM').Create('', 1, 1) src_ds.SetMetadataItem('EXIF_ApertureValue', str(val)) gdal.GetDriverByName('JPEG').CreateCopy(tmpfilename, src_ds) src_ds = None if gdal.VSIStatL(tmpfilename + '.aux.xml') is not None: gdaltest.post_reason('fail') return 'fail' ds = gdal.Open(tmpfilename) got_val = ds.GetMetadataItem('EXIF_ApertureValue') got_val = got_val.replace('(', '').replace(')', '') if float(got_val) != val: gdaltest.post_reason('fail') print(val) print(ds.GetMetadataItem('EXIF_ApertureValue')) return 'fail' # GPS tags only src_ds = gdal.GetDriverByName('MEM').Create('', 1, 1) src_ds.SetMetadataItem('EXIF_GPSLatitudeRef', 'N') src_ds.SetMetadataItem('EXIF_GPSLatitude', '49 34 56.5') gdal.GetDriverByName('JPEG').CreateCopy(tmpfilename, src_ds) src_ds = None if gdal.VSIStatL(tmpfilename + '.aux.xml') is not None: gdaltest.post_reason('fail') return 'fail' ds = gdal.Open(tmpfilename) got_md = ds.GetMetadata() if got_md != { 'EXIF_GPSLatitudeRef': 'N', 'EXIF_GPSLatitude': '(49) (34) (56.5)' }: gdaltest.post_reason('fail') print(got_md) return 'fail' ds = None # EXIF and GPS tags src_ds = gdal.GetDriverByName('MEM').Create('', 1, 1) src_ds.SetMetadataItem('EXIF_ExifVersion', '0231') src_ds.SetMetadataItem('EXIF_GPSLatitudeRef', 'N') gdal.GetDriverByName('JPEG').CreateCopy(tmpfilename, src_ds) src_ds = None if gdal.VSIStatL(tmpfilename + '.aux.xml') is not None: gdaltest.post_reason('fail') return 'fail' ds = gdal.Open(tmpfilename) got_md = ds.GetMetadata() if got_md != {'EXIF_ExifVersion': '0231', 'EXIF_GPSLatitudeRef': 'N'}: gdaltest.post_reason('fail') print(got_md) return 'fail' ds = None # EXIF and other metadata src_ds = gdal.GetDriverByName('MEM').Create('', 1, 1) src_ds.SetMetadataItem('EXIF_ExifVersion', '0231') src_ds.SetMetadataItem('EXIF_invalid', 'foo') src_ds.SetMetadataItem('FOO', 'BAR') with gdaltest.error_handler(): gdal.GetDriverByName('JPEG').CreateCopy(tmpfilename, src_ds) src_ds = None if gdal.VSIStatL(tmpfilename + '.aux.xml') is None: gdaltest.post_reason('fail') return 'fail' ds = gdal.Open(tmpfilename) got_md = ds.GetMetadata() if got_md != { 'EXIF_ExifVersion': '0231', 'EXIF_invalid': 'foo', 'FOO': 'BAR' }: gdaltest.post_reason('fail') print(got_md) return 'fail' ds = None # Too much content for EXIF src_ds = gdal.GetDriverByName('MEM').Create('', 1, 1) src_ds.SetMetadataItem('EXIF_UserComment', ''.join(['x' for i in range(65535)])) with gdaltest.error_handler(): gdal.GetDriverByName('JPEG').CreateCopy(tmpfilename, src_ds) src_ds = None ds = None # EXIF and GPS tags and EXIF overview src_ds = gdal.GetDriverByName('MEM').Create('', 1024, 1024) src_ds.SetMetadataItem('EXIF_ExifVersion', '0231') src_ds.SetMetadataItem('EXIF_GPSLatitudeRef', 'N') gdal.GetDriverByName('JPEG').CreateCopy(tmpfilename, src_ds, options=[ 'EXIF_THUMBNAIL=YES', 'THUMBNAIL_WIDTH=32', 'THUMBNAIL_HEIGHT=32' ]) src_ds = None if gdal.VSIStatL(tmpfilename + '.aux.xml') is not None: gdaltest.post_reason('fail') return 'fail' ds = gdal.Open(tmpfilename) got_md = ds.GetMetadata() if got_md != {'EXIF_ExifVersion': '0231', 'EXIF_GPSLatitudeRef': 'N'}: gdaltest.post_reason('fail') print(got_md) return 'fail' if ds.GetRasterBand(1).GetOverview( ds.GetRasterBand(1).GetOverviewCount() - 1).XSize != 32: gdaltest.post_reason('fail') return 'fail' ds = None gdal.Unlink(tmpfilename) return 'success'
def test_vsigs_2(): if gdaltest.webserver_port == 0: pytest.skip() # header file gdal.FileFromMemBuffer('/vsimem/my_headers.txt', 'foo: bar') handler = webserver.SequentialHandler() handler.add('GET', '/gs_fake_bucket_http_header_file/resource', 200, {'Content-type': 'text/plain'}, 'Y', expected_headers={'foo': 'bar'}) with webserver.install_http_handler(handler): with gdaltest.config_option('GDAL_HTTP_HEADER_FILE', '/vsimem/my_headers.txt'): f = open_for_read( '/vsigs/gs_fake_bucket_http_header_file/resource') assert f is not None data = gdal.VSIFReadL(1, 1, f) gdal.VSIFCloseL(f) assert len(data) == 1 gdal.Unlink('/vsimem/my_headers.txt') gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY') gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID') gdal.SetConfigOption('CPL_GS_TIMESTAMP', 'my_timestamp') signed_url = gdal.GetSignedURL('/vsigs/gs_fake_bucket/resource', ['START_DATE=20180212T123456Z']) assert (signed_url in ( 'http://127.0.0.1:8080/gs_fake_bucket/resource?Expires=1518442496&GoogleAccessId=GS_ACCESS_KEY_ID&Signature=xTphUyMqtKA6UmAX3PEr5VL3EOg%3D', 'http://127.0.0.1:8081/gs_fake_bucket/resource?Expires=1518442496&GoogleAccessId=GS_ACCESS_KEY_ID&Signature=xTphUyMqtKA6UmAX3PEr5VL3EOg%3D' )) handler = webserver.SequentialHandler() handler.add('GET', '/gs_fake_bucket/resource', 200, {'Content-type': 'text/plain'}, 'foo', expected_headers={ 'Authorization': 'GOOG1 GS_ACCESS_KEY_ID:8tndu9//BfmN+Kg4AFLdUMZMBDQ=' }) with webserver.install_http_handler(handler): f = open_for_read('/vsigs/gs_fake_bucket/resource') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo' handler = webserver.SequentialHandler() handler.add('GET', '/gs_fake_bucket/resource', 200, {'Content-type': 'text/plain'}, 'foo', expected_headers={ 'Authorization': 'GOOG1 GS_ACCESS_KEY_ID:8tndu9//BfmN+Kg4AFLdUMZMBDQ=' }) with webserver.install_http_handler(handler): f = open_for_read('/vsigs_streaming/gs_fake_bucket/resource') assert f is not None data = gdal.VSIFReadL(1, 4, f).decode('ascii') gdal.VSIFCloseL(f) assert data == 'foo' handler = webserver.SequentialHandler() handler.add('GET', '/gs_fake_bucket/resource2.bin', 206, {'Content-Range': 'bytes 0-0/1000000'}, 'x') with webserver.install_http_handler(handler): stat_res = gdal.VSIStatL('/vsigs/gs_fake_bucket/resource2.bin') if stat_res is None or stat_res.size != 1000000: if stat_res is not None: print(stat_res.size) else: print(stat_res) pytest.fail() handler = webserver.SequentialHandler() handler.add('HEAD', '/gs_fake_bucket/resource2.bin', 200, {'Content-Length': 1000000}) with webserver.install_http_handler(handler): stat_res = gdal.VSIStatL( '/vsigs_streaming/gs_fake_bucket/resource2.bin') if stat_res is None or stat_res.size != 1000000: if stat_res is not None: print(stat_res.size) else: print(stat_res) pytest.fail()