def test_vsiaz_fake_write(): if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() # Test creation of BlockBob f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() def method(request): h = request.headers if 'Authorization' not in h or \ h['Authorization'] != 'SharedKey myaccount:AigkrY7q66WCrx3JRKBte56k7kxV2cxB/ZyGNubxk5I=' or \ 'Expect' not in h or h['Expect'] != '100-continue' or \ 'Content-Length' not in h or h['Content-Length'] != '40000' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'BlockBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii')) content = request.rfile.read(40000).decode('ascii') if len(content) != 40000: sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) with webserver.install_http_handler(handler): ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f) ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f) if ret != 40000: gdal.VSIFCloseL(f) pytest.fail(ret) gdal.VSIFCloseL(f) # Simulate illegal read f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None with gdaltest.error_handler(): ret = gdal.VSIFReadL(1, 1, f) assert not ret gdal.VSIFCloseL(f) # Simulate illegal seek f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None with gdaltest.error_handler(): ret = gdal.VSIFSeekL(f, 1, 0) assert ret != 0 gdal.VSIFCloseL(f) # Simulate failure when putting BlockBob f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) if gdal.VSIFSeekL(f, 0, 0) != 0: gdal.VSIFCloseL(f) pytest.fail() gdal.VSIFWriteL('x' * 35000, 1, 35000, f) if gdal.VSIFTellL(f) != 35000: gdal.VSIFCloseL(f) pytest.fail() if gdal.VSIFSeekL(f, 35000, 0) != 0: gdal.VSIFCloseL(f) pytest.fail() if gdal.VSIFSeekL(f, 0, 1) != 0: gdal.VSIFCloseL(f) pytest.fail() if gdal.VSIFSeekL(f, 0, 2) != 0: gdal.VSIFCloseL(f) pytest.fail() if gdal.VSIFEofL(f) != 0: gdal.VSIFCloseL(f) pytest.fail() with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.VSIFCloseL(f) if ret == 0: gdal.VSIFCloseL(f) pytest.fail(ret) # Simulate creation of BlockBob over an existing blob of incompatible type f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') assert f is not None handler = webserver.SequentialHandler() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 409) handler.add('DELETE', '/azure/blob/myaccount/test_copy/file.bin', 202) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201) with webserver.install_http_handler(handler): gdal.VSIFCloseL(f) # Test creation of AppendBlob gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10') f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None) assert f is not None handler = webserver.SequentialHandler() def method(request): h = request.headers if 'Authorization' not in h or \ h['Authorization'] != 'SharedKey myaccount:KimVui3ptY9D5ftLlsI7CNOgK36CNAEzsXqcuHskdEY=' or \ 'Content-Length' not in h or h['Content-Length'] != '0' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) def method(request): h = request.headers if 'Content-Length' not in h or h['Content-Length'] != '10' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' content = request.rfile.read(10).decode('ascii') if content != '0123456789': sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', custom_method=method) def method(request): h = request.headers if 'Content-Length' not in h or h['Content-Length'] != '6' or \ 'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \ 'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob': sys.stderr.write('Bad headers: %s\n' % str(h)) request.send_response(403) return request.protocol_version = 'HTTP/1.1' content = request.rfile.read(6).decode('ascii') if content != 'abcdef': sys.stderr.write('Bad headers: %s\n' % str(request.headers)) request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(201) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', custom_method=method) with webserver.install_http_handler(handler): ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) if ret != 16: gdal.VSIFCloseL(f) pytest.fail(ret) gdal.VSIFCloseL(f) # Test failed creation of AppendBlob gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10') f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None) assert f is not None handler = webserver.SequentialHandler() def method(request): request.protocol_version = 'HTTP/1.1' request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', custom_method=method) with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) if ret != 0: gdal.VSIFCloseL(f) pytest.fail(ret) gdal.VSIFCloseL(f) # Test failed writing of a block of an AppendBlob gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10') f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb') gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None) assert f is not None handler = webserver.SequentialHandler() handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201) handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin?comp=appendblock', 403) with webserver.install_http_handler(handler): with gdaltest.error_handler(): ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f) if ret != 0: gdal.VSIFCloseL(f) pytest.fail(ret) gdal.VSIFCloseL(f)
def seek(self, pos, ref): gdal.VSIFSeekL(self.f, pos, ref)
def vsis3_4(): if gdaltest.webserver_port == 0: return 'skip' with gdaltest.error_handler(): f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3', 'wb') if f is not None: gdaltest.post_reason('fail') return 'fail' if gdal.VSIStatL('/vsis3/s3_fake_bucket3/empty_file.bin').size != 3: gdaltest.post_reason('fail') return 'fail' # Empty file f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' gdal.ErrorReset() gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() != '': gdaltest.post_reason('fail') return 'fail' if gdal.VSIStatL('/vsis3/s3_fake_bucket3/empty_file.bin').size != 0: gdaltest.post_reason('fail') return 'fail' # Invalid seek f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): ret = gdal.VSIFSeekL(f, 1, 0) if ret == 0: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) # Invalid read f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' with gdaltest.error_handler(): ret = gdal.VSIFReadL(1, 1, f) if len(ret) != 0: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(f) # Error case f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/empty_file_error.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' gdal.ErrorReset() with gdaltest.error_handler(): gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() == '': gdaltest.post_reason('fail') return 'fail' # Nominal case f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/another_file.bin', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) != 0: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFSeekL(f, 0, 1) != 0: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFSeekL(f, 0, 2) != 0: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFWriteL('foo', 1, 3, f) != 3: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFWriteL('bar', 1, 3, f) != 3: gdaltest.post_reason('fail') return 'fail' gdal.ErrorReset() gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() != '': gdaltest.post_reason('fail') return 'fail' # Redirect case f = gdal.VSIFOpenL('/vsis3/s3_fake_bucket3/redirect', 'wb') if f is None: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFWriteL('foobar', 1, 6, f) != 6: gdaltest.post_reason('fail') return 'fail' gdal.ErrorReset() gdal.VSIFCloseL(f) if gdal.GetLastErrorMsg() != '': gdaltest.post_reason('fail') return 'fail' return 'success'
def gdal_cp_single(srcfile, targetfile, progress): if targetfile.endswith('/'): stat_res = gdal.VSIStatL(targetfile) else: stat_res = gdal.VSIStatL(targetfile + '/') if (stat_res is None and targetfile.endswith('/')) or \ (stat_res is not None and stat.S_ISDIR(stat_res.mode)): (_, tail) = os.path.split(srcfile) if targetfile.endswith('/'): targetfile = targetfile + tail else: targetfile = targetfile + '/' + tail fin = gdal.VSIFOpenL(srcfile, "rb") if fin is None: print('Cannot open %s' % srcfile) return -1 fout = gdal.VSIFOpenL(targetfile, "wb") if fout is None: print('Cannot create %s' % targetfile) gdal.VSIFCloseL(fin) return -1 version_num = int(gdal.VersionInfo('VERSION_NUM')) total_size = 0 if version_num < 1900 or progress is not None: gdal.VSIFSeekL(fin, 0, 2) total_size = gdal.VSIFTellL(fin) gdal.VSIFSeekL(fin, 0, 0) buf_max_size = 4096 copied = 0 ret = 0 # print('Copying %s...' % srcfile) if progress is not None: if not progress.Progress(0.0, 'Copying %s' % srcfile): print('Copy stopped by user') ret = -2 while ret == 0: if total_size != 0 and copied + buf_max_size > total_size: to_read = total_size - copied else: to_read = buf_max_size buf = gdal.VSIFReadL(1, to_read, fin) if buf is None: if copied == 0: print('Cannot read %d bytes in %s' % (to_read, srcfile)) ret = -1 break buf_size = len(buf) if gdal.VSIFWriteL(buf, 1, buf_size, fout) != buf_size: print('Error writing %d bytes' % buf_size) ret = -1 break copied += buf_size if progress is not None and total_size != 0: if not progress.Progress(copied * 1.0 / total_size, 'Copying %s' % srcfile): print('Copy stopped by user') ret = -2 break if to_read < buf_max_size or buf_size != buf_max_size: break gdal.VSIFCloseL(fin) gdal.VSIFCloseL(fout) return ret
def full_check_band(f, band_name, band, errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, mask_interleaved_with_imagery): block_size = band.GetBlockSize() mask_band = None if mask_interleaved_with_imagery: mask_band = band.GetMaskBand() mask_block_size = mask_band.GetBlockSize() if block_size != mask_block_size: errors += [ band_name + ': mask block size is different from its imagery band' ] mask_band = None yblocks = (band.YSize + block_size[1] - 1) // block_size[1] xblocks = (band.XSize + block_size[0] - 1) // block_size[0] last_offset = 0 for y in range(yblocks): for x in range(xblocks): offset = int( band.GetMetadataItem('BLOCK_OFFSET_%d_%d' % (x, y), 'TIFF')) bytecount = int( band.GetMetadataItem('BLOCK_SIZE_%d_%d' % (x, y), 'TIFF')) if block_order_row_major and offset < last_offset: errors += [ band_name + ': offset of block (%d, %d) is smaller than previous block' % (x, y) ] if block_leader_size_as_uint4: gdal.VSIFSeekL(f, offset - 4, 0) leader_size = struct.unpack('<I', gdal.VSIFReadL(4, 1, f))[0] if leader_size != bytecount: errors += [ band_name + ': for block (%d, %d), size in leader bytes is %d instead of %d' % (x, y, leader_size, bytecount) ] if block_trailer_last_4_bytes_repeated: if bytecount >= 4: gdal.VSIFSeekL(f, offset + bytecount - 4, 0) last_bytes = gdal.VSIFReadL(8, 1, f) if last_bytes[0:4] != last_bytes[4:8]: errors += [ band_name + ': for block (%d, %d), trailer bytes are invalid' % (x, y) ] if mask_band: offset_mask = int( mask_band.GetMetadataItem('BLOCK_OFFSET_%d_%d' % (x, y), 'TIFF')) #bytecount_mask = int(mask_band.GetMetadataItem('BLOCK_SIZE_%d_%d' % (x,y), 'TIFF')) expected_offset_mask = offset + bytecount + \ (4 if block_leader_size_as_uint4 else 0) + \ (4 if block_trailer_last_4_bytes_repeated else 0) if offset_mask != expected_offset_mask: errors += [ 'Mask of ' + band_name + ': for block (%d, %d), offset is %d, whereas %d was expected' % (x, y, offset_mask, expected_offset_mask) ] last_offset = offset
def extract_tile(ds, src_band_nbr, tile_x, tile_y, jpg_filename): block_offset = ds.GetRasterBand(src_band_nbr).GetMetadataItem( 'BLOCK_OFFSET_%d_%d' % (tile_x, tile_y), 'TIFF') block_size = ds.GetRasterBand(src_band_nbr).GetMetadataItem( 'BLOCK_SIZE_%d_%d' % (tile_x, tile_y), 'TIFF') if block_offset is None or block_size is None: print('ERROR: Cannot find block (%d,%d)' % (tile_x, tile_y)) return 1 jpegtables = ds.GetRasterBand(src_band_nbr).GetMetadataItem( 'JPEGTABLES', 'TIFF') if jpegtables is not None: if (len(jpegtables) % 2 ) != 0 or jpegtables[0:4] != 'FFD8' or jpegtables[-2:] != 'D9': print('ERROR: Invalid JPEG tables') print(jpegtables) return 1 # Remove final D9 jpegtables = jpegtables[0:-2] tiff_f = gdal.VSIFOpenL(ds.GetDescription(), 'rb') if tiff_f is None: print('ERROR: Cannot reopen %s' % ds.GetDescription()) return 1 out_f = gdal.VSIFOpenL(jpg_filename, 'wb') if out_f is None: print('ERROR: Cannot create %s' % jpg_filename) gdal.VSIFCloseL(tiff_f) return 1 # Write JPEG tables if jpegtables is not None: for i in range(int(len(jpegtables) / 2)): c1 = ord(jpegtables[2 * i]) c2 = ord(jpegtables[2 * i + 1]) if c1 >= ord('0') and c1 <= ord('9'): val = c1 - ord('0') else: val = (c1 - ord('A')) + 10 val = val * 16 if c2 >= ord('0') and c2 <= ord('9'): val = val + (c2 - ord('0')) else: val = val + (c2 - ord('A')) + 10 gdal.VSIFWriteL(chr(val), 1, 1, out_f) else: gdal.VSIFWriteL(chr(0xFF), 1, 1, out_f) gdal.VSIFWriteL(chr(0xD8), 1, 1, out_f) # Write Adobe APP14 marker if necessary interleave = ds.GetMetadataItem('INTERLEAVE', 'IMAGE_STRUCTURE') photometric = ds.GetMetadataItem('COMPRESSION', 'IMAGE_STRUCTURE') if interleave == 'PIXEL' and photometric == 'JPEG' and ds.RasterCount == 3: adobe_app14 = [ 0xFF, 0xEE, 0x00, 0x0E, 0x41, 0x64, 0x6F, 0x62, 0x65, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00 ] for c in adobe_app14: gdal.VSIFWriteL(chr(c), 1, 1, out_f) # Write JPEG codestream # skip leading 0xFF 0xD8 gdal.VSIFSeekL(tiff_f, int(block_offset) + 2, 0) data = gdal.VSIFReadL(1, int(block_size) - 2, tiff_f) gdal.VSIFCloseL(tiff_f) gdal.VSIFWriteL(data, 1, len(data), out_f) gdal.VSIFCloseL(out_f) aux_xml_filename = '%s.aux.xml' % jpg_filename gt = ds.GetGeoTransform() srs = ds.GetProjectionRef() if srs is not None and srs != '': sub_gt = [gt[i] for i in range(6)] (blockxsize, blockysize) = ds.GetRasterBand(1).GetBlockSize() sub_gt[0] = gt[0] + tile_x * blockxsize * gt[1] sub_gt[3] = gt[3] + tile_y * blockysize * gt[5] out_f = gdal.VSIFOpenL(aux_xml_filename, 'wb') if out_f is None: print('ERROR: Cannot create %s' % aux_xml_filename) return 1 content = """<PAMDataset> <SRS>%s</SRS> <GeoTransform>%.18g,%.18g,%.18g,%.18g,%.18g,%.18g</GeoTransform> </PAMDataset> """ % (srs, sub_gt[0], sub_gt[1], sub_gt[2], sub_gt[3], sub_gt[4], sub_gt[5]) gdal.VSIFWriteL(content, 1, len(content), out_f) gdal.VSIFCloseL(out_f) else: gdal.Unlink('%s.aux.xml' % jpg_filename) return 0
def mvt(request): if not MVT_DRIVER_EXIST: return HTTPNotFound(explanation='MVT GDAL driver not found') z = int(request.GET["z"]) x = int(request.GET["x"]) y = int(request.GET["y"]) extent = int(request.GET.get('extent', 4096)) simplification = float(request.GET.get("simplification", extent / 512)) resids = map( int, filter(None, request.GET["resource"].split(",")), ) # web mercator merc = SRS.filter_by(id=3857).one() minx, miny, maxx, maxy = merc.tile_extent((z, x, y)) # 5% padding by default padding = float(request.GET.get("padding", 0.05)) bbox = ( minx - (maxx - minx) * padding, miny - (maxy - miny) * padding, maxx + (maxx - minx) * padding, maxy + (maxy - miny) * padding, ) bbox = Geometry.from_shape(box(*bbox), srid=merc.id) options = [ "FORMAT=DIRECTORY", "TILE_EXTENSION=pbf", "MINZOOM=%d" % z, "MAXZOOM=%d" % z, "EXTENT=%d" % extent, "COMPRESS=NO", ] ds = _ogr_ds("MVT", options) vsibuf = ds.GetName() for resid in resids: try: obj = Resource.filter_by(id=resid).one() except NoResultFound: raise ResourceNotFound(resid) request.resource_permission(PERM_READ, obj) query = obj.feature_query() query.intersects(bbox) query.geom() if IFeatureQueryClipByBox.providedBy(query): query.clip_by_box(bbox) if IFeatureQuerySimplify.providedBy(query): tolerance = ((obj.srs.maxx - obj.srs.minx) / (1 << z)) / extent query.simplify(tolerance * simplification) _ogr_layer_from_features(obj, query(), name="ngw:%d" % obj.id, ds=ds) # flush changes ds = None filepath = os.path.join("%s" % vsibuf, "%d" % z, "%d" % x, "%d.pbf" % y) try: f = gdal.VSIFOpenL(ensure_str(filepath), ensure_str("rb")) if f is not None: # SEEK_END = 2 gdal.VSIFSeekL(f, 0, 2) size = gdal.VSIFTellL(f) # SEEK_SET = 0 gdal.VSIFSeekL(f, 0, 0) content = gdal.VSIFReadL(1, size, f) gdal.VSIFCloseL(f) return Response( content, content_type="application/vnd.mapbox-vector-tile", ) else: return HTTPNoContent() finally: gdal.Unlink(ensure_str(vsibuf))
def validate(ds, check_tiled=True, full_check=False): """Check if a file is a (Geo)TIFF with cloud optimized compatible structure. Args: ds: GDAL Dataset for the file to inspect. check_tiled: Set to False to ignore missing tiling. full_check: Set to TRUe to check tile/strip leader/trailer bytes. Might be slow on remote files Returns: A tuple, whose first element is an array of error messages (empty if there is no error), and the second element, a dictionary with the structure of the GeoTIFF file. Raises: ValidateCloudOptimizedGeoTIFFException: Unable to open the file or the file is not a Tiff. """ if int(gdal.VersionInfo('VERSION_NUM')) < 2020000: raise ValidateCloudOptimizedGeoTIFFException( 'GDAL 2.2 or above required') unicode_type = type(''.encode('utf-8').decode('utf-8')) if isinstance(ds, (str, unicode_type)): gdal.PushErrorHandler() ds = gdal.Open(ds) gdal.PopErrorHandler() if ds is None: raise ValidateCloudOptimizedGeoTIFFException( 'Invalid file : %s' % gdal.GetLastErrorMsg()) if ds.GetDriver().ShortName != 'GTiff': raise ValidateCloudOptimizedGeoTIFFException( 'The file is not a GeoTIFF') details = {} errors = [] warnings = [] filename = ds.GetDescription() main_band = ds.GetRasterBand(1) ovr_count = main_band.GetOverviewCount() filelist = ds.GetFileList() if filelist is not None and filename + '.ovr' in filelist: errors += [ 'Overviews found in external .ovr file. They should be internal' ] if main_band.XSize > 512 or main_band.YSize > 512: if check_tiled: block_size = main_band.GetBlockSize() if block_size[0] == main_band.XSize and block_size[0] > 1024: errors += [ 'The file is greater than 512xH or Wx512, but is not tiled' ] if ovr_count == 0: warnings += [ 'The file is greater than 512xH or Wx512, it is recommended ' 'to include internal overviews' ] ifd_offset = int(main_band.GetMetadataItem('IFD_OFFSET', 'TIFF')) ifd_offsets = [ifd_offset] block_order_row_major = False block_leader_size_as_uint4 = False block_trailer_last_4_bytes_repeated = False mask_interleaved_with_imagery = False if ifd_offset not in (8, 16): # Check if there is GDAL hidden structural metadata f = gdal.VSIFOpenL(filename, 'rb') if not f: raise ValidateCloudOptimizedGeoTIFFException("Cannot open file") signature = struct.unpack('B' * 4, gdal.VSIFReadL(4, 1, f)) bigtiff = signature in ((0x49, 0x49, 0x2B, 0x00), (0x4D, 0x4D, 0x00, 0x2B)) if bigtiff: expected_ifd_pos = 16 else: expected_ifd_pos = 8 gdal.VSIFSeekL(f, expected_ifd_pos, 0) pattern = "GDAL_STRUCTURAL_METADATA_SIZE=%06d bytes\n" % 0 got = gdal.VSIFReadL(len(pattern), 1, f).decode('LATIN1') if len(got) == len(pattern) and got.startswith( 'GDAL_STRUCTURAL_METADATA_SIZE='): size = int(got[len('GDAL_STRUCTURAL_METADATA_SIZE='):][0:6]) extra_md = gdal.VSIFReadL(size, 1, f).decode('LATIN1') block_order_row_major = 'BLOCK_ORDER=ROW_MAJOR' in extra_md block_leader_size_as_uint4 = 'BLOCK_LEADER=SIZE_AS_UINT4' in extra_md block_trailer_last_4_bytes_repeated = 'BLOCK_TRAILER=LAST_4_BYTES_REPEATED' in extra_md mask_interleaved_with_imagery = 'MASK_INTERLEAVED_WITH_IMAGERY=YES' in extra_md if 'KNOWN_INCOMPATIBLE_EDITION=YES' in extra_md: errors += [ "KNOWN_INCOMPATIBLE_EDITION=YES is declared in the file" ] expected_ifd_pos += len(pattern) + size expected_ifd_pos += expected_ifd_pos % 2 # IFD offset starts on a 2-byte boundary gdal.VSIFCloseL(f) if expected_ifd_pos != ifd_offsets[0]: errors += [ 'The offset of the main IFD should be %d. It is %d instead' % (expected_ifd_pos, ifd_offsets[0]) ] details['ifd_offsets'] = {} details['ifd_offsets']['main'] = ifd_offset for i in range(ovr_count): # Check that overviews are by descending sizes ovr_band = ds.GetRasterBand(1).GetOverview(i) if i == 0: if (ovr_band.XSize > main_band.XSize or ovr_band.YSize > main_band.YSize): errors += [ 'First overview has larger dimension than main band' ] else: prev_ovr_band = ds.GetRasterBand(1).GetOverview(i - 1) if (ovr_band.XSize > prev_ovr_band.XSize or ovr_band.YSize > prev_ovr_band.YSize): errors += [ 'Overview of index %d has larger dimension than ' 'overview of index %d' % (i, i - 1) ] if check_tiled: block_size = ovr_band.GetBlockSize() if block_size[0] == ovr_band.XSize and block_size[0] > 1024: errors += ['Overview of index %d is not tiled' % i] # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int(ovr_band.GetMetadataItem('IFD_OFFSET', 'TIFF')) ifd_offsets.append(ifd_offset) details['ifd_offsets']['overview_%d' % i] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if i == 0: errors += [ 'The offset of the IFD for overview of index %d is %d, ' 'whereas it should be greater than the one of the main ' 'image, which is at byte %d' % (i, ifd_offsets[-1], ifd_offsets[-2]) ] else: errors += [ 'The offset of the IFD for overview of index %d is %d, ' 'whereas it should be greater than the one of index %d, ' 'which is at byte %d' % (i, ifd_offsets[-1], i - 1, ifd_offsets[-2]) ] # Check that the imagery starts by the smallest overview and ends with # the main resolution dataset block_offset = main_band.GetMetadataItem('BLOCK_OFFSET_0_0', 'TIFF') if not block_offset: errors += ['Missing BLOCK_OFFSET_0_0'] data_offset = int(block_offset) if block_offset else None data_offsets = [data_offset] details['data_offsets'] = {} details['data_offsets']['main'] = data_offset for i in range(ovr_count): ovr_band = ds.GetRasterBand(1).GetOverview(i) data_offset = int(ovr_band.GetMetadataItem('BLOCK_OFFSET_0_0', 'TIFF')) data_offsets.append(data_offset) details['data_offsets']['overview_%d' % i] = data_offset if data_offsets[-1] < ifd_offsets[-1]: if ovr_count > 0: errors += [ 'The offset of the first block of the smallest overview ' 'should be after its IFD' ] else: errors += [ 'The offset of the first block of the image should ' 'be after its IFD' ] for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: errors += [ 'The offset of the first block of overview of index %d should ' 'be after the one of the overview of index %d' % (i - 1, i) ] if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: errors += [ 'The offset of the first block of the main resolution image' 'should be after the one of the overview of index %d' % (ovr_count - 1) ] if full_check and (block_order_row_major or block_leader_size_as_uint4 or \ block_trailer_last_4_bytes_repeated or \ mask_interleaved_with_imagery): f = gdal.VSIFOpenL(filename, 'rb') if not f: raise ValidateCloudOptimizedGeoTIFFException("Cannot open file") full_check_band(f, 'Main resolution image', main_band, errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, mask_interleaved_with_imagery) if main_band.GetMaskFlags() == gdal.GMF_PER_DATASET and \ (filename + '.msk') not in ds.GetFileList(): full_check_band(f, 'Mask band of main resolution image', main_band.GetMaskBand(), errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, False) for i in range(ovr_count): ovr_band = ds.GetRasterBand(1).GetOverview(i) full_check_band(f, 'Overview %d' % i, ovr_band, errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, mask_interleaved_with_imagery) if ovr_band.GetMaskFlags() == gdal.GMF_PER_DATASET and \ (filename + '.msk') not in ds.GetFileList(): full_check_band(f, 'Mask band of overview %d' % i, ovr_band.GetMaskBand(), errors, block_order_row_major, block_leader_size_as_uint4, block_trailer_last_4_bytes_repeated, False) gdal.VSIFCloseL(f) return warnings, errors, details
def test_vsifile_5(): fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'wb') ref_data = ''.join(['%08X' % i for i in range(5 * 32768)]) gdal.VSIFWriteL(ref_data, 1, len(ref_data), fp) gdal.VSIFCloseL(fp) gdal.SetConfigOption('VSI_CACHE', 'YES') for i in range(3): if i == 0: gdal.SetConfigOption('VSI_CACHE_SIZE', '0') elif i == 1: gdal.SetConfigOption('VSI_CACHE_SIZE', '65536') else: gdal.SetConfigOption('VSI_CACHE_SIZE', None) fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'rb') gdal.VSIFSeekL(fp, 50000, 0) if gdal.VSIFTellL(fp) != 50000: gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) pytest.fail() gdal.VSIFSeekL(fp, 50000, 1) if gdal.VSIFTellL(fp) != 100000: gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) pytest.fail() gdal.VSIFSeekL(fp, 0, 2) if gdal.VSIFTellL(fp) != 5 * 32768 * 8: gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) pytest.fail() gdal.VSIFReadL(1, 1, fp) gdal.VSIFSeekL(fp, 0, 0) data = gdal.VSIFReadL(1, 3 * 32768, fp) if data.decode('ascii') != ref_data[0:3 * 32768]: gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) pytest.fail() gdal.VSIFSeekL(fp, 16384, 0) data = gdal.VSIFReadL(1, 5 * 32768, fp) if data.decode('ascii') != ref_data[16384:16384 + 5 * 32768]: gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) pytest.fail() data = gdal.VSIFReadL(1, 50 * 32768, fp) if data[0:1130496].decode('ascii') != ref_data[16384 + 5 * 32768:]: gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) pytest.fail() gdal.VSIFCloseL(fp) gdal.SetConfigOption('VSI_CACHE_SIZE', None) gdal.SetConfigOption('VSI_CACHE', None) gdal.Unlink('tmp/vsifile_5.bin')
def vsifile_generic(filename): start_time = time.time() fp = gdal.VSIFOpenL(filename, 'wb+') assert fp is not None assert gdal.VSIFWriteL('0123456789', 1, 10, fp) == 10 assert gdal.VSIFFlushL(fp) == 0 assert gdal.VSIFTruncateL(fp, 20) == 0 assert gdal.VSIFTellL(fp) == 10 assert gdal.VSIFTruncateL(fp, 5) == 0 assert gdal.VSIFTellL(fp) == 10 assert gdal.VSIFSeekL(fp, 0, 2) == 0 assert gdal.VSIFTellL(fp) == 5 gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) assert statBuf.size == 7 assert start_time == pytest.approx(statBuf.mtime, abs=2) fp = gdal.VSIFOpenL(filename, 'rb') buf = gdal.VSIFReadL(1, 7, fp) assert gdal.VSIFWriteL('a', 1, 1, fp) == 0 assert gdal.VSIFTruncateL(fp, 0) != 0 gdal.VSIFCloseL(fp) assert buf.decode('ascii') == '01234XX' # Test append mode on existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) assert statBuf.size == 9 assert gdal.Unlink(filename) == 0 statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG) assert statBuf is None # Test append mode on non existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) assert statBuf.size == 2 assert gdal.Unlink(filename) == 0
def test_vsicurl_test_redirect(): if gdaltest.is_travis_branch('trusty'): pytest.skip('Skipped on trusty branch, but should be investigated') if gdaltest.webserver_port == 0: pytest.skip() gdal.VSICurlClearCache() handler = webserver.SequentialHandler() handler.add('GET', '/test_redirect/', 404) # Simulate a big time difference between server and local machine current_time = 1500 def method(request): response = 'HTTP/1.1 302\r\n' response += 'Server: foo\r\n' response += 'Date: ' + time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n' response += 'Location: %s\r\n' % ( 'http://localhost:%d/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (gdaltest.webserver_port, current_time + 30)) response += '\r\n' request.wfile.write(response.encode('ascii')) handler.add('HEAD', '/test_redirect/test.bin', custom_method=method) handler.add( 'HEAD', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), 403, {'Server': 'foo'}, '') def method(request): if 'Range' in request.headers: if request.headers['Range'] == 'bytes=0-16383': request.protocol_version = 'HTTP/1.1' request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Range', 'bytes 0-16383/1000000') request.send_header('Content-Length', 16384) request.send_header('Connection', 'close') request.end_headers() request.wfile.write(('x' * 16384).encode('ascii')) elif request.headers['Range'] == 'bytes=16384-49151': # Test expiration of the signed URL request.protocol_version = 'HTTP/1.1' request.send_response(403) request.send_header('Content-Length', 0) request.end_headers() else: request.send_response(404) request.send_header('Content-Length', 0) request.end_headers() else: # After a failed attempt on a HEAD, the client should go there response = 'HTTP/1.1 200\r\n' response += 'Server: foo\r\n' response += 'Date: ' + time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n' response += 'Content-type: text/plain\r\n' response += 'Content-Length: 1000000\r\n' response += 'Connection: close\r\n' response += '\r\n' request.wfile.write(response.encode('ascii')) handler.add( 'GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) with webserver.install_http_handler(handler): f = gdal.VSIFOpenL( '/vsicurl/http://localhost:%d/test_redirect/test.bin' % gdaltest.webserver_port, 'rb') assert f is not None gdal.VSIFSeekL(f, 0, 2) if gdal.VSIFTellL(f) != 1000000: gdal.VSIFCloseL(f) pytest.fail(gdal.VSIFTellL(f)) gdal.VSIFSeekL(f, 0, 0) handler = webserver.SequentialHandler() handler.add( 'GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) handler.add( 'GET', '/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) current_time = int(time.time()) def method(request): # We should go there after expiration of the first signed URL if 'Range' in request.headers and \ request.headers['Range'] == 'bytes=16384-49151': request.protocol_version = 'HTTP/1.1' request.send_response(302) # Return a new signed URL request.send_header( 'Location', 'http://localhost:%d/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d' % (request.server.port, current_time + 30)) request.send_header('Content-Length', 16384) request.end_headers() request.wfile.write(('x' * 16384).encode('ascii')) handler.add('GET', '/test_redirect/test.bin', custom_method=method) def method(request): # Second signed URL if 'Range' in request.headers and \ request.headers['Range'] == 'bytes=16384-49151': request.protocol_version = 'HTTP/1.1' request.send_response(200) request.send_header('Content-type', 'text/plain') request.send_header('Content-Range', 'bytes 16384-16384/1000000') request.send_header('Content-Length', 1) request.end_headers() request.wfile.write('y'.encode('ascii')) handler.add( 'GET', '/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d' % (current_time + 30), custom_method=method) with webserver.install_http_handler(handler): content = gdal.VSIFReadL(1, 16383, f).decode('ascii') if len(content) != 16383 or content[0] != 'x': gdal.VSIFCloseL(f) pytest.fail(content) content = gdal.VSIFReadL(1, 2, f).decode('ascii') if content != 'xy': gdal.VSIFCloseL(f) pytest.fail(content) gdal.VSIFCloseL(f)
def vsifile_generic(filename): start_time = time.time() fp = gdal.VSIFOpenL(filename, 'wb+') if fp is None: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFWriteL('0123456789', 1, 10, fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTruncateL(fp, 20) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTruncateL(fp, 5) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 10: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFSeekL(fp, 0, 2) != 0: gdaltest.post_reason('failure') return 'fail' if gdal.VSIFTellL(fp) != 5: gdaltest.post_reason('failure') return 'fail' gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 7: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if abs(start_time - statBuf.mtime) > 2: gdaltest.post_reason('failure') print(statBuf.mtime) return 'fail' fp = gdal.VSIFOpenL(filename, 'rb') buf = gdal.VSIFReadL(1, 7, fp) if gdal.VSIFWriteL('a', 1, 1, fp) != 0: gdaltest.post_reason('fail') return 'fail' if gdal.VSIFTruncateL(fp, 0) == 0: gdaltest.post_reason('fail') return 'fail' gdal.VSIFCloseL(fp) if buf.decode('ascii') != '01234XX': gdaltest.post_reason('failure') print(buf.decode('ascii')) return 'fail' # Test append mode on existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 9: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if gdal.Unlink(filename) != 0: gdaltest.post_reason('failure') return 'fail' statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG) if statBuf is not None: gdaltest.post_reason('failure') return 'fail' # Test append mode on non existing file fp = gdal.VSIFOpenL(filename, 'ab') gdal.VSIFWriteL('XX', 1, 2, fp) gdal.VSIFCloseL(fp) statBuf = gdal.VSIStatL( filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG | gdal.VSI_STAT_SIZE_FLAG) if statBuf.size != 2: gdaltest.post_reason('failure') print(statBuf.size) return 'fail' if gdal.Unlink(filename) != 0: gdaltest.post_reason('failure') return 'fail' return 'success'
def vsicurl_streaming_1(): try: drv = gdal.GetDriverByName('HTTP') except: drv = None if drv is None: return 'skip' fp = gdal.VSIFOpenL( '/vsicurl_streaming/http://download.osgeo.org/gdal/data/usgsdem/cded/114p01_0100_deme.dem', 'rb') if fp is None: if gdaltest.gdalurlopen( 'http://download.osgeo.org/gdal/data/usgsdem/cded/114p01_0100_deme.dem' ) is None: print('cannot open URL') return 'skip' gdaltest.post_reason('fail') return 'fail' if gdal.VSIFTellL(fp) != 0: gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' data = gdal.VSIFReadL(1, 50, fp) if data.decode( 'ascii') != ' 114p01DEMe Base Ma': gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' if gdal.VSIFTellL(fp) != 50: gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' gdal.VSIFSeekL(fp, 0, 0) if gdal.VSIFTellL(fp) != 0: gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' data = gdal.VSIFReadL(1, 50, fp) if data.decode( 'ascii') != ' 114p01DEMe Base Ma': gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' if gdal.VSIFTellL(fp) != 50: gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' time.sleep(0.5) gdal.VSIFSeekL(fp, 2001, 0) data_2001 = gdal.VSIFReadL(1, 20, fp) if data_2001.decode('ascii') != '7-32767-32767-32767-': print(data_2001) gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' if gdal.VSIFTellL(fp) != 2001 + 20: gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' gdal.VSIFSeekL(fp, 0, 2) if gdal.VSIFTellL(fp) != 9839616: gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' nRet = len(gdal.VSIFReadL(1, 10, fp)) if nRet != 0: gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' gdal.VSIFSeekL(fp, 2001, 0) data_2001_2 = gdal.VSIFReadL(1, 20, fp) if gdal.VSIFTellL(fp) != 2001 + 20: gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' if data_2001 != data_2001_2: gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' gdal.VSIFSeekL(fp, 1024 * 1024 + 100, 0) data = gdal.VSIFReadL(1, 20, fp) if data.decode('ascii') != '67-32767-32767-32767': print(data) gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' if gdal.VSIFTellL(fp) != 1024 * 1024 + 100 + 20: gdaltest.post_reason('fail') gdal.VSIFCloseL(fp) return 'fail' gdal.VSIFCloseL(fp) return 'success'
def vsicrypt_4(): if not gdaltest.has_vsicrypt: return 'skip' test_file = '/vsicrypt/key=DONT_USE_IN_PROD,sector_size=32,file=/vsimem/file_enc.bin' ref_file = '/vsimem/file.bin' for seed in range(1000): gdal.Unlink(test_file) gdal.Unlink(ref_file) test_f = gdal.VSIFOpenL(test_file, 'wb+') ref_f = gdal.VSIFOpenL(ref_file, 'wb+') import random random.seed(seed) for i in range(20): random_offset = random.randint(0, 1000) gdal.VSIFSeekL(test_f, random_offset, 0) gdal.VSIFSeekL(ref_f, random_offset, 0) random_size = random.randint(1, 80) random_content = ''.join([ chr(40 + int(10 * random.random())) for i in range(random_size) ]) gdal.VSIFWriteL(random_content, 1, random_size, test_f) gdal.VSIFWriteL(random_content, 1, random_size, ref_f) if random.randint(0, 1) == 0: random_offset = random.randint(0, 1500) gdal.VSIFSeekL(test_f, random_offset, 0) gdal.VSIFSeekL(ref_f, random_offset, 0) random_size = random.randint(1, 80) test_content = gdal.VSIFReadL(1, random_size, test_f) ref_content = gdal.VSIFReadL(1, random_size, ref_f) if test_content != ref_content: print(seed) print('Test content (%d):' % len(test_content)) print(test_content) print('') print('Ref content (%d):' % len(ref_content)) print(ref_content) return 'fail' gdal.VSIFSeekL(test_f, 0, 0) gdal.VSIFSeekL(ref_f, 0, 0) test_content = gdal.VSIFReadL(1, 100000, test_f) ref_content = gdal.VSIFReadL(1, 100000, ref_f) if test_content != ref_content: print(seed) print('Test content (%d):' % len(test_content)) print(test_content) print('') print('Ref content (%d):' % len(ref_content)) print(ref_content) return 'fail' gdal.Unlink(test_file) gdal.Unlink(ref_file) return 'success'
def test_visoss_4(): if gdaltest.webserver_port == 0: pytest.skip() with webserver.install_http_handler(webserver.SequentialHandler()): with gdaltest.error_handler(): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3', 'wb') assert f is None handler = webserver.SequentialHandler() handler.add('GET', '/oss_fake_bucket3/empty_file.bin', 200, {'Connection': 'close'}, 'foo') with webserver.install_http_handler(handler): assert gdal.VSIStatL( '/vsioss/oss_fake_bucket3/empty_file.bin').size == 3 # Empty file handler = webserver.SequentialHandler() def method(request): if request.headers['Content-Length'] != '0': sys.stderr.write('Did not get expected headers: %s\n' % str(request.headers)) request.send_response(400) return request.send_response(200) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/oss_fake_bucket3/empty_file.bin', custom_method=method) with webserver.install_http_handler(handler): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb') assert f is not None gdal.ErrorReset() gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() == '' handler = webserver.SequentialHandler() handler.add('GET', '/oss_fake_bucket3/empty_file.bin', 200, {'Connection': 'close'}, '') with webserver.install_http_handler(handler): assert gdal.VSIStatL( '/vsioss/oss_fake_bucket3/empty_file.bin').size == 0 # Invalid seek handler = webserver.SequentialHandler() with webserver.install_http_handler(handler): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb') assert f is not None with gdaltest.error_handler(): ret = gdal.VSIFSeekL(f, 1, 0) assert ret != 0 gdal.VSIFCloseL(f) # Invalid read handler = webserver.SequentialHandler() with webserver.install_http_handler(handler): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file.bin', 'wb') assert f is not None with gdaltest.error_handler(): ret = gdal.VSIFReadL(1, 1, f) assert not ret gdal.VSIFCloseL(f) # Error case handler = webserver.SequentialHandler() handler.add('PUT', '/oss_fake_bucket3/empty_file_error.bin', 403) with webserver.install_http_handler(handler): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/empty_file_error.bin', 'wb') assert f is not None gdal.ErrorReset() with gdaltest.error_handler(): gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() != '' # Nominal case with webserver.install_http_handler(webserver.SequentialHandler()): f = gdal.VSIFOpenL('/vsioss/oss_fake_bucket3/another_file.bin', 'wb') assert f is not None assert gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) == 0 assert gdal.VSIFSeekL(f, 0, 1) == 0 assert gdal.VSIFSeekL(f, 0, 2) == 0 assert gdal.VSIFWriteL('foo', 1, 3, f) == 3 assert gdal.VSIFSeekL(f, gdal.VSIFTellL(f), 0) == 0 assert gdal.VSIFWriteL('bar', 1, 3, f) == 3 handler = webserver.SequentialHandler() def method(request): if request.headers['Content-Length'] != '6': sys.stderr.write('Did not get expected headers: %s\n' % str(request.headers)) request.send_response(400) request.send_header('Content-Length', 0) request.end_headers() return request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii')) content = request.rfile.read(6).decode('ascii') if content != 'foobar': sys.stderr.write('Did not get expected content: %s\n' % content) request.send_response(400) request.send_header('Content-Length', 0) request.end_headers() return request.send_response(200) request.send_header('Content-Length', 0) request.end_headers() handler.add('PUT', '/oss_fake_bucket3/another_file.bin', custom_method=method) gdal.ErrorReset() with webserver.install_http_handler(handler): gdal.VSIFCloseL(f) assert gdal.GetLastErrorMsg() == ''