def ogr_pgdump_8():
ds = ogr.GetDriverByName('PGDump').CreateDataSource(
'/vsimem/ogr_pgdump_8.sql', options=['LINEFORMAT=LF'])
lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone, options=['FID=myfid'])
lyr.CreateField(ogr.FieldDefn('str', ogr.OFTString))
gdal.PushErrorHandler()
ret = lyr.CreateField(ogr.FieldDefn('myfid', ogr.OFTString))
gdal.PopErrorHandler()
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
ret = lyr.CreateField(ogr.FieldDefn('myfid', ogr.OFTInteger))
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
lyr.CreateField(ogr.FieldDefn('str2', ogr.OFTString))
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', 'first string')
feat.SetField('myfid', 10)
feat.SetField('str2', 'second string')
gdal.SetConfigOption('PG_USE_COPY', 'YES')
ret = lyr.CreateFeature(feat)
gdal.SetConfigOption('PG_USE_COPY', None)
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
if feat.GetFID() != 10:
gdaltest.post_reason('fail')
return 'fail'
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str2', 'second string')
gdal.SetConfigOption('PG_USE_COPY', 'YES')
ret = lyr.CreateFeature(feat)
gdal.SetConfigOption('PG_USE_COPY', None)
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
if feat.GetFID() < 0:
gdaltest.post_reason('fail')
feat.DumpReadable()
return 'fail'
if feat.GetField('myfid') != feat.GetFID():
gdaltest.post_reason('fail')
feat.DumpReadable()
return 'fail'
#feat.SetField('str', 'foo')
#ret = lyr.SetFeature(feat)
#if ret != 0:
# gdaltest.post_reason('fail')
# return 'fail'
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetFID(1)
feat.SetField('myfid', 10)
gdal.PushErrorHandler()
gdal.SetConfigOption('PG_USE_COPY', 'YES')
ret = lyr.CreateFeature(feat)
gdal.SetConfigOption('PG_USE_COPY', None)
gdal.PopErrorHandler()
if ret == 0:
gdaltest.post_reason('fail')
return 'fail'
#gdal.PushErrorHandler()
#ret = lyr.SetFeature(feat)
#gdal.PopErrorHandler()
#if ret == 0:
# gdaltest.post_reason('fail')
# return 'fail'
#feat.UnsetField('myfid')
#gdal.PushErrorHandler()
#ret = lyr.SetFeature(feat)
#gdal.PopErrorHandler()
#if ret == 0:
# gdaltest.post_reason('fail')
# return 'fail'
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', 'first string')
feat.SetField('myfid', 12)
feat.SetField('str2', 'second string')
gdal.SetConfigOption('PG_USE_COPY', 'YES')
ret = lyr.CreateFeature(feat)
gdal.SetConfigOption('PG_USE_COPY', None)
if ret != 0:
gdaltest.post_reason('fail')
return 'fail'
if feat.GetFID() != 12:
gdaltest.post_reason('fail')
return 'fail'
ds = None
f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_8.sql', 'rb')
sql = gdal.VSIFReadL(1, 10000, f).decode('ascii')
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/ogr_pgdump_8.sql')
if sql.find("""CREATE TABLE "public"."test" ( "myfid" SERIAL, CONSTRAINT "test_pk" PRIMARY KEY ("myfid") )""") < 0 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "myfid" """) >= 0 or \
sql.find("""10\tfirst string\tsecond string""") == -1 or \
sql.find("""INSERT INTO "public"."test" ("str2") VALUES ('second string');""") == -1 or \
sql.find("""12\tfirst string\tsecond string""") == -1:
print(sql)
return 'fail'
return 'success'
def vsizip_2():
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/foo.bar", "wb")
if fmain is None:
gdaltest.post_reason('fail 1')
return 'fail'
gdal.VSIFWriteL("12345", 1, 5, fmain)
gdal.VSIFCloseL(fmain)
content = gdal.ReadDir("/vsizip/vsimem/test2.zip")
if content != ['foo.bar']:
gdaltest.post_reason('bad content 1')
print(content)
return 'fail'
# Now append a second file
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/bar.baz", "wb")
if fmain is None:
gdaltest.post_reason('fail 2')
return 'fail'
gdal.VSIFWriteL("67890", 1, 5, fmain)
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
content = gdal.ReadDir("/vsizip/vsimem/test2.zip")
gdal.PopErrorHandler()
if gdal.GetLastErrorMsg() != 'Cannot read a zip file being written':
gdaltest.post_reason('expected error')
print(gdal.GetLastErrorMsg())
return 'fail'
if content != None:
gdaltest.post_reason('bad content 2')
print(content)
return 'fail'
gdal.VSIFCloseL(fmain)
content = gdal.ReadDir("/vsizip/vsimem/test2.zip")
if content != ['foo.bar', 'bar.baz']:
gdaltest.post_reason('bad content 3')
print(content)
return 'fail'
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/foo.bar", "rb")
if fmain is None:
gdaltest.post_reason('fail 3')
return 'fail'
data = gdal.VSIFReadL(1, 5, fmain)
gdal.VSIFCloseL(fmain)
if data.decode('ASCII') != '12345':
print(data)
return 'fail'
fmain = gdal.VSIFOpenL("/vsizip/vsimem/test2.zip/bar.baz", "rb")
if fmain is None:
gdaltest.post_reason('fail 4')
return 'fail'
data = gdal.VSIFReadL(1, 5, fmain)
gdal.VSIFCloseL(fmain)
if data.decode('ASCII') != '67890':
print(data)
return 'fail'
gdal.Unlink("/vsimem/test2.zip")
return 'success'
def validate(ds, check_tiled=True, full_check=False):
"""Check if a file is a (Geo)TIFF with cloud optimized compatible structure.
Args:
ds: GDAL Dataset for the file to inspect.
check_tiled: Set to False to ignore missing tiling.
full_check: Set to TRUe to check tile/strip leader/trailer bytes. Might be slow on remote files
Returns:
A tuple, whose first element is an array of error messages
(empty if there is no error), and the second element, a dictionary
with the structure of the GeoTIFF file.
Raises:
ValidateCloudOptimizedGeoTIFFException: Unable to open the file or the
file is not a Tiff.
"""
if int(gdal.VersionInfo('VERSION_NUM')) < 2020000:
raise ValidateCloudOptimizedGeoTIFFException(
'GDAL 2.2 or above required')
unicode_type = type(''.encode('utf-8').decode('utf-8'))
if isinstance(ds, (str, unicode_type)):
gdal.PushErrorHandler()
ds = gdal.Open(ds)
gdal.PopErrorHandler()
if ds is None:
raise ValidateCloudOptimizedGeoTIFFException(
'Invalid file : %s' % gdal.GetLastErrorMsg())
if ds.GetDriver().ShortName != 'GTiff':
raise ValidateCloudOptimizedGeoTIFFException(
'The file is not a GeoTIFF')
details = {}
errors = []
warnings = []
filename = ds.GetDescription()
main_band = ds.GetRasterBand(1)
ovr_count = main_band.GetOverviewCount()
filelist = ds.GetFileList()
if filelist is not None and filename + '.ovr' in filelist:
errors += [
'Overviews found in external .ovr file. They should be internal'
]
if main_band.XSize > 512 or main_band.YSize > 512:
if check_tiled:
block_size = main_band.GetBlockSize()
if block_size[0] == main_band.XSize and block_size[0] > 1024:
errors += [
'The file is greater than 512xH or Wx512, but is not tiled'
]
if ovr_count == 0:
warnings += [
'The file is greater than 512xH or Wx512, it is recommended '
'to include internal overviews'
]
ifd_offset = int(main_band.GetMetadataItem('IFD_OFFSET', 'TIFF'))
ifd_offsets = [ifd_offset]
block_order_row_major = False
block_leader_size_as_uint4 = False
block_trailer_last_4_bytes_repeated = False
mask_interleaved_with_imagery = False
if ifd_offset not in (8, 16):
# Check if there is GDAL hidden structural metadata
f = gdal.VSIFOpenL(filename, 'rb')
if not f:
raise ValidateCloudOptimizedGeoTIFFException("Cannot open file")
signature = struct.unpack('B' * 4, gdal.VSIFReadL(4, 1, f))
bigtiff = signature in ((0x49, 0x49, 0x2B, 0x00), (0x4D, 0x4D, 0x00,
0x2B))
if bigtiff:
expected_ifd_pos = 16
else:
expected_ifd_pos = 8
gdal.VSIFSeekL(f, expected_ifd_pos, 0)
pattern = "GDAL_STRUCTURAL_METADATA_SIZE=%06d bytes\n" % 0
got = gdal.VSIFReadL(len(pattern), 1, f).decode('LATIN1')
if len(got) == len(pattern) and got.startswith(
'GDAL_STRUCTURAL_METADATA_SIZE='):
size = int(got[len('GDAL_STRUCTURAL_METADATA_SIZE='):][0:6])
extra_md = gdal.VSIFReadL(size, 1, f).decode('LATIN1')
block_order_row_major = 'BLOCK_ORDER=ROW_MAJOR' in extra_md
block_leader_size_as_uint4 = 'BLOCK_LEADER=SIZE_AS_UINT4' in extra_md
block_trailer_last_4_bytes_repeated = 'BLOCK_TRAILER=LAST_4_BYTES_REPEATED' in extra_md
mask_interleaved_with_imagery = 'MASK_INTERLEAVED_WITH_IMAGERY=YES' in extra_md
if 'KNOWN_INCOMPATIBLE_EDITION=YES' in extra_md:
errors += [
"KNOWN_INCOMPATIBLE_EDITION=YES is declared in the file"
]
expected_ifd_pos += len(pattern) + size
expected_ifd_pos += expected_ifd_pos % 2 # IFD offset starts on a 2-byte boundary
gdal.VSIFCloseL(f)
if expected_ifd_pos != ifd_offsets[0]:
errors += [
'The offset of the main IFD should be %d. It is %d instead' %
(expected_ifd_pos, ifd_offsets[0])
]
details['ifd_offsets'] = {}
details['ifd_offsets']['main'] = ifd_offset
for i in range(ovr_count):
# Check that overviews are by descending sizes
ovr_band = ds.GetRasterBand(1).GetOverview(i)
if i == 0:
if (ovr_band.XSize > main_band.XSize
or ovr_band.YSize > main_band.YSize):
errors += [
'First overview has larger dimension than main band'
]
else:
prev_ovr_band = ds.GetRasterBand(1).GetOverview(i - 1)
if (ovr_band.XSize > prev_ovr_band.XSize
or ovr_band.YSize > prev_ovr_band.YSize):
errors += [
'Overview of index %d has larger dimension than '
'overview of index %d' % (i, i - 1)
]
if check_tiled:
block_size = ovr_band.GetBlockSize()
if block_size[0] == ovr_band.XSize and block_size[0] > 1024:
errors += ['Overview of index %d is not tiled' % i]
# Check that the IFD of descending overviews are sorted by increasing
# offsets
ifd_offset = int(ovr_band.GetMetadataItem('IFD_OFFSET', 'TIFF'))
ifd_offsets.append(ifd_offset)
details['ifd_offsets']['overview_%d' % i] = ifd_offset
if ifd_offsets[-1] < ifd_offsets[-2]:
if i == 0:
errors += [
'The offset of the IFD for overview of index %d is %d, '
'whereas it should be greater than the one of the main '
'image, which is at byte %d' %
(i, ifd_offsets[-1], ifd_offsets[-2])
]
else:
errors += [
'The offset of the IFD for overview of index %d is %d, '
'whereas it should be greater than the one of index %d, '
'which is at byte %d' %
(i, ifd_offsets[-1], i - 1, ifd_offsets[-2])
]
# Check that the imagery starts by the smallest overview and ends with
# the main resolution dataset
def get_block_offset(band):
blockxsize, blockysize = band.GetBlockSize()
for y in range(int((band.YSize + blockysize - 1) / blockysize)):
for x in range(int((band.XSize + blockxsize - 1) / blockxsize)):
block_offset = band.GetMetadataItem(
'BLOCK_OFFSET_%d_%d' % (x, y), 'TIFF')
if block_offset:
return int(block_offset)
return 0
block_offset = get_block_offset(main_band)
data_offsets = [block_offset]
details['data_offsets'] = {}
details['data_offsets']['main'] = block_offset
for i in range(ovr_count):
ovr_band = ds.GetRasterBand(1).GetOverview(i)
block_offset = get_block_offset(ovr_band)
data_offsets.append(block_offset)
details['data_offsets']['overview_%d' % i] = block_offset
if data_offsets[-1] != 0 and data_offsets[-1] < ifd_offsets[-1]:
if ovr_count > 0:
errors += [
'The offset of the first block of the smallest overview '
'should be after its IFD'
]
else:
errors += [
'The offset of the first block of the image should '
'be after its IFD'
]
for i in range(len(data_offsets) - 2, 0, -1):
if data_offsets[i] != 0 and data_offsets[i] < data_offsets[i + 1]:
errors += [
'The offset of the first block of overview of index %d should '
'be after the one of the overview of index %d' % (i - 1, i)
]
if len(data_offsets) >= 2 and data_offsets[0] != 0 and data_offsets[
0] < data_offsets[1]:
errors += [
'The offset of the first block of the main resolution image '
'should be after the one of the overview of index %d' %
(ovr_count - 1)
]
if full_check and (block_order_row_major or block_leader_size_as_uint4
or block_trailer_last_4_bytes_repeated
or mask_interleaved_with_imagery):
f = gdal.VSIFOpenL(filename, 'rb')
if not f:
raise ValidateCloudOptimizedGeoTIFFException("Cannot open file")
full_check_band(f, 'Main resolution image', main_band, errors,
block_order_row_major, block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery)
if main_band.GetMaskFlags() == gdal.GMF_PER_DATASET and \
(filename + '.msk') not in ds.GetFileList():
full_check_band(f, 'Mask band of main resolution image',
main_band.GetMaskBand(), errors,
block_order_row_major, block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated, False)
for i in range(ovr_count):
ovr_band = ds.GetRasterBand(1).GetOverview(i)
full_check_band(f, 'Overview %d' % i, ovr_band, errors,
block_order_row_major, block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery)
if ovr_band.GetMaskFlags() == gdal.GMF_PER_DATASET and \
(filename + '.msk') not in ds.GetFileList():
full_check_band(f, 'Mask band of overview %d' % i,
ovr_band.GetMaskBand(), errors,
block_order_row_major,
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated, False)
gdal.VSIFCloseL(f)
return warnings, errors, details
def test_vsigs_read_credentials_oauth2_service_account_json_file():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.FileFromMemBuffer(
'/vsimem/service_account.json', """{
"private_key": "-----BEGIN PRIVATE KEY-----\nMIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk\nVNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm\ni2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ\nDVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy\nTabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I\n2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd\nDrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+\nbpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg\nWz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3\n4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5\nKvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj\nj+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI\nyzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq\ngwE6fxOLyJDxuWRf\n-----END PRIVATE KEY-----\n",
"client_email": "CLIENT_EMAIL"
}""")
gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS',
'/vsimem/service_account.json')
gdal.SetConfigOption(
'GO2A_AUD',
'http://localhost:%d/oauth2/v4/token' % gdaltest.webserver_port)
gdal.SetConfigOption('GOA2_NOW', '123456')
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(
request.headers['Content-Length'])).decode('ascii')
content_8080 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODAvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.DAhqWtBgKpObxZ%2BGiXqwF%2Fa4SS%2FNWQRhLCI7DYZCuOTuf2w7dL8j4CdpiwwzQg1diIus7dyViRfzpsFmuZKAXwL%2B84iBoVVqnJJZ4TgwH49NdfMAnc4Rgm%2Bo2a2nEcMjX%2FbQ3jRY%2B9WNVl96hzULGvLrVeyego2f06wivqmvxHA%3D'
content_8081 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODEvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.0abOEg4%2FRApWTSeAs6YTHaNzdwOgZLm8DTMO2MKlOA%2Fiagyb4cBJxDpkD5gECPvi7qhkg7LsyFuj0a%2BK48Bsuj%2FgLHOU4MpB0dHwYnDO2UXzH%2FUPdgFCVak1P1V%2ByiDA%2B%2Ft4aDI5fD9qefKQiu3wsMDHzP71MNLzayrjqaqKKS4%3D'
if content not in [content_8080, content_8081]:
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST', '/oauth2/v4/token', custom_method=method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" %
str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
try:
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
if f is None:
gdal.Unlink('/vsimem/service_account.json')
pytest.fail()
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
signed_url = gdal.GetSignedURL('/vsigs/gs_fake_bucket/resource',
['START_DATE=20180212T123456Z'])
if signed_url not in (
'http://127.0.0.1:8080/gs_fake_bucket/resource?Expires=1518442496&GoogleAccessId=CLIENT_EMAIL&Signature=b19I62KdqV51DpWGxhxGXLGJIA8MHvSJofwOygoeQuIxkM6PmmQFvJYTNWRt9zUVTUoVC0UHVB7ee5Z35NqDC8K4i0quu1hb8Js2B4h0W6OAupvyF3nSQ5D0OJmiSbomGMq0Ehyro5cqJ%2FU%2Fd8oAaKrGKVQScKfXoFrSJBbWkNs%3D',
'http://127.0.0.1:8081/gs_fake_bucket/resource?Expires=1518442496&GoogleAccessId=CLIENT_EMAIL&Signature=b19I62KdqV51DpWGxhxGXLGJIA8MHvSJofwOygoeQuIxkM6PmmQFvJYTNWRt9zUVTUoVC0UHVB7ee5Z35NqDC8K4i0quu1hb8Js2B4h0W6OAupvyF3nSQ5D0OJmiSbomGMq0Ehyro5cqJ%2FU%2Fd8oAaKrGKVQScKfXoFrSJBbWkNs%3D'
):
gdal.Unlink('/vsimem/service_account.json')
pytest.fail(signed_url)
except:
if gdal.GetLastErrorMsg().find(
'CPLRSASHA256Sign() not implemented') >= 0:
pytest.skip()
finally:
gdal.SetConfigOption('GO2A_AUD', None)
gdal.SetConfigOption('GO2A_NOW', None)
gdal.SetConfigOption('GOOGLE_APPLICATION_CREDENTIALS', '')
gdal.Unlink('/vsimem/service_account.json')
assert data == 'foo'
def eedai_4():
if gdaltest.eedai_drv is None:
return 'skip'
gdal.FileFromMemBuffer(
'/vsimem/ee/assets/image',
json.dumps({
'type':
'IMAGE',
'bands': [{
"id": "B1",
"dataType": {
"precision": "INTEGER",
"range": {
"max": 255
}
},
"grid": {
"crsCode": "EPSG:32610",
"affineTransform": {
"translateX": 499980,
"translateY": 4200000,
"scaleX": 60,
"scaleY": -60
},
"dimensions": {
"width": 1830,
"height": 1831
}
}
}, {
"id": "B2",
"dataType": {
"precision": "INTEGER",
"range": {
"max": 255
}
},
"grid": {
"crsCode": "EPSG:32610",
"affineTransform": {
"translateX": 499980,
"translateY": 4200000,
"scaleX": 60,
"scaleY": -60
},
"dimensions": {
"width": 1830,
"height": 1831
}
}
}, {
"id": "B3",
"dataType": {
"precision": "INTEGER",
"range": {
"max": 255
}
},
"grid": {
"crsCode": "EPSG:32610",
"affineTransform": {
"translateX": 499980,
"translateY": 4200000,
"scaleX": 60,
"scaleY": -60
},
"dimensions": {
"width": 1830,
"height": 1831
}
}
}]
}))
gdal.SetConfigOption('EEDA_BEARER', 'mybearer')
gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/')
ds = gdal.Open('EEDAI:image')
gdal.SetConfigOption('EEDA_URL', None)
mem_ds = gdal.GetDriverByName('MEM').Create('', 256, 256, 3)
mem_ds.GetRasterBand(1).Fill(127)
mem_ds.GetRasterBand(2).Fill(128)
mem_ds.GetRasterBand(3).Fill(129)
gdal.GetDriverByName('PNG').CreateCopy('/vsimem/out.png', mem_ds)
f = gdal.VSIFOpenL('/vsimem/out.png', 'rb')
png_data = gdal.VSIFReadL(1, 1000000, f)
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/out.png')
gdal.FileFromMemBuffer(
'/vsimem/ee/assets:getPixels&CUSTOMREQUEST=POST&POSTFIELDS={ "path": "image", "encoding": "PNG", "bandIds": [ "B1", "B2", "B3" ], "grid": { "affineTransform": { "translateX": 499980.0, "translateY": 4200000.0, "scaleX": 60.0, "scaleY": -60.0, "shearX": 0.0, "shearY": 0.0 }, "dimensions": { "width": 256, "height": 256 } } }',
png_data)
got_data = ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1)
got_data = struct.unpack('B', got_data)[0]
if got_data != 127:
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
# Same with dataset RasterIO
got_data = ds.ReadRaster(0, 0, 1, 1)
got_data = struct.unpack('B' * 3, got_data)
if got_data != (127, 128, 129):
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
# Same after flushing cache
ds.FlushCache()
got_data = ds.ReadRaster(0, 0, 1, 1)
got_data = struct.unpack('B' * 3, got_data)
if got_data != (127, 128, 129):
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
# Sub-sampled query
gdal.FileFromMemBuffer(
'/vsimem/ee/assets:getPixels&CUSTOMREQUEST=POST&POSTFIELDS={ "path": "image", "encoding": "PNG", "bandIds": [ "B1", "B2", "B3" ], "grid": { "affineTransform": { "translateX": 499980.0, "translateY": 4200000.0, "scaleX": 120.0, "scaleY": -120.06557377049181, "shearX": 0.0, "shearY": 0.0 }, "dimensions": { "width": 256, "height": 256 } } }',
png_data)
got_data = ds.GetRasterBand(1).ReadRaster(0,
0,
2,
2,
buf_xsize=1,
buf_ysize=1)
got_data = struct.unpack('B', got_data)[0]
if got_data != 127:
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
# Same after flushing cache with dataset RasterIO
ds.FlushCache()
got_data = ds.ReadRaster(0, 0, 2, 2, buf_xsize=1, buf_ysize=1)
got_data = struct.unpack('B' * 3, got_data)
if got_data != (127, 128, 129):
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
ds = None
gdal.SetConfigOption('EEDA_BEARER', None)
return 'success'
def test_vsigs_write():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = ''
while True:
numchars = int(request.rfile.readline().strip(), 16)
content += request.rfile.read(numchars).decode('ascii')
request.rfile.read(2)
if numchars == 0:
break
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(200)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Simulate failure while transmitting
f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if ret != 0:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Simulate failure at end of transfer
f = gdal.VSIFOpenL('/vsigs/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = ''
while True:
numchars = int(request.rfile.readline().strip(), 16)
content += request.rfile.read(numchars).decode('ascii')
request.rfile.read(2)
if numchars == 0:
break
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT', '/test_copy/file.bin', custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if ret != 35000:
gdal.VSIFCloseL(f)
pytest.fail(ret)
with gdaltest.error_handler():
ret = gdal.VSIFCloseL(f)
assert ret != 0
def test_vsigs_read_credentials_oauth2_service_account():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
# Generated with 'openssl genrsa -out rsa-openssl.pem 1024' and
# 'openssl pkcs8 -nocrypt -in rsa-openssl.pem -inform PEM -topk8 -outform PEM -out rsa-openssl.pkcs8.pem'
# DO NOT USE in production !!!!
key = """-----BEGIN PRIVATE KEY-----
MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAOlwJQLLDG1HeLrk
VNcFR5Qptto/rJE5emRuy0YmkVINT4uHb1be7OOo44C2Ev8QPVtNHHS2XwCY5gTm
i2RfIBLv+VDMoVQPqqE0LHb0WeqGmM5V1tHbmVnIkCcKMn3HpK30grccuBc472LQ
DVkkGqIiGu0qLAQ89JP/r0LWWySRAgMBAAECgYAWjsS00WRBByAOh1P/dz4kfidy
TabiXbiLDf3MqJtwX2Lpa8wBjAc+NKrPXEjXpv0W3ou6Z4kkqKHJpXGg4GRb4N5I
2FA+7T1lA0FCXa7dT2jvgJLgpBepJu5b//tqFqORb4A4gMZw0CiPN3sUsWsSw5Hd
DrRXwp6sarzG77kvZQJBAPgysAmmXIIp9j1hrFSkctk4GPkOzZ3bxKt2Nl4GFrb+
bpKSon6OIhP1edrxTz1SMD1k5FiAAVUrMDKSarbh5osCQQDwxq4Tvf/HiYz79JBg
Wz5D51ySkbg01dOVgFW3eaYAdB6ta/o4vpHhnbrfl6VO9oUb3QR4hcrruwnDHsw3
4mDTAkEA9FPZjbZSTOSH/cbgAXbdhE4/7zWOXj7Q7UVyob52r+/p46osAk9i5qj5
Kvnv2lrFGDrwutpP9YqNaMtP9/aLnwJBALLWf9n+GAv3qRZD0zEe1KLPKD1dqvrj
j+LNjd1Xp+tSVK7vMs4PDoAMDg+hrZF3HetSQM3cYpqxNFEPgRRJOy0CQQDQlZHI
yzpSgEiyx8O3EK1iTidvnLXbtWabvjZFfIE/0OhfBmN225MtKG3YLV2HoUvpajLq
gwE6fxOLyJDxuWRf
-----END PRIVATE KEY-----
"""
for i in range(2):
gdal.SetConfigOption(
'GO2A_AUD',
'http://localhost:%d/oauth2/v4/token' % gdaltest.webserver_port)
gdal.SetConfigOption('GOA2_NOW', '123456')
if i == 0:
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY', key)
else:
gdal.FileFromMemBuffer('/vsimem/pkey', key)
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY_FILE', '/vsimem/pkey')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_EMAIL', 'CLIENT_EMAIL')
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(
request.headers['Content-Length'])).decode('ascii')
content_8080 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODAvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.DAhqWtBgKpObxZ%2BGiXqwF%2Fa4SS%2FNWQRhLCI7DYZCuOTuf2w7dL8j4CdpiwwzQg1diIus7dyViRfzpsFmuZKAXwL%2B84iBoVVqnJJZ4TgwH49NdfMAnc4Rgm%2Bo2a2nEcMjX%2FbQ3jRY%2B9WNVl96hzULGvLrVeyego2f06wivqmvxHA%3D'
content_8081 = 'grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiAiQ0xJRU5UX0VNQUlMIiwgInNjb3BlIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL2F1dGgvZGV2c3RvcmFnZS5yZWFkX3dyaXRlIiwgImF1ZCI6ICJodHRwOi8vbG9jYWxob3N0OjgwODEvb2F1dGgyL3Y0L3Rva2VuIiwgImlhdCI6IDEyMzQ1NiwgImV4cCI6IDEyNzA1Nn0%3D.0abOEg4%2FRApWTSeAs6YTHaNzdwOgZLm8DTMO2MKlOA%2Fiagyb4cBJxDpkD5gECPvi7qhkg7LsyFuj0a%2BK48Bsuj%2FgLHOU4MpB0dHwYnDO2UXzH%2FUPdgFCVak1P1V%2ByiDA%2B%2Ft4aDI5fD9qefKQiu3wsMDHzP71MNLzayrjqaqKKS4%3D'
if content not in [content_8080, content_8081]:
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST', '/oauth2/v4/token', custom_method=method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" %
str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
try:
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
except:
if gdal.GetLastErrorMsg().find(
'CPLRSASHA256Sign() not implemented') >= 0:
pytest.skip()
finally:
gdal.SetConfigOption('GO2A_AUD', None)
gdal.SetConfigOption('GO2A_NOW', None)
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY', '')
gdal.SetConfigOption('GS_OAUTH2_PRIVATE_KEY_FILE', '')
gdal.SetConfigOption('GS_OAUTH2_CLIENT_EMAIL', '')
assert data == 'foo'
gdal.Unlink('/vsimem/pkey')
def parse_jp2_box(xml_tree, out_f, src_jp2file):
if not(xml_tree[XML_TYPE_IDX] == gdal.CXT_Element and xml_tree[XML_VALUE_IDX] == 'JP2Box'):
print('Not a JP2Box element')
return False
jp2box_name = get_attribute_val(xml_tree, 'name')
if jp2box_name is None:
print('Cannot find JP2Box.name attribute')
return False
if len(jp2box_name) != 4:
print('Invalid JP2Box.name : %s' % jp2box_name)
return False
hex_binary_content = get_node_content(find_xml_node(xml_tree, 'BinaryContent', immediate_child = True))
decoded_content = find_xml_node(xml_tree, 'DecodedContent', immediate_child = True)
decoded_geotiff = find_xml_node(xml_tree, 'DecodedGeoTIFF', immediate_child = True)
text_content = get_node_content(find_xml_node(xml_tree, 'TextContent', immediate_child = True))
xml_content = find_xml_node(xml_tree, 'XMLContent', immediate_child = True)
jp2box = find_xml_node(xml_tree, 'JP2Box', immediate_child = True)
jp2codestream = find_xml_node(xml_tree, 'JP2KCodeStream', immediate_child = True)
if hex_binary_content:
if decoded_content or decoded_geotiff or text_content or xml_content or jp2box:
print('BinaryContent found, and one of DecodedContent/DecodedGeoTIFF/TextContent/XMLContent/JP2Box. The latter will be ignored')
if jp2box_name == 'uuid':
uuid = get_node_content(find_xml_node(xml_tree, 'UUID', immediate_child = True))
if uuid is None:
print('Cannot find JP2Box.UUID element')
return False
else:
uuid = ''
out_f.write(struct.pack('>I' * 1, 8 + int(len(hex_binary_content)/2) + int(len(uuid)/2)))
out_f.write(jp2box_name.encode('ascii'))
write_hexstring_as_binary(uuid, out_f)
write_hexstring_as_binary(hex_binary_content, out_f)
elif decoded_content:
if decoded_geotiff or text_content or xml_content or jp2box:
print('DecodedContent found, and one of DecodedGeoTIFF/TextContent/XMLContent/JP2Box. The latter will be ignored')
pos = out_f.tell()
out_f.write(struct.pack('>I' * 1, 0))
out_f.write(jp2box_name.encode('ascii'))
for child_idx in range(XML_FIRST_CHILD_IDX, len(decoded_content)):
child = decoded_content[child_idx]
if child[XML_TYPE_IDX] == gdal.CXT_Element and child[XML_VALUE_IDX] == 'Field':
if not parse_field(child, out_f, src_jp2file):
return False
new_pos = out_f.tell()
out_f.seek(pos, 0)
out_f.write(struct.pack('>I' * 1, new_pos - pos))
out_f.seek(new_pos, 0)
elif text_content:
if decoded_geotiff or xml_content or jp2box:
print('TextContent found, and one of DecodedGeoTIFF/XMLContent/JP2Box. The latter will be ignored')
out_f.write(struct.pack('>I' * 1, 8 + len(text_content)))
out_f.write(jp2box_name.encode('ascii'))
out_f.write(text_content.encode('latin1'))
elif xml_content:
if decoded_geotiff or jp2box:
print('XMLContent found, and one of DecodedGeoTIFF/JP2Box. The latter will be ignored')
serialized_xml_content = gdal.SerializeXMLTree(xml_content[XML_FIRST_CHILD_IDX])
out_f.write(struct.pack('>I' * 1, 8 + len(serialized_xml_content)))
out_f.write(jp2box_name.encode('ascii'))
out_f.write(serialized_xml_content.encode('latin1'))
elif jp2box:
if decoded_geotiff:
print('JP2Box found, and one of DecodedGeoTIFF. The latter will be ignored')
pos = out_f.tell()
out_f.write(struct.pack('>I' * 1, 0))
out_f.write(jp2box_name.encode('ascii'))
for child_idx in range(XML_FIRST_CHILD_IDX, len(xml_tree)):
child = xml_tree[child_idx]
if child[XML_TYPE_IDX] == gdal.CXT_Element and child[XML_VALUE_IDX] == 'JP2Box':
if not parse_jp2_box(child, out_f, src_jp2file):
return False
new_pos = out_f.tell()
out_f.seek(pos, 0)
out_f.write(struct.pack('>I' * 1, new_pos - pos))
out_f.seek(new_pos, 0)
elif decoded_geotiff:
serialized_xml_content = gdal.SerializeXMLTree(decoded_geotiff[XML_FIRST_CHILD_IDX])
vrt_ds = gdal.Open(serialized_xml_content)
if vrt_ds is None:
print('Cannot decode VRTDataset. Outputting empty content')
binary_content = ''
else:
tmpfilename = '/vsimem/build_jp2_from_xml_tmp.tif'
gdal.GetDriverByName('GTiff').CreateCopy(tmpfilename, vrt_ds)
tif_f = gdal.VSIFOpenL(tmpfilename, 'rb')
binary_content = gdal.VSIFReadL(1, 10000, tif_f)
gdal.VSIFCloseL(tif_f)
gdal.Unlink(tmpfilename)
uuid = get_node_content(find_xml_node(xml_tree, 'UUID', immediate_child = True))
if uuid is None:
uuid = 'B14BF8BD083D4B43A5AE8CD7D5A6CE03'
out_f.write(struct.pack('>I' * 1, 8 + len(binary_content) + int(len(uuid)/2)))
out_f.write(jp2box_name.encode('ascii'))
write_hexstring_as_binary(uuid, out_f)
out_f.write(binary_content)
elif jp2codestream:
pos = out_f.tell()
out_f.write(struct.pack('>I' * 1, 0))
out_f.write(jp2box_name.encode('ascii'))
if not parse_jp2codestream(None, jp2codestream, out_f, src_jp2file):
return False
new_pos = out_f.tell()
out_f.seek(pos, 0)
out_f.write(struct.pack('>I' * 1, new_pos - pos))
out_f.seek(new_pos, 0)
else:
data_offset = get_attribute_val(xml_tree, 'data_offset')
if data_offset is None:
print('Cannot find JP2Box.data_offset attribute')
return False
data_offset = int(data_offset)
data_length = get_attribute_val(xml_tree, 'data_length')
if data_length is None:
print('Cannot find JP2Box.data_length attribute')
return False
data_length = int(data_length)
src_jp2file.seek(data_offset, 0)
data = src_jp2file.read(data_length)
out_f.write(struct.pack('>I' * 1, 8 + data_length))
out_f.write(jp2box_name.encode('ascii'))
out_f.write(data)
return True
def close(self):
gdal.VSIFCloseL(self.f)
self.f = None
def test_vsiaz_fake_readdir():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=a_dir%2F&restype=container',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Blobs>
<Blob>
<Name>a_dir/resource3.bin</Name>
<Properties>
<Last-Modified>01 Jan 1970 00:00:01</Last-Modified>
<Content-Length>123456</Content-Length>
</Properties>
</Blob>
</Blobs>
</EnumerationResults>
""")
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&marker=bla&prefix=a_dir%2F&restype=container',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Prefix>a_dir/</Prefix>
<Blobs>
<Blob>
<Name>a_dir/resource4.bin</Name>
<Properties>
<Last-Modified>16 Oct 2016 12:34:56</Last-Modified>
<Content-Length>456789</Content-Length>
</Properties>
</Blob>
<BlobPrefix>
<Name>a_dir/subdir/</Name>
</BlobPrefix>
</Blobs>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
pytest.fail()
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir')
assert dir_contents == ['resource3.bin', 'resource4.bin', 'subdir']
assert gdal.VSIStatL(
'/vsiaz/az_fake_bucket2/a_dir/resource3.bin').size == 123456
assert gdal.VSIStatL(
'/vsiaz/az_fake_bucket2/a_dir/resource3.bin').mtime == 1
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/a_dir/resource3.bin')
assert dir_contents is None
# Test error on ReadDir()
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/azure/blob/myaccount/az_fake_bucket2?comp=list&delimiter=%2F&prefix=error_test%2F&restype=container',
500)
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/az_fake_bucket2/error_test/')
assert dir_contents is None
# List containers (empty result)
handler = webserver.SequentialHandler()
handler.add(
'GET', '/azure/blob/myaccount/?comp=list', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults ServiceEndpoint="https://myaccount.blob.core.windows.net">
<Containers/>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
assert dir_contents == ['.']
gdal.VSICurlClearCache()
# List containers
handler = webserver.SequentialHandler()
handler.add(
'GET', '/azure/blob/myaccount/?comp=list', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer1</Name>
</Container>
</Containers>
<NextMarker>bla</NextMarker>
</EnumerationResults>
""")
handler.add(
'GET', '/azure/blob/myaccount/?comp=list&marker=bla', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<EnumerationResults>
<Containers>
<Container>
<Name>mycontainer2</Name>
</Container>
</Containers>
</EnumerationResults>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsiaz/')
assert dir_contents == ['mycontainer1', 'mycontainer2']
def test_vsiaz_fake_write():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.VSICurlClearCache()
# Test creation of BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:AigkrY7q66WCrx3JRKBte56k7kxV2cxB/ZyGNubxk5I=' or \
'Expect' not in h or h['Expect'] != '100-continue' or \
'Content-Length' not in h or h['Content-Length'] != '40000' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'BlockBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.wfile.write('HTTP/1.1 100 Continue\r\n\r\n'.encode('ascii'))
content = request.rfile.read(40000).decode('ascii')
if len(content) != 40000:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
ret += gdal.VSIFWriteL('x' * 5000, 1, 5000, f)
if ret != 40000:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Simulate illegal read
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFReadL(1, 1, f)
assert not ret
gdal.VSIFCloseL(f)
# Simulate illegal seek
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
with gdaltest.error_handler():
ret = gdal.VSIFSeekL(f, 1, 0)
assert ret != 0
gdal.VSIFCloseL(f)
# Simulate failure when putting BlockBob
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
if gdal.VSIFSeekL(f, 0, 0) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
gdal.VSIFWriteL('x' * 35000, 1, 35000, f)
if gdal.VSIFTellL(f) != 35000:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFSeekL(f, 35000, 0) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFSeekL(f, 0, 1) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFSeekL(f, 0, 2) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
if gdal.VSIFEofL(f) != 0:
gdal.VSIFCloseL(f)
pytest.fail()
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFCloseL(f)
if ret == 0:
gdal.VSIFCloseL(f)
pytest.fail(ret)
# Simulate creation of BlockBob over an existing blob of incompatible type
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
assert f is not None
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 409)
handler.add('DELETE', '/azure/blob/myaccount/test_copy/file.bin', 202)
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
with webserver.install_http_handler(handler):
gdal.VSIFCloseL(f)
# Test creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:KimVui3ptY9D5ftLlsI7CNOgK36CNAEzsXqcuHskdEY=' or \
'Content-Length' not in h or h['Content-Length'] != '0' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '10' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(10).decode('ascii')
if content != '0123456789':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
def method(request):
h = request.headers
if 'Content-Length' not in h or h['Content-Length'] != '6' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'x-ms-blob-type' not in h or h['x-ms-blob-type'] != 'AppendBlob':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.protocol_version = 'HTTP/1.1'
content = request.rfile.read(6).decode('ascii')
if content != 'abcdef':
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
return
request.send_response(201)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
custom_method=method)
with webserver.install_http_handler(handler):
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 16:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Test failed creation of AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
assert f is not None
handler = webserver.SequentialHandler()
def method(request):
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin',
custom_method=method)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
# Test failed writing of a block of an AppendBlob
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', '10')
f = gdal.VSIFOpenL('/vsiaz/test_copy/file.bin', 'wb')
gdal.SetConfigOption('VSIAZ_CHUNK_SIZE_BYTES', None)
assert f is not None
handler = webserver.SequentialHandler()
handler.add('PUT', '/azure/blob/myaccount/test_copy/file.bin', 201)
handler.add('PUT',
'/azure/blob/myaccount/test_copy/file.bin?comp=appendblock',
403)
with webserver.install_http_handler(handler):
with gdaltest.error_handler():
ret = gdal.VSIFWriteL('0123456789abcdef', 1, 16, f)
if ret != 0:
gdal.VSIFCloseL(f)
pytest.fail(ret)
gdal.VSIFCloseL(f)
def test_vsiaz_fake_basic():
if gdaltest.webserver_port == 0:
pytest.skip()
signed_url = gdal.GetSignedURL('/vsiaz/az_fake_bucket/resource',
['START_DATE=20180213T123456'])
assert (signed_url in (
'http://127.0.0.1:8080/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12',
'http://127.0.0.1:8081/azure/blob/myaccount/az_fake_bucket/resource?se=2018-02-13T13%3A34%3A56Z&sig=9Jc4yBFlSRZSSxf059OohN6pYRrjuHWJWSEuryczN%2FM%3D&sp=r&sr=c&st=2018-02-13T12%3A34%3A56Z&sv=2012-02-12'
))
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:zKb0EXnM/RinBjcUE9EU+qfRIGaIItoUElSWc+FE24E=' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET',
'/azure/blob/myaccount/az_fake_bucket/resource',
custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz/az_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
def method(request):
request.protocol_version = 'HTTP/1.1'
h = request.headers
if 'Authorization' not in h or \
h['Authorization'] != 'SharedKey myaccount:8d6IEeOsl7qGpKAxaTTxx2xMNpvqWq8DGlFE67lsmQ4=' or \
'x-ms-date' not in h or h['x-ms-date'] != 'my_timestamp' or \
'Accept-Encoding' not in h or h['Accept-Encoding'] != 'gzip':
sys.stderr.write('Bad headers: %s\n' % str(h))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
handler.add('GET',
'/azure/blob/myaccount/az_fake_bucket/resource',
custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsiaz_streaming/az_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin',
200, {'Content-Length': '1000000'})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsiaz/az_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
pytest.fail()
handler = webserver.SequentialHandler()
handler.add('HEAD', '/azure/blob/myaccount/az_fake_bucket/resource2.bin',
200, {'Content-Length': 1000000})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL(
'/vsiaz_streaming/az_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
pytest.fail()
def ogr_pgdump_13():
tests = [
[
ogr.wkbUnknown, [], 'POINT ZM (1 2 3 4)',
["'GEOMETRY',2)", "0101000000000000000000F03F0000000000000040"]
],
[
ogr.wkbUnknown, ['GEOM_TYPE=geography'], 'POINT ZM (1 2 3 4)',
[
"geography(GEOMETRY",
"0101000000000000000000F03F0000000000000040"
]
],
[
ogr.wkbUnknown, ['DIM=XYZ'], 'POINT ZM (1 2 3 4)',
[
"'GEOMETRY',3)",
"0101000080000000000000F03F00000000000000400000000000000840"
]
],
[
ogr.wkbUnknown, ['DIM=XYZ', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(GEOMETRYZ,",
"0101000080000000000000F03F00000000000000400000000000000840"
]
],
[
ogr.wkbPoint, ['DIM=XYZ'], 'POINT ZM (1 2 3 4)',
[
"'POINT',3)",
"0101000080000000000000F03F00000000000000400000000000000840"
]
],
[
ogr.wkbPoint25D, [], 'POINT ZM (1 2 3 4)',
[
"'POINT',3)",
"0101000080000000000000F03F00000000000000400000000000000840"
]
],
[
ogr.wkbPoint, ['DIM=XYZ', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(POINTZ,",
"0101000080000000000000F03F00000000000000400000000000000840"
]
],
[
ogr.wkbUnknown, ['DIM=XYM'], 'POINT ZM (1 2 3 4)',
[
"'GEOMETRY',3)",
"0101000040000000000000F03F00000000000000400000000000001040"
]
],
[
ogr.wkbUnknown, ['DIM=XYM', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(GEOMETRYM,",
"0101000040000000000000F03F00000000000000400000000000001040"
]
],
[
ogr.wkbPoint, ['DIM=XYM'], 'POINT ZM (1 2 3 4)',
[
"'POINTM',3)",
"0101000040000000000000F03F00000000000000400000000000001040"
]
],
[
ogr.wkbPointM, [], 'POINT ZM (1 2 3 4)',
[
"'POINTM',3)",
"0101000040000000000000F03F00000000000000400000000000001040"
]
],
[
ogr.wkbPoint, ['DIM=XYM', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(POINTM,",
"0101000040000000000000F03F00000000000000400000000000001040"
]
],
[
ogr.wkbUnknown, ['DIM=XYZM'], 'POINT ZM (1 2 3 4)',
[
"'GEOMETRY',4)",
"01010000C0000000000000F03F000000000000004000000000000008400000000000001040"
]
],
[
ogr.wkbUnknown, ['DIM=XYZM', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(GEOMETRYZM,",
"01010000C0000000000000F03F000000000000004000000000000008400000000000001040"
]
],
[
ogr.wkbPoint, ['DIM=XYZM'], 'POINT ZM (1 2 3 4)',
[
"'POINT',4)",
"01010000C0000000000000F03F000000000000004000000000000008400000000000001040"
]
],
[
ogr.wkbPointZM, [], 'POINT ZM (1 2 3 4)',
[
"'POINT',4)",
"01010000C0000000000000F03F000000000000004000000000000008400000000000001040"
]
],
[
ogr.wkbPoint, ['DIM=XYZM', 'GEOM_TYPE=geography'],
'POINT ZM (1 2 3 4)',
[
"geography(POINTZM,",
"01010000C0000000000000F03F000000000000004000000000000008400000000000001040"
]
],
]
for (geom_type, options, wkt, expected_strings) in tests:
ds = ogr.GetDriverByName('PGDump').CreateDataSource(
'/vsimem/ogr_pgdump_13.sql', options=['LINEFORMAT=LF'])
lyr = ds.CreateLayer('test', geom_type=geom_type, options=options)
f = ogr.Feature(lyr.GetLayerDefn())
f.SetGeometryDirectly(ogr.CreateGeometryFromWkt(wkt))
lyr.CreateFeature(f)
f = None
ds = None
f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_13.sql', 'rb')
sql = gdal.VSIFReadL(1, 10000, f).decode('utf8')
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/ogr_pgdump_13.sql')
for expected_string in expected_strings:
if sql.find(expected_string) < 0:
gdaltest.post_reason('fail')
print(geom_type, options, wkt, expected_string)
print(sql)
return 'fail'
if 'GEOM_TYPE=geography' in options:
continue
ds = ogr.GetDriverByName('PGDump').CreateDataSource(
'/vsimem/ogr_pgdump_13.sql', options=['LINEFORMAT=LF'])
lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone, options=options)
lyr.CreateGeomField(ogr.GeomFieldDefn("my_geom", geom_type))
f = ogr.Feature(lyr.GetLayerDefn())
f.SetGeometryDirectly(ogr.CreateGeometryFromWkt(wkt))
lyr.CreateFeature(f)
f = None
ds = None
f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_13.sql', 'rb')
sql = gdal.VSIFReadL(1, 10000, f).decode('utf8')
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/ogr_pgdump_13.sql')
for expected_string in expected_strings:
if sql.find(expected_string) < 0:
gdaltest.post_reason('fail')
print(geom_type, options, wkt, expected_string)
print(sql)
return 'fail'
return 'success'
def ogr_pgdump_9(pg_use_copy='YES'):
gdal.SetConfigOption('PG_USE_COPY', pg_use_copy)
ds = ogr.GetDriverByName('PGDump').CreateDataSource(
'/vsimem/ogr_pgdump_9.sql', options=['LINEFORMAT=LF'])
lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone)
fld = ogr.FieldDefn('str', ogr.OFTString)
fld.SetWidth(5)
lyr.CreateField(fld)
fld = ogr.FieldDefn('str2', ogr.OFTString)
lyr.CreateField(fld)
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', '01234')
lyr.CreateFeature(feat)
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', 'ABCDEF')
lyr.CreateFeature(feat)
if sys.version_info >= (3, 0, 0):
val4 = '\u00e9\u00e9\u00e9\u00e9'
val5 = val4 + '\u00e9'
val6 = val5 + '\u00e9'
else:
exec("val4 = u'\\u00e9\\u00e9\\u00e9\\u00e9'")
exec("val5 = val4 + u'\\u00e9'")
exec("val6 = val5 + u'\\u00e9'")
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', val6)
lyr.CreateFeature(feat)
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField('str', 'a' + val5)
lyr.CreateFeature(feat)
gdal.SetConfigOption('PG_USE_COPY', None)
ds = None
f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_9.sql', 'rb')
sql = gdal.VSIFReadL(1, 10000, f).decode('utf8')
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/ogr_pgdump_9.sql')
if pg_use_copy == 'YES':
eofield = '\t'
else:
eofield = "'"
if sql.find("""01234%s""" % eofield) < 0 or \
sql.find("""ABCDE%s""" % eofield) < 0 or \
sql.find("""%s%s""" % (val5, eofield)) < 0 or \
sql.find("""%s%s""" % ('a'+val4, eofield)) < 0:
print(sql)
return 'fail'
return 'success'
def test_vsigs_2():
if gdaltest.webserver_port == 0:
pytest.skip()
# header file
gdal.FileFromMemBuffer('/vsimem/my_headers.txt', 'foo: bar')
handler = webserver.SequentialHandler()
handler.add('GET',
'/gs_fake_bucket_http_header_file/resource',
200, {'Content-type': 'text/plain'},
'Y',
expected_headers={'foo': 'bar'})
with webserver.install_http_handler(handler):
with gdaltest.config_option('GDAL_HTTP_HEADER_FILE',
'/vsimem/my_headers.txt'):
f = open_for_read(
'/vsigs/gs_fake_bucket_http_header_file/resource')
assert f is not None
data = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(data) == 1
gdal.Unlink('/vsimem/my_headers.txt')
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID')
gdal.SetConfigOption('CPL_GS_TIMESTAMP', 'my_timestamp')
signed_url = gdal.GetSignedURL('/vsigs/gs_fake_bucket/resource',
['START_DATE=20180212T123456Z'])
assert (signed_url in (
'http://127.0.0.1:8080/gs_fake_bucket/resource?Expires=1518442496&GoogleAccessId=GS_ACCESS_KEY_ID&Signature=xTphUyMqtKA6UmAX3PEr5VL3EOg%3D',
'http://127.0.0.1:8081/gs_fake_bucket/resource?Expires=1518442496&GoogleAccessId=GS_ACCESS_KEY_ID&Signature=xTphUyMqtKA6UmAX3PEr5VL3EOg%3D'
))
handler = webserver.SequentialHandler()
handler.add('GET',
'/gs_fake_bucket/resource',
200, {'Content-type': 'text/plain'},
'foo',
expected_headers={
'Authorization':
'GOOG1 GS_ACCESS_KEY_ID:8tndu9//BfmN+Kg4AFLdUMZMBDQ='
})
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
handler.add('GET',
'/gs_fake_bucket/resource',
200, {'Content-type': 'text/plain'},
'foo',
expected_headers={
'Authorization':
'GOOG1 GS_ACCESS_KEY_ID:8tndu9//BfmN+Kg4AFLdUMZMBDQ='
})
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs_streaming/gs_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
handler = webserver.SequentialHandler()
handler.add('GET', '/gs_fake_bucket/resource2.bin', 206,
{'Content-Range': 'bytes 0-0/1000000'}, 'x')
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL('/vsigs/gs_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
pytest.fail()
handler = webserver.SequentialHandler()
handler.add('HEAD', '/gs_fake_bucket/resource2.bin', 200,
{'Content-Length': 1000000})
with webserver.install_http_handler(handler):
stat_res = gdal.VSIStatL(
'/vsigs_streaming/gs_fake_bucket/resource2.bin')
if stat_res is None or stat_res.size != 1000000:
if stat_res is not None:
print(stat_res.size)
else:
print(stat_res)
pytest.fail()
def vrt_read_10():
src_ds = gdal.Open('data/byte.tif')
mem_ds = gdal.GetDriverByName('GTiff').CreateCopy(
'/vsimem/vrt_read_10.tif', src_ds)
vrt_ds = gdal.GetDriverByName('VRT').CreateCopy('/vsimem/vrt_read_10.vrt',
mem_ds)
vrt_hist = vrt_ds.GetRasterBand(1).GetHistogram()
mem_hist = mem_ds.GetRasterBand(1).GetHistogram()
mem_ds = None
vrt_ds = None
f = gdal.VSIFOpenL('/vsimem/vrt_read_10.vrt', 'rb')
content = gdal.VSIFReadL(1, 10000, f).decode('ascii')
gdal.VSIFCloseL(f)
if vrt_hist != mem_hist:
gdaltest.post_reason('fail')
print(vrt_hist)
print(mem_hist)
return 'fail'
if content.find('<Histograms>') < 0:
gdaltest.post_reason('fail')
print(content)
return 'fail'
# Single source optimization
for i in range(2):
gdal.FileFromMemBuffer(
'/vsimem/vrt_read_10.vrt',
"""<VRTDataset rasterXSize="20" rasterYSize="20">
<VRTRasterBand dataType="Byte" band="1">
<SimpleSource>
<SourceFilename relativeToVRT="1">vrt_read_10.tif</SourceFilename>
</SimpleSource>
</VRTRasterBand>
</VRTDataset>""")
ds = gdal.Open('/vsimem/vrt_read_10.vrt')
if i == 0:
ds.GetRasterBand(1).GetDefaultHistogram()
else:
ds.GetRasterBand(1).GetHistogram()
ds = None
f = gdal.VSIFOpenL('/vsimem/vrt_read_10.vrt', 'rb')
content = gdal.VSIFReadL(1, 10000, f).decode('ascii')
gdal.VSIFCloseL(f)
if content.find('<Histograms>') < 0:
gdaltest.post_reason('fail')
print(content)
return 'fail'
# Two sources general case
for i in range(2):
gdal.FileFromMemBuffer(
'/vsimem/vrt_read_10.vrt',
"""<VRTDataset rasterXSize="20" rasterYSize="20">
<VRTRasterBand dataType="Byte" band="1">
<SimpleSource>
<SourceFilename relativeToVRT="1">vrt_read_10.tif</SourceFilename>
</SimpleSource>
<SimpleSource>
<SourceFilename relativeToVRT="1">vrt_read_10.tif</SourceFilename>
</SimpleSource>
</VRTRasterBand>
</VRTDataset>""")
ds = gdal.Open('/vsimem/vrt_read_10.vrt')
if i == 0:
ds.GetRasterBand(1).GetDefaultHistogram()
else:
ds.GetRasterBand(1).GetHistogram()
ds = None
f = gdal.VSIFOpenL('/vsimem/vrt_read_10.vrt', 'rb')
content = gdal.VSIFReadL(1, 10000, f).decode('ascii')
gdal.VSIFCloseL(f)
if content.find('<Histograms>') < 0:
gdaltest.post_reason('fail')
print(content)
return 'fail'
gdal.GetDriverByName('GTiff').Delete('/vsimem/vrt_read_10.tif')
gdal.GetDriverByName('VRT').Delete('/vsimem/vrt_read_10.vrt')
return 'success'
def test_vsigs_readdir():
if gdaltest.webserver_port == 0:
pytest.skip()
handler = webserver.SequentialHandler()
handler.add(
'GET', '/gs_fake_bucket2/?delimiter=%2F&prefix=a_dir%2F', 200,
{'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<NextMarker>bla</NextMarker>
<Contents>
<Key>a_dir/resource3.bin</Key>
<LastModified>1970-01-01T00:00:01.000Z</LastModified>
<Size>123456</Size>
</Contents>
</ListBucketResult>
""")
handler.add(
'GET', '/gs_fake_bucket2/?delimiter=%2F&marker=bla&prefix=a_dir%2F',
200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix>a_dir/</Prefix>
<Contents>
<Key>a_dir/resource4.bin</Key>
<LastModified>2015-10-16T12:34:56.000Z</LastModified>
<Size>456789</Size>
</Contents>
<CommonPrefixes>
<Prefix>a_dir/subdir/</Prefix>
</CommonPrefixes>
</ListBucketResult>
""")
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket2/a_dir/resource3.bin')
if f is None:
if gdaltest.is_travis_branch('trusty'):
pytest.skip('Skipped on trusty branch, but should be investigated')
pytest.fail()
gdal.VSIFCloseL(f)
dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir')
assert dir_contents == ['resource3.bin', 'resource4.bin', 'subdir']
assert gdal.VSIStatL(
'/vsigs/gs_fake_bucket2/a_dir/resource3.bin').size == 123456
assert gdal.VSIStatL(
'/vsigs/gs_fake_bucket2/a_dir/resource3.bin').mtime == 1
# ReadDir on something known to be a file shouldn't cause network access
dir_contents = gdal.ReadDir('/vsigs/gs_fake_bucket2/a_dir/resource3.bin')
assert dir_contents is None
# List buckets
handler = webserver.SequentialHandler()
handler.add(
'GET', '/', 200, {'Content-type': 'application/xml'},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult>
<Buckets>
<Bucket>
<Name>mybucket</Name>
</Bucket>
</Buckets>
</ListAllMyBucketsResult>
""")
with webserver.install_http_handler(handler):
dir_contents = gdal.ReadDir('/vsigs/')
assert dir_contents == ['mybucket']
def vsifile_5():
fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'wb')
ref_data = ''.join(['%08X' % i for i in range(5 * 32768)])
gdal.VSIFWriteL(ref_data, 1, len(ref_data), fp)
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSI_CACHE', 'YES')
for i in range(3):
if i == 0:
gdal.SetConfigOption('VSI_CACHE_SIZE', '0')
elif i == 1:
gdal.SetConfigOption('VSI_CACHE_SIZE', '65536')
else:
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
fp = gdal.VSIFOpenL('tmp/vsifile_5.bin', 'rb')
gdal.VSIFSeekL(fp, 50000, 0)
if gdal.VSIFTellL(fp) != 50000:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 50000, 1)
if gdal.VSIFTellL(fp) != 100000:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 0, 2)
if gdal.VSIFTellL(fp) != 5 * 32768 * 8:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFReadL(1, 1, fp)
gdal.VSIFSeekL(fp, 0, 0)
data = gdal.VSIFReadL(1, 3 * 32768, fp)
if data.decode('ascii') != ref_data[0:3 * 32768]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFSeekL(fp, 16384, 0)
data = gdal.VSIFReadL(1, 5 * 32768, fp)
if data.decode('ascii') != ref_data[16384:16384 + 5 * 32768]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
data = gdal.VSIFReadL(1, 50 * 32768, fp)
if data[0:1130496].decode('ascii') != ref_data[16384 + 5 * 32768:]:
gdaltest.post_reason('fail')
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
return 'fail'
gdal.VSIFCloseL(fp)
gdal.SetConfigOption('VSI_CACHE_SIZE', None)
gdal.SetConfigOption('VSI_CACHE', None)
gdal.Unlink('tmp/vsifile_5.bin')
return 'success'
def test_vsigs_read_credentials_refresh_token_default_gdal_app():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption(
'GOA2_AUTH_URL_TOKEN',
'http://localhost:%d/accounts.google.com/o/oauth2/token' %
gdaltest.webserver_port)
gdal.SetConfigOption('GS_OAUTH2_REFRESH_TOKEN', 'REFRESH_TOKEN')
with gdaltest.error_handler():
assert gdal.GetSignedURL('/vsigs/foo/bar') is None
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(
request.headers['Content-Length'])).decode('ascii')
if content != 'refresh_token=REFRESH_TOKEN&client_id=265656308688.apps.googleusercontent.com&client_secret=0IbTUDOYzaL6vnIdWTuQnvLz&grant_type=refresh_token':
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST',
'/accounts.google.com/o/oauth2/token',
custom_method=method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" %
str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN', None)
gdal.SetConfigOption('GS_OAUTH2_REFRESH_TOKEN', '')
def vsifile_generic(filename):
start_time = time.time()
fp = gdal.VSIFOpenL(filename, 'wb+')
if fp is None:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFWriteL('0123456789', 1, 10, fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTruncateL(fp, 20) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTruncateL(fp, 5) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 10:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFSeekL(fp, 0, 2) != 0:
gdaltest.post_reason('failure')
return 'fail'
if gdal.VSIFTellL(fp) != 5:
gdaltest.post_reason('failure')
return 'fail'
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 7:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if abs(start_time - statBuf.mtime) > 2:
gdaltest.post_reason('failure')
print(statBuf.mtime)
return 'fail'
fp = gdal.VSIFOpenL(filename, 'rb')
buf = gdal.VSIFReadL(1, 7, fp)
if gdal.VSIFWriteL('a', 1, 1, fp) != 0:
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFTruncateL(fp, 0) == 0:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(fp)
if buf.decode('ascii') != '01234XX':
gdaltest.post_reason('failure')
print(buf.decode('ascii'))
return 'fail'
# Test append mode on existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 9:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if gdal.Unlink(filename) != 0:
gdaltest.post_reason('failure')
return 'fail'
statBuf = gdal.VSIStatL(filename, gdal.VSI_STAT_EXISTS_FLAG)
if statBuf is not None:
gdaltest.post_reason('failure')
return 'fail'
# Test append mode on non existing file
fp = gdal.VSIFOpenL(filename, 'ab')
gdal.VSIFWriteL('XX', 1, 2, fp)
gdal.VSIFCloseL(fp)
statBuf = gdal.VSIStatL(
filename, gdal.VSI_STAT_EXISTS_FLAG | gdal.VSI_STAT_NATURE_FLAG
| gdal.VSI_STAT_SIZE_FLAG)
if statBuf.size != 2:
gdaltest.post_reason('failure')
print(statBuf.size)
return 'fail'
if gdal.Unlink(filename) != 0:
gdaltest.post_reason('failure')
return 'fail'
return 'success'
def test_vsigs_1():
if not gdaltest.built_against_curl():
pytest.skip()
# Invalid header filename
gdal.ErrorReset()
with gdaltest.config_option('GDAL_HTTP_HEADER_FILE', '/i_dont/exist.py'):
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
if f is not None:
gdal.VSIFCloseL(f)
pytest.fail()
last_err = gdal.GetLastErrorMsg()
assert 'Cannot read' in last_err
# Invalid content for header file
with gdaltest.config_option('GDAL_HTTP_HEADER_FILE', 'vsigs.py'):
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
f = open_for_read('/vsigs/foo/bar')
if f is not None:
gdal.VSIFCloseL(f)
pytest.fail()
# Missing GS_SECRET_ACCESS_KEY
gdal.ErrorReset()
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'GS_SECRET_ACCESS_KEY') >= 0
gdal.ErrorReset()
with gdaltest.config_option('CPL_GCE_SKIP', 'YES'):
with gdaltest.error_handler():
f = open_for_read('/vsigs_streaming/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'GS_SECRET_ACCESS_KEY') >= 0
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', 'GS_SECRET_ACCESS_KEY')
# Missing GS_ACCESS_KEY_ID
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar')
assert f is None and gdal.VSIGetLastErrorMsg().find(
'GS_ACCESS_KEY_ID') >= 0
gdal.SetConfigOption('GS_ACCESS_KEY_ID', 'GS_ACCESS_KEY_ID')
# ERROR 1: The User Id you provided does not exist in our records.
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs/foo/bar.baz')
if f is not None or gdal.VSIGetLastErrorMsg() == '':
if f is not None:
gdal.VSIFCloseL(f)
if gdal.GetConfigOption('APPVEYOR') is not None:
return
pytest.fail(gdal.VSIGetLastErrorMsg())
gdal.ErrorReset()
with gdaltest.error_handler():
f = open_for_read('/vsigs_streaming/foo/bar.baz')
assert f is None and gdal.VSIGetLastErrorMsg() != ''
def jpeg_19():
for width, height, iX in [(32, 32, 12), (25, 25, 8), (24, 25, 8)]:
src_ds = gdal.GetDriverByName('GTiff').Create('/vsimem/jpeg_19.tif',
width, height, 1)
src_ds.CreateMaskBand(gdal.GMF_PER_DATASET)
src_ds.GetRasterBand(1).GetMaskBand().WriteRaster(
0, 0, iX, height, struct.pack('B' * 1, 255), 1, 1)
src_ds.GetRasterBand(1).GetMaskBand().WriteRaster(
iX, 0, width - iX, height, struct.pack('B' * 1, 0), 1, 1)
tiff_mask_data = src_ds.GetRasterBand(1).GetMaskBand().ReadRaster(
0, 0, width, height)
# Generate a JPEG file with a (default) LSB bit mask order
out_ds = gdal.GetDriverByName('JPEG').CreateCopy(
'/vsimem/jpeg_19.jpg', src_ds)
out_ds = None
# Generate a JPEG file with a MSB bit mask order
gdal.SetConfigOption('JPEG_WRITE_MASK_BIT_ORDER', 'MSB')
out_ds = gdal.GetDriverByName('JPEG').CreateCopy(
'/vsimem/jpeg_19_msb.jpg', src_ds)
del out_ds
gdal.SetConfigOption('JPEG_WRITE_MASK_BIT_ORDER', None)
src_ds = None
# Check that the file are indeed different
statBuf = gdal.VSIStatL('/vsimem/jpeg_19.jpg')
f = gdal.VSIFOpenL('/vsimem/jpeg_19.jpg', 'rb')
data1 = gdal.VSIFReadL(1, statBuf.size, f)
gdal.VSIFCloseL(f)
statBuf = gdal.VSIStatL('/vsimem/jpeg_19_msb.jpg')
f = gdal.VSIFOpenL('/vsimem/jpeg_19_msb.jpg', 'rb')
data2 = gdal.VSIFReadL(1, statBuf.size, f)
gdal.VSIFCloseL(f)
if (width, height, iX) == (24, 25, 8):
if data1 != data2:
gdaltest.post_reason('fail')
return 'fail'
else:
if data1 == data2:
gdaltest.post_reason('fail')
return 'fail'
# Check the file with the LSB bit mask order
ds = gdal.Open('/vsimem/jpeg_19.jpg')
jpg_mask_data = ds.GetRasterBand(1).GetMaskBand().ReadRaster(
0, 0, width, height)
ds = None
if tiff_mask_data != jpg_mask_data:
gdaltest.post_reason('fail')
return 'fail'
# Check the file with the MSB bit mask order
ds = gdal.Open('/vsimem/jpeg_19_msb.jpg')
jpg_mask_data = ds.GetRasterBand(1).GetMaskBand().ReadRaster(
0, 0, width, height)
ds = None
if tiff_mask_data != jpg_mask_data:
gdaltest.post_reason('fail')
return 'fail'
gdal.GetDriverByName('GTiff').Delete('/vsimem/jpeg_19.tif')
gdal.GetDriverByName('JPEG').Delete('/vsimem/jpeg_19.jpg')
gdal.GetDriverByName('JPEG').Delete('/vsimem/jpeg_19_msb.jpg')
return 'success'
def test_vsigs_read_credentials_file_refresh_token():
if gdaltest.webserver_port == 0:
pytest.skip()
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '/vsimem/.boto')
gdal.SetConfigOption(
'GOA2_AUTH_URL_TOKEN',
'http://localhost:%d/accounts.google.com/o/oauth2/token' %
gdaltest.webserver_port)
gdal.VSICurlClearCache()
gdal.FileFromMemBuffer(
'/vsimem/.boto', """
[Credentials]
gs_oauth2_refresh_token = REFRESH_TOKEN
[OAuth2]
client_id = CLIENT_ID
client_secret = CLIENT_SECRET
""")
handler = webserver.SequentialHandler()
def method(request):
content = request.rfile.read(int(
request.headers['Content-Length'])).decode('ascii')
if content != 'refresh_token=REFRESH_TOKEN&client_id=CLIENT_ID&client_secret=CLIENT_SECRET&grant_type=refresh_token':
sys.stderr.write('Bad POST content: %s\n' % content)
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
content = """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 3600,
}"""
request.send_header('Content-Length', len(content))
request.end_headers()
request.wfile.write(content.encode('ascii'))
handler.add('POST',
'/accounts.google.com/o/oauth2/token',
custom_method=method)
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" %
str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '')
gdal.SetConfigOption('GOA2_AUTH_URL_TOKEN', None)
gdal.Unlink('/vsimem/.boto')
def vsicurl_streaming_1():
try:
drv = gdal.GetDriverByName( 'HTTP' )
except:
drv = None
if drv is None:
return 'skip'
gdal.SetConfigOption('GDAL_HTTP_CONNECTTIMEOUT', '5')
fp = gdal.VSIFOpenL('/vsicurl_streaming/http://download.osgeo.org/gdal/data/usgsdem/cded/114p01_0100_deme.dem', 'rb')
gdal.SetConfigOption('GDAL_HTTP_CONNECTTIMEOUT', None)
if fp is None:
if gdaltest.gdalurlopen('http://download.osgeo.org/gdal/data/usgsdem/cded/114p01_0100_deme.dem', timeout = 4) is None:
print('cannot open URL')
return 'skip'
gdaltest.post_reason('fail')
return 'fail'
if gdal.VSIFTellL(fp) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
data = gdal.VSIFReadL(1, 50, fp)
if data.decode('ascii') != ' 114p01DEMe Base Ma':
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
if gdal.VSIFTellL(fp) != 50:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
gdal.VSIFSeekL(fp, 0, 0)
if gdal.VSIFTellL(fp) != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
data = gdal.VSIFReadL(1, 50, fp)
if data.decode('ascii') != ' 114p01DEMe Base Ma':
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
if gdal.VSIFTellL(fp) != 50:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
time.sleep(0.5)
gdal.VSIFSeekL(fp, 2001, 0)
data_2001 = gdal.VSIFReadL(1, 20, fp)
if data_2001.decode('ascii') != '7-32767-32767-32767-':
print(data_2001)
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
if gdal.VSIFTellL(fp) != 2001+20:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
gdal.VSIFSeekL(fp, 0, 2)
if gdal.VSIFTellL(fp) != 9839616:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
nRet = len(gdal.VSIFReadL(1, 10, fp))
if nRet != 0:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
gdal.VSIFSeekL(fp, 2001, 0)
data_2001_2 = gdal.VSIFReadL(1, 20, fp)
if gdal.VSIFTellL(fp) != 2001+20:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
if data_2001 != data_2001_2:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
gdal.VSIFSeekL(fp, 1024 * 1024 + 100, 0)
data = gdal.VSIFReadL(1, 20, fp)
if data.decode('ascii') != '67-32767-32767-32767':
print(data)
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
if gdal.VSIFTellL(fp) != 1024 * 1024 + 100+ 20:
gdaltest.post_reason('fail')
gdal.VSIFCloseL(fp)
return 'fail'
gdal.VSIFCloseL(fp)
return 'success'
def eedai_geotiff():
if gdaltest.eedai_drv is None:
return 'skip'
gdal.FileFromMemBuffer(
'/vsimem/ee/assets/image',
json.dumps({
'type':
'IMAGE',
'bands': [{
"id": "B1",
"dataType": {
"precision": "INTEGER",
"range": {
"max": 65535
}
},
"grid": {
"crsCode": "EPSG:32610",
"affineTransform": {
"translateX": 499980,
"translateY": 4200000,
"scaleX": 60,
"scaleY": -60
},
"dimensions": {
"width": 1830,
"height": 1831
}
}
}]
}))
gdal.SetConfigOption('EEDA_BEARER', 'mybearer')
gdal.SetConfigOption('EEDA_URL', '/vsimem/ee/')
ds = gdal.Open('EEDAI:image')
gdal.SetConfigOption('EEDA_URL', None)
mem_ds = gdal.GetDriverByName('MEM').Create('', 256, 256, 1,
gdal.GDT_UInt16)
mem_ds.GetRasterBand(1).Fill(12345)
gdal.GetDriverByName('GTiff').CreateCopy('/vsimem/out.tif', mem_ds)
f = gdal.VSIFOpenL('/vsimem/out.tif', 'rb')
data = gdal.VSIFReadL(1, 1000000, f)
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/out.tif')
gdal.FileFromMemBuffer(
'/vsimem/ee/assets:getPixels&CUSTOMREQUEST=POST&POSTFIELDS={ "path": "image", "encoding": "GEO_TIFF", "bandIds": [ "B1" ], "grid": { "affineTransform": { "translateX": 499980.0, "translateY": 4200000.0, "scaleX": 60.0, "scaleY": -60.0, "shearX": 0.0, "shearY": 0.0 }, "dimensions": { "width": 256, "height": 256 } } }',
data)
got_data = ds.GetRasterBand(1).ReadRaster(0, 0, 1, 1)
got_data = struct.unpack('H', got_data)[0]
if got_data != 12345:
gdaltest.post_reason('fail')
print(got_data)
return 'fail'
ds = None
gdal.SetConfigOption('EEDA_BEARER', None)
return 'success'
def test_vsigs_read_credentials_gce_expiration():
if gdaltest.webserver_port == 0:
pytest.skip()
if sys.platform not in ('linux', 'linux2', 'win32'):
pytest.skip()
gdal.SetConfigOption('CPL_GS_CREDENTIALS_FILE', '')
gdal.SetConfigOption('GS_SECRET_ACCESS_KEY', '')
gdal.SetConfigOption('GS_ACCESS_KEY_ID', '')
gdal.SetConfigOption(
'CPL_GCE_CREDENTIALS_URL',
'http://localhost:%d/computeMetadata/v1/instance/service-accounts/default/token'
% gdaltest.webserver_port)
# Disable hypervisor related check to test if we are really on EC2
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', 'NO')
gdal.VSICurlClearCache()
def method(request):
if 'Authorization' not in request.headers:
sys.stderr.write('Bad headers: %s\n' % str(request.headers))
request.send_response(403)
return
expected_authorization = 'Bearer ACCESS_TOKEN'
if request.headers['Authorization'] != expected_authorization:
sys.stderr.write("Bad Authorization: '%s'\n" %
str(request.headers['Authorization']))
request.send_response(403)
return
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Length', 3)
request.end_headers()
request.wfile.write("""foo""".encode('ascii'))
handler = webserver.SequentialHandler()
# First time is used when trying to establish if GCE authentication is available
handler.add(
'GET', '/computeMetadata/v1/instance/service-accounts/default/token',
200, {}, """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 0,
}""")
# Second time is needed because f the access to th file
handler.add(
'GET', '/computeMetadata/v1/instance/service-accounts/default/token',
200, {}, """{
"access_token" : "ACCESS_TOKEN",
"token_type" : "Bearer",
"expires_in" : 0,
}""")
handler.add('GET', '/gs_fake_bucket/resource', custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read('/vsigs/gs_fake_bucket/resource')
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode('ascii')
gdal.VSIFCloseL(f)
assert data == 'foo'
gdal.SetConfigOption('CPL_GCE_CREDENTIALS_URL', '')
gdal.SetConfigOption('CPL_GCE_CHECK_LOCAL_FILES', None)
def vsizip_1():
# We can keep the handle open during all the ZIP writing
hZIP = gdal.VSIFOpenL("/vsizip/vsimem/test.zip", "wb")
if hZIP is None:
gdaltest.post_reason('fail 1')
return 'fail'
# One way to create a directory
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir2/", "wb")
if f is None:
gdaltest.post_reason('fail 2')
return 'fail'
gdal.VSIFCloseL(f)
# A more natural one
gdal.Mkdir("/vsizip/vsimem/test.zip/subdir1", 0)
# Create 1st file
f2 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "wb")
if f2 is None:
gdaltest.post_reason('fail 3')
return 'fail'
gdal.VSIFWriteL("abcd", 1, 4, f2)
gdal.VSIFCloseL(f2)
# Test that we cannot read a zip file being written
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb")
gdal.PopErrorHandler()
if gdal.GetLastErrorMsg() != 'Cannot read a zip file being written':
gdaltest.post_reason('expected error')
print(gdal.GetLastErrorMsg())
return 'fail'
if f is not None:
gdaltest.post_reason('should not have been successful 1')
return 'fail'
# Create 2nd file
f3 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/efghi", "wb")
if f3 is None:
gdaltest.post_reason('fail 4')
return 'fail'
gdal.VSIFWriteL("efghi", 1, 5, f3)
# Try creating a 3d file
gdal.ErrorReset()
gdal.PushErrorHandler('CPLQuietErrorHandler')
f4 = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/that_wont_work", "wb")
gdal.PopErrorHandler()
if gdal.GetLastErrorMsg(
) != 'Cannot create that_wont_work while another file is being written in the .zip':
gdaltest.post_reason('expected error')
print(gdal.GetLastErrorMsg())
return 'fail'
if f4 is not None:
gdaltest.post_reason('should not have been successful 2')
return 'fail'
gdal.VSIFCloseL(f3)
# Now we can close the main handle
gdal.VSIFCloseL(hZIP)
f = gdal.VSIFOpenL("/vsizip/vsimem/test.zip/subdir3/abcd", "rb")
if f is None:
gdaltest.post_reason('fail 5')
return 'fail'
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
if data.decode('ASCII') != 'abcd':
gdaltest.post_reason('fail')
print(data)
return 'fail'
# Test alternate uri syntax
gdal.Rename("/vsimem/test.zip", "/vsimem/test.xxx")
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd", "rb")
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
if data.decode('ASCII') != 'abcd':
gdaltest.post_reason('fail')
print(data)
return 'fail'
# With a trailing slash
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/subdir3/abcd/", "rb")
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFCloseL(f)
# Test ReadDir()
if len(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}")) != 3:
gdaltest.post_reason('fail')
print(gdal.ReadDir("/vsizip/{/vsimem/test.xxx}"))
return 'fail'
# Unbalanced curls
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Non existing mainfile
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}/bla", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Non existing subfile
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.zzz}/bla", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Wrong syntax
f = gdal.VSIFOpenL("/vsizip/{/vsimem/test.xxx}.aux.xml", "rb")
if f is not None:
gdaltest.post_reason('fail')
return 'fail'
# Test nested { { } }
hZIP = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}", "wb")
if hZIP is None:
gdaltest.post_reason('fail 1')
return 'fail'
f = gdal.VSIFOpenL("/vsizip/{/vsimem/zipinzip.yyy}/test.xxx", "wb")
f_src = gdal.VSIFOpenL("/vsimem/test.xxx", "rb")
data = gdal.VSIFReadL(1, 10000, f_src)
gdal.VSIFCloseL(f_src)
gdal.VSIFWriteL(data, 1, len(data), f)
gdal.VSIFCloseL(f)
gdal.VSIFCloseL(hZIP)
f = gdal.VSIFOpenL(
"/vsizip/{/vsizip/{/vsimem/zipinzip.yyy}/test.xxx}/subdir3/abcd/",
"rb")
if f is None:
gdaltest.post_reason('fail')
return 'fail'
data = gdal.VSIFReadL(1, 4, f)
gdal.VSIFCloseL(f)
if data.decode('ASCII') != 'abcd':
gdaltest.post_reason('fail')
print(data)
return 'fail'
gdal.Unlink("/vsimem/test.xxx")
gdal.Unlink("/vsimem/zipinzip.yyy")
return 'success'
def test_vsigs_extra_1():
if not gdaltest.built_against_curl():
pytest.skip()
# if gdal.GetConfigOption('GS_SECRET_ACCESS_KEY') is None:
# print('Missing GS_SECRET_ACCESS_KEY')
# pytest.skip()
# elif gdal.GetConfigOption('GS_ACCESS_KEY_ID') is None:
# print('Missing GS_ACCESS_KEY_ID')
# pytest.skip()
gs_resource = gdal.GetConfigOption('GS_RESOURCE')
if gs_resource is None:
pytest.skip('Missing GS_RESOURCE')
if '/' not in gs_resource:
path = '/vsigs/' + gs_resource
statres = gdal.VSIStatL(path)
assert statres is not None and stat.S_ISDIR(statres.mode), \
('%s is not a valid bucket' % path)
readdir = gdal.ReadDir(path)
assert readdir is not None, 'ReadDir() should not return empty list'
for filename in readdir:
if filename != '.':
subpath = path + '/' + filename
assert gdal.VSIStatL(subpath) is not None, \
('Stat(%s) should not return an error' % subpath)
unique_id = 'vsigs_test'
subpath = path + '/' + unique_id
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id in readdir, \
('ReadDir(%s) should contain %s' % (path, unique_id))
ret = gdal.Mkdir(subpath, 0)
assert ret != 0, ('Mkdir(%s) repeated should return an error' %
subpath)
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
readdir = gdal.ReadDir(path)
assert unique_id not in readdir, \
('ReadDir(%s) should not contain %s' % (path, unique_id))
ret = gdal.Rmdir(subpath)
assert ret != 0, ('Rmdir(%s) repeated should return an error' %
subpath)
ret = gdal.Mkdir(subpath, 0)
assert ret >= 0, ('Mkdir(%s) should not return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'wb')
assert f is not None
gdal.VSIFWriteL('hello', 1, 5, f)
gdal.VSIFCloseL(f)
ret = gdal.Rmdir(subpath)
assert ret != 0, \
('Rmdir(%s) on non empty directory should return an error' % subpath)
f = gdal.VSIFOpenL(subpath + '/test.txt', 'rb')
assert f is not None
data = gdal.VSIFReadL(1, 5, f).decode('utf-8')
assert data == 'hello'
gdal.VSIFCloseL(f)
ret = gdal.Unlink(subpath + '/test.txt')
assert ret >= 0, \
('Unlink(%s) should not return an error' % (subpath + '/test.txt'))
ret = gdal.Rmdir(subpath)
assert ret >= 0, ('Rmdir(%s) should not return an error' % subpath)
return
f = open_for_read('/vsigs/' + gs_resource)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
# Same with /vsigs_streaming/
f = open_for_read('/vsigs_streaming/' + gs_resource)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
if False: # pylint: disable=using-constant-test
# we actually try to read at read() time and bSetError = false
# Invalid bucket : "The specified bucket does not exist"
gdal.ErrorReset()
f = open_for_read('/vsigs/not_existing_bucket/foo')
with gdaltest.error_handler():
gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert gdal.VSIGetLastErrorMsg() != ''
# Invalid resource
gdal.ErrorReset()
f = open_for_read('/vsigs_streaming/' + gs_resource +
'/invalid_resource.baz')
assert f is None, gdal.VSIGetLastErrorMsg()
# Test GetSignedURL()
signed_url = gdal.GetSignedURL('/vsigs/' + gs_resource)
f = open_for_read('/vsicurl_streaming/' + signed_url)
assert f is not None
ret = gdal.VSIFReadL(1, 1, f)
gdal.VSIFCloseL(f)
assert len(ret) == 1
def vsicurl_test_redirect():
if gdaltest.is_travis_branch('trusty'):
print('Skipped on trusty branch, but should be investigated')
return 'skip'
if gdaltest.webserver_port == 0:
return 'skip'
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add('GET', '/test_redirect/', 404)
# Simulate a big time difference between server and local machine
current_time = 1500
def method(request):
response = 'HTTP/1.1 302\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(current_time)) + '\r\n'
response += 'Location: %s\r\n' % (
'http://localhost:%d/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (gdaltest.webserver_port, current_time + 30))
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add('HEAD', '/test_redirect/test.bin', custom_method=method)
handler.add(
'HEAD',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30), 403, {'Server': 'foo'}, '')
def method(request):
if 'Range' in request.headers:
if request.headers['Range'] == 'bytes=0-16383':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 0-16383/1000000')
request.send_header('Content-Length', 16384)
request.send_header('Connection', 'close')
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
elif request.headers['Range'] == 'bytes=16384-49151':
# Test expiration of the signed URL
request.protocol_version = 'HTTP/1.1'
request.send_response(403)
request.send_header('Content-Length', 0)
request.end_headers()
else:
request.send_response(404)
request.send_header('Content-Length', 0)
request.end_headers()
else:
# After a failed attempt on a HEAD, the client should go there
response = 'HTTP/1.1 200\r\n'
response += 'Server: foo\r\n'
response += 'Date: ' + time.strftime(
"%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(current_time)) + '\r\n'
response += 'Content-type: text/plain\r\n'
response += 'Content-Length: 1000000\r\n'
response += 'Connection: close\r\n'
response += '\r\n'
request.wfile.write(response.encode('ascii'))
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
with webserver.install_http_handler(handler):
f = gdal.VSIFOpenL(
'/vsicurl/http://localhost:%d/test_redirect/test.bin' %
gdaltest.webserver_port, 'rb')
if f is None:
gdaltest.post_reason('fail')
return 'fail'
gdal.VSIFSeekL(f, 0, 2)
if gdal.VSIFTellL(f) != 1000000:
gdaltest.post_reason('fail')
print(gdal.VSIFTellL(f))
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFSeekL(f, 0, 0)
handler = webserver.SequentialHandler()
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
current_time = int(time.time())
def method(request):
# We should go there after expiration of the first signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(302)
# Return a new signed URL
request.send_header(
'Location',
'http://localhost:%d/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d'
% (request.server.port, current_time + 30))
request.send_header('Content-Length', 16384)
request.end_headers()
request.wfile.write(('x' * 16384).encode('ascii'))
handler.add('GET', '/test_redirect/test.bin', custom_method=method)
def method(request):
# Second signed URL
if 'Range' in request.headers and \
request.headers['Range'] == 'bytes=16384-49151':
request.protocol_version = 'HTTP/1.1'
request.send_response(200)
request.send_header('Content-type', 'text/plain')
request.send_header('Content-Range', 'bytes 16384-16384/1000000')
request.send_header('Content-Length', 1)
request.end_headers()
request.wfile.write('y'.encode('ascii'))
handler.add(
'GET',
'/foo.s3.amazonaws.com/test_redirected2/test.bin?Signature=foo&Expires=%d'
% (current_time + 30),
custom_method=method)
with webserver.install_http_handler(handler):
content = gdal.VSIFReadL(1, 16383, f).decode('ascii')
if len(content) != 16383 or content[0] != 'x':
gdaltest.post_reason('fail')
print(content)
gdal.VSIFCloseL(f)
return 'fail'
content = gdal.VSIFReadL(1, 2, f).decode('ascii')
if content != 'xy':
gdaltest.post_reason('fail')
print(content)
gdal.VSIFCloseL(f)
return 'fail'
gdal.VSIFCloseL(f)
return 'success'
def ogr_pgdump_6():
ds = ogr.GetDriverByName('PGDump').CreateDataSource(
'/vsimem/ogr_pgdump_6.sql', options=['LINEFORMAT=LF'])
lyr = ds.CreateLayer('test', geom_type=ogr.wkbNone)
field_defn = ogr.FieldDefn('field_string', ogr.OFTString)
field_defn.SetDefault("'a''b'")
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_int', ogr.OFTInteger)
field_defn.SetDefault('123')
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_real', ogr.OFTReal)
field_defn.SetDefault('1.23')
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_nodefault', ogr.OFTInteger)
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_datetime', ogr.OFTDateTime)
field_defn.SetDefault("CURRENT_TIMESTAMP")
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_datetime2', ogr.OFTDateTime)
field_defn.SetDefault("'2015/06/30 12:34:56'")
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_date', ogr.OFTDate)
field_defn.SetDefault("CURRENT_DATE")
lyr.CreateField(field_defn)
field_defn = ogr.FieldDefn('field_time', ogr.OFTTime)
field_defn.SetDefault("CURRENT_TIME")
lyr.CreateField(field_defn)
gdal.SetConfigOption('PG_USE_COPY', 'YES')
f = ogr.Feature(lyr.GetLayerDefn())
f.SetField('field_string', 'a')
f.SetField('field_int', 456)
f.SetField('field_real', 4.56)
f.SetField('field_datetime', '2015/06/30 12:34:56')
f.SetField('field_datetime2', '2015/06/30 12:34:56')
f.SetField('field_date', '2015/06/30')
f.SetField('field_time', '12:34:56')
lyr.CreateFeature(f)
f = None
# Transition from COPY to INSERT
f = ogr.Feature(lyr.GetLayerDefn())
lyr.CreateFeature(f)
f = None
# Transition from INSERT to COPY
f = ogr.Feature(lyr.GetLayerDefn())
f.SetField('field_string', 'b')
f.SetField('field_int', 456)
f.SetField('field_real', 4.56)
f.SetField('field_datetime', '2015/06/30 12:34:56')
f.SetField('field_datetime2', '2015/06/30 12:34:56')
f.SetField('field_date', '2015/06/30')
f.SetField('field_time', '12:34:56')
lyr.CreateFeature(f)
f = None
gdal.SetConfigOption('PG_USE_COPY', None)
ds = None
f = gdal.VSIFOpenL('/vsimem/ogr_pgdump_6.sql', 'rb')
sql = gdal.VSIFReadL(1, 10000, f).decode('ascii')
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/ogr_pgdump_6.sql')
if sql.find("""a\t456\t4.56\t\\N\t2015/06/30 12:34:56\t2015/06/30 12:34:56\t2015/06/30\t12:34:56""") < 0 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_string" VARCHAR DEFAULT 'a''b';""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_int" INTEGER DEFAULT 123;""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_real" FLOAT8 DEFAULT 1.23;""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_datetime" timestamp with time zone DEFAULT CURRENT_TIMESTAMP;""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_datetime2" timestamp with time zone DEFAULT '2015/06/30 12:34:56+00'::timestamp with time zone;""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_date" date DEFAULT CURRENT_DATE;""") == -1 or \
sql.find("""ALTER TABLE "public"."test" ADD COLUMN "field_time" time DEFAULT CURRENT_TIME;""") == -1 or \
sql.find("""b\t456\t4.56\t\\N\t2015/06/30 12:34:56\t2015/06/30 12:34:56\t2015/06/30\t12:34:56""") < 0:
print(sql)
return 'fail'
return 'success'
