def testMergeEmptyString(self):
"""Test merging the empty or space only string."""
message = protojson.decode_message(test_util.OptionalMessage, '')
self.assertEquals(test_util.OptionalMessage(), message)
message = protojson.decode_message(test_util.OptionalMessage, ' ')
self.assertEquals(test_util.OptionalMessage(), message)
def testProtojsonUnrecognizedNull(self):
"""Test that unrecognized fields that are None are skipped."""
decoded = protojson.decode_message(
MyMessage,
'{"an_integer": 1, "unrecognized_null": null}')
self.assertEquals(decoded.an_integer, 1)
self.assertEquals(decoded.all_unrecognized_fields(), [])
def testUnrecognizedFieldVariants(self):
"""Test that unrecognized fields are mapped to the right variants."""
for encoded, expected_variant in (
('{"an_integer": 1, "unknown_val": 2}',
messages.Variant.INT64),
('{"an_integer": 1, "unknown_val": 2.0}',
messages.Variant.DOUBLE),
('{"an_integer": 1, "unknown_val": "string value"}',
messages.Variant.STRING),
('{"an_integer": 1, "unknown_val": [1, 2, 3]}',
messages.Variant.INT64),
('{"an_integer": 1, "unknown_val": [1, 2.0, 3]}',
messages.Variant.DOUBLE),
('{"an_integer": 1, "unknown_val": [1, "foo", 3]}',
messages.Variant.STRING),
('{"an_integer": 1, "unknown_val": true}',
messages.Variant.BOOL)):
decoded = protojson.decode_message(MyMessage, encoded)
self.assertEquals(decoded.an_integer, 1)
self.assertEquals(1, len(decoded.all_unrecognized_fields()))
self.assertEquals(
'unknown_val', decoded.all_unrecognized_fields()[0])
_, decoded_variant = decoded.get_unrecognized_field_info(
'unknown_val')
self.assertEquals(expected_variant, decoded_variant)
def _SetIamWrapper(cls, iter_result, thread_state):
(serialized_policy, expansion_result) = iter_result
return cls.SetIamHelper(
expansion_result.expanded_storage_url,
# Deserialize the JSON object passed from Command.Apply.
protojson.decode_message(apitools_messages.Policy, serialized_policy),
thread_state=thread_state)
def testAlphaEnumeration(self):
"""Test that alpha enum values work."""
message = protojson.decode_message(MyMessage, '{"an_enum": "RED"}')
expected_message = MyMessage()
expected_message.an_enum = MyMessage.Color.RED
self.assertEquals(expected_message, message)
def testNullValues(self):
"""Test that null values overwrite existing values."""
self.assertEquals(MyMessage(),
protojson.decode_message(MyMessage,
('{"an_integer": null,'
' "a_nested": null,'
' "an_enum": null'
'}')))
def testNumericEnumeration(self):
"""Test that numbers work for enum values."""
message = protojson.decode_message(MyMessage, '{"an_enum": 2}')
expected_message = MyMessage()
expected_message.an_enum = MyMessage.Color.GREEN
self.assertEquals(expected_message, message)
def DeserializeBindingsTuple(serialized_bindings_tuple):
(is_grant, bindings) = serialized_bindings_tuple
return BindingsTuple(
is_grant=is_grant,
bindings=[
protojson.decode_message(
apitools_messages.Policy.BindingsValueListEntry, t)
for t in bindings])
def testEnumerationNegativeTestWithEmptyString(self):
"""The enum value is an empty string."""
# The message should successfully decode.
message = protojson.decode_message(MyMessage, '{"an_enum": ""}')
expected_message = MyMessage()
self.assertEquals(expected_message, message)
# The roundtrip should result in equivalent encoded message.
self.assertEquals('{"an_enum": ""}', protojson.encode_message(message))
def testProtojsonUnrecognizedFieldName(self):
"""Test that unrecognized fields are saved and can be accessed."""
decoded = protojson.decode_message(
MyMessage,
('{"an_integer": 1, "unknown_val": 2}'))
self.assertEquals(decoded.an_integer, 1)
self.assertEquals(1, len(decoded.all_unrecognized_fields()))
self.assertEquals('unknown_val', decoded.all_unrecognized_fields()[0])
self.assertEquals((2, messages.Variant.INT64),
decoded.get_unrecognized_field_info('unknown_val'))
def testDecodeDateTime(self):
for datetime_string, datetime_vals in (
('2012-09-30T15:31:50.262', (2012, 9, 30, 15, 31, 50, 262000)),
('2012-09-30T15:31:50', (2012, 9, 30, 15, 31, 50, 0))):
message = protojson.decode_message(
MyMessage, '{"a_datetime": "%s"}' % datetime_string)
expected_message = MyMessage(
a_datetime=datetime.datetime(*datetime_vals))
self.assertEquals(expected_message, message)
def testConvertIntegerToFloat(self):
"""Test that integers passed in to float fields are converted.
This is necessary because JSON outputs integers for numbers
with 0 decimals.
"""
message = protojson.decode_message(MyMessage, '{"a_float": 10}')
self.assertTrue(isinstance(message.a_float, float))
self.assertEquals(10.0, message.a_float)
def testNumericEnumerationNegativeTest(self):
"""Test with an invalid number for the enum value."""
# The message should successfully decode.
message = protojson.decode_message(MyMessage,
'{"an_enum": 89}')
expected_message = MyMessage()
self.assertEquals(expected_message, message)
# The roundtrip should result in equivalent encoded
# message.
self.assertEquals('{"an_enum": 89}', protojson.encode_message(message))
def testDecodeRepeatedDateTime(self):
message = protojson.decode_message(
MyMessage,
'{"a_repeated_datetime": ["2012-09-30T15:31:50.262", '
'"2010-01-21T09:52:00", "2000-01-01T01:00:59.999999"]}')
expected_message = MyMessage(
a_repeated_datetime=[
datetime.datetime(2012, 9, 30, 15, 31, 50, 262000),
datetime.datetime(2010, 1, 21, 9, 52),
datetime.datetime(2000, 1, 1, 1, 0, 59, 999999)])
self.assertEquals(expected_message, message)
def testAlphaEnumerationNegativeTest(self):
"""The alpha enum value is invalid."""
# The message should successfully decode.
message = protojson.decode_message(MyMessage,
'{"an_enum": "IAMINVALID"}')
expected_message = MyMessage()
self.assertEquals(expected_message, message)
# The roundtrip should result in equivalent encoded message.
self.assertEquals('{"an_enum": "IAMINVALID"}',
protojson.encode_message(message))
def testConvertStringToNumbers(self):
"""Test that strings passed to integer fields are converted."""
message = protojson.decode_message(MyMessage,
"""{"an_integer": "10",
"a_float": "3.5",
"a_repeated": ["1", "2"],
"a_repeated_float": ["1.5", "2", 10]
}""")
self.assertEquals(MyMessage(an_integer=10,
a_float=3.5,
a_repeated=[1, 2],
a_repeated_float=[1.5, 2.0, 10.0]),
message)
def testProtojsonUnrecognizedFieldNumber(self):
"""Test that unrecognized fields are saved and can be accessed."""
decoded = protojson.decode_message(
MyMessage,
'{"an_integer": 1, "1001": "unknown", "-123": "negative", '
'"456_mixed": 2}')
self.assertEquals(decoded.an_integer, 1)
self.assertEquals(3, len(decoded.all_unrecognized_fields()))
self.assertTrue(1001 in decoded.all_unrecognized_fields())
self.assertEquals(('unknown', messages.Variant.STRING),
decoded.get_unrecognized_field_info(1001))
self.assertTrue('-123' in decoded.all_unrecognized_fields())
self.assertEquals(('negative', messages.Variant.STRING),
decoded.get_unrecognized_field_info('-123'))
self.assertTrue('456_mixed' in decoded.all_unrecognized_fields())
self.assertEquals((2, messages.Variant.INT64),
decoded.get_unrecognized_field_info('456_mixed'))
def testDecodeRepeatedCustom(self):
message = protojson.decode_message(MyMessage,
'{"a_repeated_custom": [1, 2, 3]}')
self.assertEquals(MyMessage(a_repeated_custom=[1, 2, 3]), message)
def testDecodeCustom(self):
message = protojson.decode_message(MyMessage, '{"a_custom": 1}')
self.assertEquals(MyMessage(a_custom=1), message)
def testProtojsonUnrecognizedNull(self):
"""Test that unrecognized fields that are None are skipped."""
decoded = protojson.decode_message(
MyMessage, '{"an_integer": 1, "unrecognized_null": null}')
self.assertEquals(decoded.an_integer, 1)
self.assertEquals(decoded.all_unrecognized_fields(), [])
def testDecodeCustom(self):
message = protojson.decode_message(MyMessage, '{"a_custom": 1}')
self.assertEquals(MyMessage(a_custom=1), message)
def testDecodeNone(self):
message = protojson.decode_message(MyMessage, '{"an_integer": []}')
self.assertEquals(MyMessage(an_integer=None), message)
def testEmptyList(self):
"""Test that empty lists are ignored."""
self.assertEquals(
MyMessage(),
protojson.decode_message(MyMessage, '{"a_repeated": []}'))
def _SetIam(self):
"""Set IAM policy for given wildcards on the command line."""
self.continue_on_error = False
self.recursion_requested = False
self.all_versions = False
force_etag = False
etag = ''
if self.sub_opts:
for o, arg in self.sub_opts:
if o in ['-r', '-R']:
self.recursion_requested = True
elif o == '-f':
self.continue_on_error = True
elif o == '-a':
self.all_versions = True
elif o == '-e':
etag = str(arg)
force_etag = True
else:
self.RaiseInvalidArgumentException()
file_url = self.args[0]
patterns = self.args[1:]
# Load the IAM policy file and raise error if the file is invalid JSON or
# does not exist.
try:
with open(file_url, 'r') as fp:
policy = json.loads(fp.read())
except IOError:
raise ArgumentException('Specified IAM policy file "%s" does not exist.' %
file_url)
except ValueError as e:
self.logger.debug('Invalid IAM policy file, ValueError:\n', e)
raise ArgumentException('Invalid IAM policy file "%s".' % file_url)
bindings = policy.get('bindings', [])
if not force_etag:
etag = policy.get('etag', '')
policy_json = json.dumps({
'bindings': bindings,
'etag': etag,
'version': IAM_POLICY_VERSION
})
try:
policy = protojson.decode_message(apitools_messages.Policy, policy_json)
except DecodeError:
raise ArgumentException('Invalid IAM policy file "%s" or etag "%s".' %
(file_url, etag))
self.everything_set_okay = True
# This list of wildcard strings will be handled by NameExpansionIterator.
threaded_wildcards = []
for pattern in patterns:
surl = StorageUrlFromString(pattern)
if surl.IsBucket():
if self.recursion_requested:
surl.object_name = '*'
threaded_wildcards.append(surl.url_string)
else:
self.SetIamHelper(surl, policy)
else:
threaded_wildcards.append(surl.url_string)
# N.B.: If threaded_wildcards contains a non-existent bucket
# (e.g. ["gs://non-existent", "gs://existent"]), NameExpansionIterator
# will raise an exception in iter.next. This halts all iteration, even
# when -f is set. This behavior is also evident in acl set. This behavior
# also appears for any exception that will be raised when iterating over
# wildcard expansions (access denied if bucket cannot be listed, etc.).
if threaded_wildcards:
name_expansion_iterator = NameExpansionIterator(
self.command_name,
self.debug,
self.logger,
self.gsutil_api,
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions,
continue_on_error=self.continue_on_error or self.parallel_operations,
bucket_listing_fields=['name'])
seek_ahead_iterator = SeekAheadNameExpansionIterator(
self.command_name,
self.debug,
self.GetSeekAheadGsutilApi(),
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions)
policy_it = itertools.repeat(protojson.encode_message(policy))
self.Apply(_SetIamWrapper,
zip(policy_it, name_expansion_iterator),
_SetIamExceptionHandler,
fail_on_error=not self.continue_on_error,
seek_ahead_iterator=seek_ahead_iterator)
self.everything_set_okay &= not GetFailureCount() > 0
# TODO: Add an error counter for files and objects.
if not self.everything_set_okay:
raise CommandException('Some IAM policies could not be set.')
def _SetIam(self):
"""Set IAM policy for given wildcards on the command line."""
self.continue_on_error = False
self.recursion_requested = False
self.all_versions = False
force_etag = False
etag = ''
if self.sub_opts:
for o, arg in self.sub_opts:
if o in ['-r', '-R']:
self.recursion_requested = True
elif o == '-f':
self.continue_on_error = True
elif o == '-a':
self.all_versions = True
elif o == '-e':
etag = str(arg)
force_etag = True
else:
self.RaiseInvalidArgumentException()
file_url = self.args[0]
patterns = self.args[1:]
# Load the IAM policy file and raise error if the file is invalid JSON or
# does not exist.
try:
with open(file_url, 'r') as fp:
policy = json.loads(fp.read())
except IOError:
raise ArgumentException('Specified IAM policy file "%s" does not exist.' %
file_url)
except ValueError as e:
self.logger.debug('Invalid IAM policy file, ValueError:\n', e)
raise ArgumentException('Invalid IAM policy file "%s".' % file_url)
bindings = policy.get('bindings', [])
if not force_etag:
etag = policy.get('etag', '')
policy_json = json.dumps({'bindings': bindings, 'etag': etag})
try:
policy = protojson.decode_message(apitools_messages.Policy, policy_json)
except DecodeError:
raise ArgumentException('Invalid IAM policy file "%s" or etag "%s".' %
(file_url, etag))
self.everything_set_okay = True
# This list of wildcard strings will be handled by NameExpansionIterator.
threaded_wildcards = []
for pattern in patterns:
surl = StorageUrlFromString(pattern)
if surl.IsBucket():
if self.recursion_requested:
surl.object_name = '*'
threaded_wildcards.append(surl.url_string)
else:
self.SetIamHelper(surl, policy)
else:
threaded_wildcards.append(surl.url_string)
# N.B.: If threaded_wildcards contains a non-existent bucket
# (e.g. ["gs://non-existent", "gs://existent"]), NameExpansionIterator
# will raise an exception in iter.next. This halts all iteration, even
# when -f is set. This behavior is also evident in acl set. This behavior
# also appears for any exception that will be raised when iterating over
# wildcard expansions (access denied if bucket cannot be listed, etc.).
if threaded_wildcards:
name_expansion_iterator = NameExpansionIterator(
self.command_name,
self.debug,
self.logger,
self.gsutil_api,
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions,
continue_on_error=self.continue_on_error or self.parallel_operations,
bucket_listing_fields=['name'])
seek_ahead_iterator = SeekAheadNameExpansionIterator(
self.command_name,
self.debug,
self.GetSeekAheadGsutilApi(),
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions)
policy_it = itertools.repeat(protojson.encode_message(policy))
self.Apply(_SetIamWrapper,
zip(policy_it, name_expansion_iterator),
_SetIamExceptionHandler,
fail_on_error=not self.continue_on_error,
seek_ahead_iterator=seek_ahead_iterator)
self.everything_set_okay &= not GetFailureCount() > 0
# TODO: Add an error counter for files and objects.
if not self.everything_set_okay:
raise CommandException('Some IAM policies could not be set.')
def testDecodeRepeatedCustom(self):
message = protojson.decode_message(
MyMessage, '{"a_repeated_custom": [1, 2, 3]}')
self.assertEquals(MyMessage(a_repeated_custom=[1, 2, 3]), message)
def testDecodeNone(self):
message = protojson.decode_message(
MyMessage, '{"an_integer": []}')
self.assertEquals(MyMessage(an_integer=None), message)
def testEmptyList(self):
"""Test that empty lists are ignored."""
self.assertEquals(MyMessage(),
protojson.decode_message(MyMessage,
'{"a_repeated": []}'))
def testDecodeRepeatedEmpty(self):
message = protojson.decode_message(
MyMessage, '{"a_repeated": []}')
self.assertEquals(MyMessage(a_repeated=[]), message)
def testDecodeRepeatedEmpty(self):
message = protojson.decode_message(MyMessage, '{"a_repeated": []}')
self.assertEquals(MyMessage(a_repeated=[]), message)
