def _postprocess_yaml(data):
'''
This function is a hack to let us use datetimes in JSON-formatted feedback
objects. Otherwise the datetimes will remain strings after loading the YAML.
Modifies the YAML object directly.
It's also used for any other YAML massaging.
'''
TIMESTAMP_SUFFIX = '!!timestamp'
# First just collect the paths to change, so we're not modifying while
# walking the object (which might risk the walk changing...?).
timestamps = [(path, val) for path, val in utils.objwalk(data)
if str(path[-1]).endswith(TIMESTAMP_SUFFIX)]
# Replace the timestamp strings with actual datetimes and change the key name.
for path, val in timestamps:
new_path = list(path[:-1])
new_path.append(path[-1][:path[-1].rindex(TIMESTAMP_SUFFIX)])
new_val = datetime.datetime.strptime(val, '%Y-%m-%dT%H:%M:%S.%fZ')
utils.rename_key_in_obj_at_path(data, path, new_path[-1])
utils.assign_value_to_obj_at_path(data, new_path, new_val)
#
# Fix integer-looking IDs
#
# If a hex ID happens to have all numbers, YAML will decode it as an
# integer rather than a string. This could mess up processing later on.
_ensure_field_is_string(str, data, ('Metadata', 'id'))
# Fix data type of other fields.
# For example, if just a number is entered in the feedback text, it should
# still be interpreted as a string.
_ensure_field_is_string(unicode, data, ('Feedback', 'email'))
_ensure_field_is_string(unicode, data, ('Feedback', 'Message', 'text'))
def _postprocess_yaml(data):
'''
This function is a hack to let us use datetimes in JSON-formatted feedback
objects. Otherwise the datetimes will remain strings after loading the YAML.
Modifies the YAML object directly.
It's also used for any other YAML massaging.
'''
TIMESTAMP_SUFFIX = '!!timestamp'
# First just collect the paths to change, so we're not modifying while
# walking the object (which might risk the walk changing...?).
timestamps = [(path, val) for path, val in utils.objwalk(data)
if str(path[-1]).endswith(TIMESTAMP_SUFFIX)]
# Replace the timestamp strings with actual datetimes and change the key name.
for path, val in timestamps:
new_path = list(path[:-1])
new_path.append(path[-1][:path[-1].rindex(TIMESTAMP_SUFFIX)])
new_val = datetime.datetime.strptime(val, '%Y-%m-%dT%H:%M:%S.%fZ')
utils.rename_key_in_obj_at_path(data, path, new_path[-1])
utils.assign_value_to_obj_at_path(data, new_path, new_val)
#
# Fix integer-looking IDs
#
# If a hex ID happens to have all numbers, YAML will decode it as an
# integer rather than a string. This could mess up processing later on.
data['Metadata']['id'] = str(data['Metadata']['id'])
def _sanitize_keys(data):
"""
MongoDB does not allow dots ('.') in keys, but we do (or may) use dots in
the diagnostic data. So we'll replace dots with an allowable character.
"""
for path, val in utils.objwalk(data):
if isinstance(path[-1],
utils.string_types) and path[-1].find('.') >= 0:
utils.rename_key_in_obj_at_path(data, path,
path[-1].replace('.', '_'))
def _upgrade_old_object(yaml_docs):
'''
The diagnostic info stuff was released for Android before versioning was
added.
Returns the appropriately modified YAML dict.
'''
logger.debug_log('maildecryptor._upgrade_old_object start')
# The non-versioned YAML used multiple docs, so that's the main test
if len(yaml_docs) == 1:
return yaml_docs.pop()
# Our old YAML had '.' in some key names, which is illegal.
for path, val in utils.objwalk(yaml_docs):
if type(path[-1]) == str and path[-1].find('.') >= 0:
utils.rename_key_in_obj_at_path(yaml_docs,
path,
path[-1].replace('.', '__'))
# Our old YAML format was multiple YAML docs in a single string. We'll
# convert that to the new format.
obj = {}
# Old YAML had no Metadata section
metadata = {}
metadata['platform'] = 'android'
metadata['version'] = 1
metadata['id'] = binascii.hexlify(os.urandom(8))
obj['Metadata'] = metadata
idx = 0
obj['SystemInformation'] = yaml_docs[idx]
idx += 1
obj['ServerResponseCheck'] = yaml_docs[idx]
idx += 1
# The presence of DiagnosticHistory was optional once upon a time
if len(yaml_docs) > 3:
obj['DiagnosticHistory'] = yaml_docs[idx]
idx += 1
else:
obj['DiagnosticHistory'] = []
obj['StatusHistory'] = yaml_docs[idx]
idx += 1
logger.debug_log('maildecryptor._upgrade_old_object end')
return obj
def _upgrade_old_object(yaml_docs):
'''
The diagnostic info stuff was released for Android before versioning was
added.
Returns the appropriately modified YAML dict.
'''
logger.debug_log('maildecryptor._upgrade_old_object start')
# The non-versioned YAML used multiple docs, so that's the main test
if len(yaml_docs) == 1:
return yaml_docs.pop()
# Our old YAML had '.' in some key names, which is illegal.
for path, val in utils.objwalk(yaml_docs):
if isinstance(path[-1],
utils.string_types) and path[-1].find('.') >= 0:
utils.rename_key_in_obj_at_path(yaml_docs, path,
path[-1].replace('.', '__'))
# Our old YAML format was multiple YAML docs in a single string. We'll
# convert that to the new format.
obj = {}
# Old YAML had no Metadata section
metadata = {}
metadata['platform'] = 'android'
metadata['version'] = 1
metadata['id'] = binascii.hexlify(os.urandom(8))
obj['Metadata'] = metadata
idx = 0
obj['SystemInformation'] = yaml_docs[idx]
idx += 1
obj['ServerResponseCheck'] = yaml_docs[idx]
idx += 1
# The presence of DiagnosticHistory was optional once upon a time
if len(yaml_docs) > 3:
obj['DiagnosticHistory'] = yaml_docs[idx]
idx += 1
else:
obj['DiagnosticHistory'] = []
obj['StatusHistory'] = yaml_docs[idx]
idx += 1
logger.debug_log('maildecryptor._upgrade_old_object end')
return obj
def _sanitize_keys(data):
"""
MongoDB does not allow dots ('.') in keys, but we do (or may) use dots in
the diagnostic data. So we'll replace dots with an allowable character.
"""
paths_to_sanitize = []
for path, _ in utils.objwalk(data):
for i in xrange(len(path)):
if isinstance(path[i],
utils.string_types) and path[i].find('.') >= 0:
paths_to_sanitize.append(path[:i + 1])
# paths_to_sanitize has the paths that end in keys with dots; e.g.:
# [('a.a',), ('a.a', 'b.b'), ('c', 'd.d')]
# We need to iterate through the list backward, so we don't invalidate the higher-up
# keys before we've used all of them; e.g., if we change ('a.a',) before we change
# ('a.a', 'b.b'), then we won't actually find the latter.
for path in reversed(paths_to_sanitize):
utils.rename_key_in_obj_at_path(data, path, path[-1].replace('.', '_'))
