def _process_annotation_items(items):
""" Process annotation items after the scalyr config prefix has been stripped
"""
result = {}
def sort_annotation(pair):
(key, value) = pair
m = SCALYR_ANNOTATION_ELEMENT_RE.match(key)
if m:
root_key = m.group(1)
if _is_int(root_key):
return int(root_key)
return root_key
return key
def sort_numeric(pair):
(key, value) = pair
if _is_int(key):
return int(key)
return key
# sort dict by the value of the first sub key (up to the first '.')
# this ensures that all items of the same key are processed together
sorted_items = sorted(items.iteritems(), key=sort_annotation)
current_object = None
previous_key = None
is_array = False
is_object = False
for (key, value) in sorted_items:
# split out the sub key from the rest of the key
m = SCALYR_ANNOTATION_ELEMENT_RE.match(key)
if m:
root_key = m.group(1)
child_key = m.group(2)
# check for mixed list and dict keys and raise an error if they exist
if _is_int(root_key):
is_array = True
else:
is_object = True
if is_object == is_array:
raise BadAnnotationConfig(
"Annotation cannot be both a dict and a list for '%s'. Current key: %s, previous key: %s"
% (key, str(root_key), str(previous_key)))
# create an empty object if None exists
if current_object is None:
current_object = {}
# else if the keys are different which means we have a new key,
# so add the current object to the list of results and create a new object
elif previous_key is not None and root_key != previous_key:
result[previous_key] = _process_annotation_items(
current_object)
current_object = {}
current_object[child_key] = value
previous_key = root_key
else: # no more subkeys so just process as the full key
# check for mixed list and dict keys and raise an error if they exist
if _is_int(key):
is_array = True
else:
is_object = True
if is_object == is_array:
raise BadAnnotationConfig(
"Annotation cannot be both a dict and a list. Current key: %s, previous key: %s"
% (key, str(previous_key)))
# if there was a previous key
if previous_key is not None and current_object is not None:
# stick it in the result
result[previous_key] = _process_annotation_items(
current_object)
# add the current value to the result
result[key] = value
current_object = None
previous_key = None
# add the final object if there was one
if previous_key is not None and current_object is not None:
result[previous_key] = _process_annotation_items(current_object)
# if the result should be an array, return values as a JsonArray, sorted by numeric order of keys
if is_array:
result = JsonArray(
*[r[1] for r in sorted(result.iteritems(), key=sort_numeric)])
else:
# return values as a JsonObject
result = JsonObject(content=result)
return result
def _process_annotation_items(items, hyphens_as_underscores):
""" Process annotation items after the scalyr config prefix has been stripped
"""
def sort_annotation(pair):
(key, value) = pair
m = SCALYR_ANNOTATION_ELEMENT_RE.match(key)
if m:
root_key = m.group(1)
if _is_int(root_key):
return int(root_key)
return root_key
return key
def sort_numeric(pair):
(key, value) = pair
if _is_int(key):
return int(key)
return key
def normalize_key_name(key, convert_hyphens):
"""
Normalizes the name of the key by converting any hyphens to underscores (or not)
depending on the `convert_hyphens` parameter.
Typically, `convert_hypens` will be false when converting k8s annotations because
label and annotation keys in k8s can contain underscores.
Keys for docker labels however cannot container underscores, therefore `convert_hyphens`
can be used to convert any label keys to use underscores, which are expected by various
log_config options.
This function means the code that processes the labels/annotations doesn't need to care
if it is running under docker or k8s, because the root caller decides whether or not hyphens
need converting.
@param key: string - a key for an annotation/label
@param convert_hyphens: bool - if True, any hyphens in the `key` parameter will be
converted to underscores
"""
if convert_hyphens:
key = key.replace("-", "_")
return key
result = {}
# sort dict by the value of the first sub key (up to the first '.')
# this ensures that all items of the same key are processed together
sorted_items = sorted(items.iteritems(), key=sort_annotation)
current_object = None
previous_key = None
is_array = False
is_object = False
for (key, value) in sorted_items:
# split out the sub key from the rest of the key
m = SCALYR_ANNOTATION_ELEMENT_RE.match(key)
if m:
root_key = m.group(1)
child_key = m.group(2)
# check for mixed list and dict keys and raise an error if they exist
if _is_int(root_key):
is_array = True
else:
is_object = True
if is_object == is_array:
raise BadAnnotationConfig(
"Annotation cannot be both a dict and a list for '%s'. Current key: %s, previous key: %s"
% (key, str(root_key), str(previous_key)))
# create an empty object if None exists
if current_object is None:
current_object = {}
# else if the keys are different which means we have a new key,
# so add the current object to the list of results and create a new object
elif previous_key is not None and root_key != previous_key:
updated_key = normalize_key_name(previous_key,
hyphens_as_underscores)
result[updated_key] = _process_annotation_items(
current_object, hyphens_as_underscores)
current_object = {}
current_object[child_key] = value
previous_key = root_key
else: # no more subkeys so just process as the full key
# check for mixed list and dict keys and raise an error if they exist
if _is_int(key):
is_array = True
else:
is_object = True
if is_object == is_array:
raise BadAnnotationConfig(
"Annotation cannot be both a dict and a list. Current key: %s, previous key: %s"
% (key, str(previous_key)))
# if there was a previous key
if previous_key is not None and current_object is not None:
# stick it in the result
updated_key = normalize_key_name(previous_key,
hyphens_as_underscores)
result[updated_key] = _process_annotation_items(
current_object, hyphens_as_underscores)
# add the current value to the result
updated_key = normalize_key_name(key, hyphens_as_underscores)
result[updated_key] = value
current_object = None
previous_key = None
# add the final object if there was one
if previous_key is not None and current_object is not None:
updated_key = normalize_key_name(previous_key, hyphens_as_underscores)
result[updated_key] = _process_annotation_items(
current_object, hyphens_as_underscores)
# if the result should be an array, return values as a JsonArray, sorted by numeric order of keys
if is_array:
result = JsonArray(
*[r[1] for r in sorted(result.iteritems(), key=sort_numeric)])
else:
# return values as a JsonObject
result = JsonObject(content=result)
return result
def _process_annotation_items( items, hyphens_as_underscores ):
""" Process annotation items after the scalyr config prefix has been stripped
"""
def sort_annotation( pair ):
(key, value) = pair
m = SCALYR_ANNOTATION_ELEMENT_RE.match( key )
if m:
root_key = m.group(1)
if _is_int( root_key ):
return int( root_key )
return root_key
return key
def sort_numeric( pair ):
(key, value) = pair
if _is_int( key ):
return int( key )
return key
def normalize_key_name( key, convert_hyphens ):
"""
Normalizes the name of the key by converting any hyphens to underscores (or not)
depending on the `convert_hyphens` parameter.
Typically, `convert_hypens` will be false when converting k8s annotations because
label and annotation keys in k8s can contain underscores.
Keys for docker labels however cannot container underscores, therefore `convert_hyphens`
can be used to convert any label keys to use underscores, which are expected by various
log_config options.
This function means the code that processes the labels/annotations doesn't need to care
if it is running under docker or k8s, because the root caller decides whether or not hyphens
need converting.
@param: key - string - a key for an annotation/label
@param: convert_hyphens - bool - if True, any hyphens in the `key` parameter will be
converted to underscores
"""
if convert_hyphens:
key = key.replace( '-', '_' )
return key
result = {}
# sort dict by the value of the first sub key (up to the first '.')
# this ensures that all items of the same key are processed together
sorted_items = sorted( items.iteritems(), key=sort_annotation)
current_object = None
previous_key = None
is_array = False
is_object = False
for (key, value) in sorted_items:
# split out the sub key from the rest of the key
m = SCALYR_ANNOTATION_ELEMENT_RE.match( key )
if m:
root_key = m.group(1)
child_key = m.group(2)
# check for mixed list and dict keys and raise an error if they exist
if _is_int( root_key ):
is_array = True
else:
is_object = True
if is_object == is_array:
raise BadAnnotationConfig( "Annotation cannot be both a dict and a list for '%s'. Current key: %s, previous key: %s" % (key, str(root_key), str(previous_key)) )
# create an empty object if None exists
if current_object is None:
current_object = {}
# else if the keys are different which means we have a new key,
# so add the current object to the list of results and create a new object
elif previous_key is not None and root_key != previous_key:
updated_key = normalize_key_name(previous_key, hyphens_as_underscores)
result[updated_key] = _process_annotation_items( current_object, hyphens_as_underscores )
current_object = {}
current_object[child_key] = value
previous_key = root_key
else: # no more subkeys so just process as the full key
# check for mixed list and dict keys and raise an error if they exist
if _is_int( key ):
is_array = True
else:
is_object = True
if is_object == is_array:
raise BadAnnotationConfig( "Annotation cannot be both a dict and a list. Current key: %s, previous key: %s" % (key, str(previous_key)) )
# if there was a previous key
if previous_key is not None and current_object is not None:
# stick it in the result
updated_key = normalize_key_name(previous_key, hyphens_as_underscores)
result[updated_key] = _process_annotation_items( current_object, hyphens_as_underscores )
# add the current value to the result
updated_key = normalize_key_name(key, hyphens_as_underscores)
result[updated_key] = value
current_object = None
previous_key = None
# add the final object if there was one
if previous_key is not None and current_object is not None:
updated_key = normalize_key_name(previous_key, hyphens_as_underscores)
result[updated_key] = _process_annotation_items( current_object, hyphens_as_underscores )
# if the result should be an array, return values as a JsonArray, sorted by numeric order of keys
if is_array:
result = JsonArray( *[r[1] for r in sorted( result.iteritems(), key=sort_numeric )] )
else:
# return values as a JsonObject
result = JsonObject( content=result )
return result
