def test_list_of_dicts(self):
annotations = {
"log.config.scalyr.com/10.match_expression": "fourth",
"log.config.scalyr.com/10.sampling_rate": 4,
"log.config.scalyr.com/2.match_expression": "third",
"log.config.scalyr.com/2.sampling_rate": 3,
"log.config.scalyr.com/0.match_expression": "first",
"log.config.scalyr.com/0.sampling_rate": 1,
"log.config.scalyr.com/1.match_expression": "second",
"log.config.scalyr.com/1.sampling_rate": 2
}
result = annotation_config.process_annotations(annotations)
self.assertEquals(4, len(result))
self.assertTrue(isinstance(result, JsonArray))
self.assertEquals("first", result[0]['match_expression'])
self.assertEquals(1, result[0]['sampling_rate'])
self.assertEquals("second", result[1]['match_expression'])
self.assertEquals(2, result[1]['sampling_rate'])
self.assertEquals("third", result[2]['match_expression'])
self.assertEquals(3, result[2]['sampling_rate'])
self.assertEquals("fourth", result[3]['match_expression'])
self.assertEquals(4, result[3]['sampling_rate'])
def test_dict_with_list(self):
annotations = {
"log.config.scalyr.com/rules.10.match_expression": "fourth",
"log.config.scalyr.com/rules.10.sampling_rate": 4,
"log.config.scalyr.com/rules.2.match_expression": "third",
"log.config.scalyr.com/rules.2.sampling_rate": 3,
"log.config.scalyr.com/rules.0.match_expression": "first",
"log.config.scalyr.com/rules.0.sampling_rate": 1,
"log.config.scalyr.com/rules.1.match_expression": "second",
"log.config.scalyr.com/rules.1.sampling_rate": 2,
}
result = annotation_config.process_annotations(annotations)
self.assertEquals(1, len(result))
result = result["rules"]
self.assertEquals(4, len(result))
self.assertTrue(isinstance(result, JsonArray))
self.assertEquals("first", result[0]["match_expression"])
self.assertEquals(1, result[0]["sampling_rate"])
self.assertEquals("second", result[1]["match_expression"])
self.assertEquals(2, result[1]["sampling_rate"])
self.assertEquals("third", result[2]["match_expression"])
self.assertEquals(3, result[2]["sampling_rate"])
self.assertEquals("fourth", result[3]["match_expression"])
self.assertEquals(4, result[3]["sampling_rate"])
self.assertEquals(4, result[3]["sampling_rate"])
def test_dict_values(self):
annotations = {
"log.config.scalyr.com/attributes.parser": "accessLog",
"log.config.scalyr.com/attributes.container": "my-container",
"log.config.scalyr.com/attributes.amazing": "yes it is",
"log.config.scalyr.com/rename_logfile.match": "/var/log/(.*).log",
"log.config.scalyr.com/rename_logfile.replacement":
"/scalyr/\\1.log",
}
result = annotation_config.process_annotations(annotations)
self.assertEquals(2, len(result))
self.assertTrue('attributes' in result)
self.assertTrue('rename_logfile' in result)
attrs = result['attributes']
self.assertEquals(3, len(attrs))
self.assertTrue('parser' in attrs)
self.assertTrue('container' in attrs)
self.assertTrue('amazing' in attrs)
self.assertEquals("accessLog", attrs['parser'])
self.assertEquals("my-container", attrs['container'])
self.assertEquals("yes it is", attrs['amazing'])
def test_list_of_dicts(self):
annotations = {
"log.config.scalyr.com/10.match_expression": "fourth",
"log.config.scalyr.com/10.sampling_rate": 4,
"log.config.scalyr.com/2.match_expression": "third",
"log.config.scalyr.com/2.sampling_rate": 3,
"log.config.scalyr.com/0.match_expression": "first",
"log.config.scalyr.com/0.sampling_rate": 1,
"log.config.scalyr.com/1.match_expression": "second",
"log.config.scalyr.com/1.sampling_rate": 2
}
result = annotation_config.process_annotations( annotations )
self.assertEquals( 4, len( result ) )
self.assertTrue( isinstance( result, JsonArray ) )
self.assertEquals( "first", result[0]['match_expression'] )
self.assertEquals( 1, result[0]['sampling_rate'] )
self.assertEquals( "second", result[1]['match_expression'] )
self.assertEquals( 2, result[1]['sampling_rate'] )
self.assertEquals( "third", result[2]['match_expression'] )
self.assertEquals( 3, result[2]['sampling_rate'] )
self.assertEquals( "fourth", result[3]['match_expression'] )
self.assertEquals( 4, result[3]['sampling_rate'] )
def test_dict_values(self):
annotations = {
"log.config.scalyr.com/attributes.parser": "accessLog",
"log.config.scalyr.com/attributes.container": "my-container",
"log.config.scalyr.com/attributes.amazing": "yes it is",
"log.config.scalyr.com/rename_logfile.match": "/var/log/(.*).log",
"log.config.scalyr.com/rename_logfile.replacement": "/scalyr/\\1.log",
}
result = annotation_config.process_annotations( annotations )
self.assertEquals( 2, len( result ) )
self.assertTrue( 'attributes' in result )
self.assertTrue( 'rename_logfile' in result )
attrs = result['attributes']
self.assertEquals( 3, len( attrs ) )
self.assertTrue( 'parser' in attrs )
self.assertTrue( 'container' in attrs )
self.assertTrue( 'amazing' in attrs )
self.assertEquals( "accessLog", attrs['parser'] )
self.assertEquals( "my-container", attrs['container'] )
self.assertEquals( "yes it is", attrs['amazing'] )
def test_invalid_annotations( self ):
annotations = {
"some.other.value": 10,
"not.a.scalyr.annotation": "no it's not",
}
result = process_annotations( annotations )
self.assertEquals( 0, len( result.keys() ) )
def test_mixed_list_and_dict_key(self):
annotations = {
"log.config.scalyr.com/test.2.match_expression": "third",
"log.config.scalyr.com/test.2.sampling_rate": 3,
"log.config.scalyr.com/test.bad.match_expression": "first",
"log.config.scalyr.com/test.bad.sampling_rate": 1,
"log.config.scalyr.com/test.1.match_expression": "second",
"log.config.scalyr.com/test.1.sampling_rate": 2
}
self.assertRaises(annotation_config.BadAnnotationConfig, lambda: annotation_config.process_annotations( annotations ) )
def test_no_scalyr_annotations(self):
annotations = {
"config.agent.not-scalyr.com/attributes.parser": "accessLog",
"config.agent.not-scalyr.com/log_path": "/var/log/access.log",
"config.agent.not-scalyr.com/sampling_rules.0.match_expression": "INFO",
"config.agent.not-scalyr.com/sampling_rules.0.sampling_rate": 0.1,
}
result = annotation_config.process_annotations( annotations )
self.assertFalse( result )
def test_mixed_list_and_dict_key(self):
annotations = {
"log.config.scalyr.com/test.2.match_expression": "third",
"log.config.scalyr.com/test.2.sampling_rate": 3,
"log.config.scalyr.com/test.bad.match_expression": "first",
"log.config.scalyr.com/test.bad.sampling_rate": 1,
"log.config.scalyr.com/test.1.match_expression": "second",
"log.config.scalyr.com/test.1.sampling_rate": 2
}
self.assertRaises(annotation_config.BadAnnotationConfig, lambda: annotation_config.process_annotations( annotations ) )
def test_no_scalyr_annotations(self):
annotations = {
"config.agent.not-scalyr.com/attributes.parser": "accessLog",
"config.agent.not-scalyr.com/log_path": "/var/log/access.log",
"config.agent.not-scalyr.com/sampling_rules.0.match_expression": "INFO",
"config.agent.not-scalyr.com/sampling_rules.0.sampling_rate": 0.1,
}
result = annotation_config.process_annotations( annotations )
self.assertFalse( result )
def test_annotation_object( self ):
annotations = {
"log.config.scalyr.com/item1" : "item1",
"log.config.scalyr.com/item2" : "item2",
"log.config.scalyr.com/item3" : "item3"
}
result = process_annotations( annotations )
self.assertEquals( 3, len( result.keys() ) )
self.assertEquals( 'item1', result['item1'] )
self.assertEquals( 'item2', result['item2'] )
self.assertEquals( 'item3', result['item3'] )
def test_annotation_object(self):
annotations = {
"log.config.scalyr.com/item1": "item1",
"log.config.scalyr.com/item2": "item2",
"log.config.scalyr.com/item3": "item3",
}
result = process_annotations(annotations)
self.assertEquals(3, len(list(result.keys())))
self.assertEquals("item1", result["item1"])
self.assertEquals("item2", result["item2"])
self.assertEquals("item3", result["item3"])
def test_list_value(self):
annotations = {
"log.config.scalyr.com/3": "three",
"log.config.scalyr.com/2": "two",
"log.config.scalyr.com/1": "one",
}
result = annotation_config.process_annotations( annotations )
self.assertTrue( isinstance( result, JsonArray ) )
self.assertEqual( 3, len( result ) )
self.assertEqual( 'one', result[0] )
self.assertEqual( 'two', result[1] )
self.assertEqual( 'three', result[2] )
def test_list_value(self):
annotations = {
"log.config.scalyr.com/3": "three",
"log.config.scalyr.com/2": "two",
"log.config.scalyr.com/1": "one",
}
result = annotation_config.process_annotations(annotations)
self.assertTrue(isinstance(result, JsonArray))
self.assertEqual(3, len(result))
self.assertEqual('one', result[0])
self.assertEqual('two', result[1])
self.assertEqual('three', result[2])
def test_annotation_keys_with_hyphens( self ):
annotations = {
"com.scalyr.config.log.item1-1.element-1" : "item1 element 1",
"com.scalyr.config.log.item1-2.element-2" : "item1 element 2",
"com.scalyr.config.log.item1-3.element-3" : "item1 element 3",
"com.scalyr.config.log.item2-1.element-1" : "item2 element 1",
"com.scalyr.config.log.item2-2.element-2" : "item2 element 2",
"com.scalyr.config.log.item2-3.element-3" : "item2 element 3",
}
result = process_annotations( annotations, annotation_prefix_re=re.compile( '^(com\.scalyr\.config\.log\.)(.+)' ), hyphens_as_underscores=True )
self.assertEquals( 'item1 element 1', result['item1_1']['element_1'] )
self.assertEquals( 'item1 element 2', result['item1_2']['element_2'] )
self.assertEquals( 'item1 element 3', result['item1_3']['element_3'] )
self.assertEquals( 'item2 element 1', result['item2_1']['element_1'] )
self.assertEquals( 'item2 element 2', result['item2_2']['element_2'] )
self.assertEquals( 'item2 element 3', result['item2_3']['element_3'] )
def test_plain_values(self):
annotations = {
"log.config.scalyr.com/someKey": "someValue",
"log.config.scalyr.com/log_path": "/var/log/access.log",
"log.config.scalyr.com/important_field": "important_value",
}
result = annotation_config.process_annotations( annotations )
self.assertEquals( 3, len( result ) )
self.assertTrue( 'someKey' in result )
self.assertTrue( 'log_path' in result )
self.assertTrue( 'important_field' in result )
self.assertEquals( "someValue", result['someKey'] )
self.assertEquals( "/var/log/access.log", result['log_path'] )
self.assertEquals( "important_value", result['important_field'] )
def test_plain_values(self):
annotations = {
"log.config.scalyr.com/someKey": "someValue",
"log.config.scalyr.com/log_path": "/var/log/access.log",
"log.config.scalyr.com/important_field": "important_value",
}
result = annotation_config.process_annotations(annotations)
self.assertEquals(3, len(result))
self.assertTrue('someKey' in result)
self.assertTrue('log_path' in result)
self.assertTrue('important_field' in result)
self.assertEquals("someValue", result['someKey'])
self.assertEquals("/var/log/access.log", result['log_path'])
self.assertEquals("important_value", result['important_field'])
def process_object(self, obj):
""" Generate a PodInfo object from a JSON object
@param pod: The JSON object returned as a response to querying
a specific pod from the k8s API
@return A PodInfo object
"""
result = {}
metadata = obj.get('metadata', {})
spec = obj.get('spec', {})
labels = metadata.get('labels', {})
annotations = metadata.get('annotations', {})
owners = metadata.get('ownerReferences', [])
pod_name = metadata.get("name", '')
namespace = metadata.get("namespace", '')
deployment_name = self._get_deployment_name_from_owners(
owners, namespace)
daemonset_name = self._get_daemonset_name_from_owners(
owners, namespace)
container_names = []
for container in spec.get('containers', []):
container_names.append(
container.get('name', 'invalid-container-name'))
try:
annotations = annotation_config.process_annotations(annotations)
except BadAnnotationConfig, e:
self._logger.warning(
"Bad Annotation config for %s/%s. All annotations ignored. %s"
% (namespace, pod_name, str(e)),
limit_once_per_x_secs=300,
limit_key='bad-annotation-config-%s' % info.uid)
annotations = JsonObject()
def test_list_of_lists(self):
annotations = {
"log.config.scalyr.com/2.10.match_expression": "fourth",
"log.config.scalyr.com/2.10.sampling_rate": 4,
"log.config.scalyr.com/2.2.match_expression": "third",
"log.config.scalyr.com/2.2.sampling_rate": 3,
"log.config.scalyr.com/1.0.match_expression": "first",
"log.config.scalyr.com/1.0.sampling_rate": 1,
"log.config.scalyr.com/1.1.match_expression": "second",
"log.config.scalyr.com/1.1.sampling_rate": 2
}
#import pdb; pdb.set_trace()
result = annotation_config.process_annotations( annotations )
self.assertEquals( 2, len( result ) )
first = result[0]
self.assertEquals( 2, len( first ) )
self.assertTrue( isinstance( first, JsonArray ) )
self.assertEquals( "first", first[0]['match_expression'] )
self.assertEquals( 1, first[0]['sampling_rate'] )
self.assertEquals( "second", first[1]['match_expression'] )
self.assertEquals( 2, first[1]['sampling_rate'] )
second = result[1]
self.assertEquals( 2, len( second ) )
self.assertTrue( isinstance( second, JsonArray ) )
self.assertEquals( "third", second[0]['match_expression'] )
self.assertEquals( 3, second[0]['sampling_rate'] )
self.assertEquals( "fourth", second[1]['match_expression'] )
self.assertEquals( 4, second[1]['sampling_rate'] )
def test_list_of_lists(self):
annotations = {
"log.config.scalyr.com/2.10.match_expression": "fourth",
"log.config.scalyr.com/2.10.sampling_rate": 4,
"log.config.scalyr.com/2.2.match_expression": "third",
"log.config.scalyr.com/2.2.sampling_rate": 3,
"log.config.scalyr.com/1.0.match_expression": "first",
"log.config.scalyr.com/1.0.sampling_rate": 1,
"log.config.scalyr.com/1.1.match_expression": "second",
"log.config.scalyr.com/1.1.sampling_rate": 2
}
#import pdb; pdb.set_trace()
result = annotation_config.process_annotations(annotations)
self.assertEquals(2, len(result))
first = result[0]
self.assertEquals(2, len(first))
self.assertTrue(isinstance(first, JsonArray))
self.assertEquals("first", first[0]['match_expression'])
self.assertEquals(1, first[0]['sampling_rate'])
self.assertEquals("second", first[1]['match_expression'])
self.assertEquals(2, first[1]['sampling_rate'])
second = result[1]
self.assertEquals(2, len(second))
self.assertTrue(isinstance(second, JsonArray))
self.assertEquals("third", second[0]['match_expression'])
self.assertEquals(3, second[0]['sampling_rate'])
self.assertEquals("fourth", second[1]['match_expression'])
self.assertEquals(4, second[1]['sampling_rate'])
def test_annotation_nested_object(self):
annotations = {
"log.config.scalyr.com/item1.nest1": "item1 nest1",
"log.config.scalyr.com/item1.nest2": "item1 nest2",
"log.config.scalyr.com/item1.nest3": "item1 nest3",
"log.config.scalyr.com/item2.nest1": "item2 nest1",
"log.config.scalyr.com/item2.nest2": "item2 nest2",
"log.config.scalyr.com/item2.nest3": "item2 nest3",
"log.config.scalyr.com/item2.nest4": "item2 nest4",
}
result = process_annotations(annotations)
self.assertEquals(2, len(list(result.keys())))
self.assertEquals(3, len(result["item1"]))
self.assertEquals(4, len(result["item2"]))
self.assertEquals("item1 nest1", result["item1"]["nest1"])
self.assertEquals("item1 nest2", result["item1"]["nest2"])
self.assertEquals("item1 nest3", result["item1"]["nest3"])
self.assertEquals("item2 nest1", result["item2"]["nest1"])
self.assertEquals("item2 nest2", result["item2"]["nest2"])
self.assertEquals("item2 nest3", result["item2"]["nest3"])
self.assertEquals("item2 nest4", result["item2"]["nest4"])
def test_annotation_nested_object( self ):
annotations = {
"log.config.scalyr.com/item1.nest1" : "item1 nest1",
"log.config.scalyr.com/item1.nest2" : "item1 nest2",
"log.config.scalyr.com/item1.nest3" : "item1 nest3",
"log.config.scalyr.com/item2.nest1" : "item2 nest1",
"log.config.scalyr.com/item2.nest2" : "item2 nest2",
"log.config.scalyr.com/item2.nest3" : "item2 nest3",
"log.config.scalyr.com/item2.nest4" : "item2 nest4"
}
result = process_annotations( annotations )
self.assertEquals( 2, len( result.keys() ) )
self.assertEquals( 3, len( result['item1'] ) )
self.assertEquals( 4, len( result['item2'] ) )
self.assertEquals( 'item1 nest1', result['item1']['nest1'] )
self.assertEquals( 'item1 nest2', result['item1']['nest2'] )
self.assertEquals( 'item1 nest3', result['item1']['nest3'] )
self.assertEquals( 'item2 nest1', result['item2']['nest1'] )
self.assertEquals( 'item2 nest2', result['item2']['nest2'] )
self.assertEquals( 'item2 nest3', result['item2']['nest3'] )
self.assertEquals( 'item2 nest4', result['item2']['nest4'] )
def test_annotation_array( self ):
annotations = {
"log.config.scalyr.com/item1.20" : "item1 element 2",
"log.config.scalyr.com/item1.0" : "item1 element 0",
"log.config.scalyr.com/item1.10" : "item1 element 1",
"log.config.scalyr.com/item2.0" : "item2 element 0",
"log.config.scalyr.com/item2.1" : "item2 element 1",
"log.config.scalyr.com/item2.2" : "item2 element 2",
"log.config.scalyr.com/item2.3" : "item2 element 3",
}
result = process_annotations( annotations )
self.assertEquals( 2, len( result.keys() ) )
self.assertEquals( 3, len( result['item1'] ) )
self.assertEquals( 4, len( result['item2'] ) )
self.assertEquals( 'item1 element 0', result['item1'][0] )
self.assertEquals( 'item1 element 1', result['item1'][1] )
self.assertEquals( 'item1 element 2', result['item1'][2] )
self.assertEquals( 'item2 element 0', result['item2'][0] )
self.assertEquals( 'item2 element 1', result['item2'][1] )
self.assertEquals( 'item2 element 2', result['item2'][2] )
self.assertEquals( 'item2 element 3', result['item2'][3] )
def test_full_config(self):
annotations = {
"log.config.scalyr.com/path": "/some/path.log",
"log.config.scalyr.com/attributes.parser": "accessLog",
"log.config.scalyr.com/attributes.service": "memcache",
"log.config.scalyr.com/sampling_rules.10.match_expression": "10-INFO",
"log.config.scalyr.com/sampling_rules.10.sampling_rate": 10,
"log.config.scalyr.com/sampling_rules.2.match_expression": "2-INFO",
"log.config.scalyr.com/sampling_rules.2.sampling_rate": 2,
"log.config.scalyr.com/line_groupers.4.start": "start4",
"log.config.scalyr.com/line_groupers.4.continueThrough": "continueThrough",
"log.config.scalyr.com/line_groupers.3.start": "start3",
"log.config.scalyr.com/line_groupers.3.continuePast": "continuePast",
"log.config.scalyr.com/line_groupers.2.start": "start2",
"log.config.scalyr.com/line_groupers.2.haltBefore": "haltBefore",
"log.config.scalyr.com/line_groupers.1.start": "start1",
"log.config.scalyr.com/line_groupers.1.haltWith": "haltWith",
"log.config.scalyr.com/exclude.1": "exclude1",
"log.config.scalyr.com/exclude.2": "exclude2",
"log.config.scalyr.com/exclude.3": "exclude3",
"log.config.scalyr.com/rename_logfile.match": "rename",
"log.config.scalyr.com/rename_logfile.replacement": "renamed",
"log.config.scalyr.com/redaction_rules.1.match_expression": "redacted1",
"log.config.scalyr.com/redaction_rules.2.match_expression": "redacted2",
"log.config.scalyr.com/redaction_rules.2.replacement": "replaced2",
"log.config.scalyr.com/redaction_rules.3.match_expression": "redacted3",
"log.config.scalyr.com/redaction_rules.3.replacement": "replaced3",
"log.config.scalyr.com/redaction_rules.3.hash_salt": "salt3",
}
result = annotation_config.process_annotations( annotations )
self.assertEquals( 7, len( result ) )
self.assertTrue( 'path' in result )
self.assertTrue( 'attributes' in result )
self.assertTrue( 'sampling_rules' in result )
self.assertTrue( 'line_groupers' in result )
self.assertTrue( 'exclude' in result )
self.assertTrue( 'rename_logfile' in result )
self.assertTrue( 'redaction_rules' in result )
attrs = result['attributes']
self.assertTrue( isinstance( attrs, JsonObject ) )
self.assertEquals( 2, len( attrs ) )
self.assertTrue( 'parser' in attrs )
self.assertTrue( 'service' in attrs )
self.assertEquals( 'accessLog', attrs['parser'] )
self.assertEquals( 'memcache', attrs['service'] )
sampling = result['sampling_rules']
self.assertTrue( isinstance( sampling, JsonArray ) )
self.assertEqual( 2, len( sampling ) )
self.assertEqual( '2-INFO', sampling[0]['match_expression'] )
self.assertEqual( 2, sampling[0]['sampling_rate'] )
self.assertEqual( '10-INFO', sampling[1]['match_expression'] )
self.assertEqual( 10, sampling[1]['sampling_rate'] )
groupers = result['line_groupers']
self.assertTrue( isinstance( groupers, JsonArray ) )
self.assertEqual( 4, len( groupers ) )
self.assertEqual( 'start1', groupers[0]['start'] )
self.assertEqual( 'haltWith', groupers[0]['haltWith'] )
self.assertEqual( 'start2', groupers[1]['start'] )
self.assertEqual( 'haltBefore', groupers[1]['haltBefore'] )
self.assertEqual( 'start3', groupers[2]['start'] )
self.assertEqual( 'continuePast', groupers[2]['continuePast'] )
self.assertEqual( 'start4', groupers[3]['start'] )
self.assertEqual( 'continueThrough', groupers[3]['continueThrough'] )
exclude = result['exclude']
self.assertTrue( isinstance( exclude, JsonArray ) )
self.assertEqual( 3, len( exclude ) )
self.assertEqual( 'exclude1', exclude[0] )
self.assertEqual( 'exclude2', exclude[1] )
self.assertEqual( 'exclude3', exclude[2] )
rename = result['rename_logfile']
self.assertTrue( isinstance( rename, JsonObject ) )
self.assertEqual( 'rename', rename['match'] )
self.assertEqual( 'renamed', rename['replacement'] )
redaction = result['redaction_rules']
self.assertTrue( isinstance( exclude, JsonArray ) )
self.assertEqual( 3, len( exclude ) )
self.assertEqual( 'redacted1', redaction[0]['match_expression'] )
self.assertEqual( 'redacted2', redaction[1]['match_expression'] )
self.assertEqual( 'replaced2', redaction[1]['replacement'] )
self.assertEqual( 'redacted3', redaction[2]['match_expression'] )
self.assertEqual( 'replaced3', redaction[2]['replacement'] )
self.assertEqual( 'salt3', redaction[2]['hash_salt'] )
def test_full_config(self):
annotations = {
"log.config.scalyr.com/path": "/some/path.log",
"log.config.scalyr.com/attributes.parser": "accessLog",
"log.config.scalyr.com/attributes.service": "memcache",
"log.config.scalyr.com/sampling_rules.10.match_expression":
"10-INFO",
"log.config.scalyr.com/sampling_rules.10.sampling_rate": 10,
"log.config.scalyr.com/sampling_rules.2.match_expression":
"2-INFO",
"log.config.scalyr.com/sampling_rules.2.sampling_rate": 2,
"log.config.scalyr.com/line_groupers.4.start": "start4",
"log.config.scalyr.com/line_groupers.4.continueThrough":
"continueThrough",
"log.config.scalyr.com/line_groupers.3.start": "start3",
"log.config.scalyr.com/line_groupers.3.continuePast":
"continuePast",
"log.config.scalyr.com/line_groupers.2.start": "start2",
"log.config.scalyr.com/line_groupers.2.haltBefore": "haltBefore",
"log.config.scalyr.com/line_groupers.1.start": "start1",
"log.config.scalyr.com/line_groupers.1.haltWith": "haltWith",
"log.config.scalyr.com/exclude.1": "exclude1",
"log.config.scalyr.com/exclude.2": "exclude2",
"log.config.scalyr.com/exclude.3": "exclude3",
"log.config.scalyr.com/rename_logfile.match": "rename",
"log.config.scalyr.com/rename_logfile.replacement": "renamed",
"log.config.scalyr.com/redaction_rules.1.match_expression":
"redacted1",
"log.config.scalyr.com/redaction_rules.2.match_expression":
"redacted2",
"log.config.scalyr.com/redaction_rules.2.replacement": "replaced2",
"log.config.scalyr.com/redaction_rules.3.match_expression":
"redacted3",
"log.config.scalyr.com/redaction_rules.3.replacement": "replaced3",
"log.config.scalyr.com/redaction_rules.3.hash_salt": "salt3",
}
result = annotation_config.process_annotations(annotations)
self.assertEquals(7, len(result))
self.assertTrue("path" in result)
self.assertTrue("attributes" in result)
self.assertTrue("sampling_rules" in result)
self.assertTrue("line_groupers" in result)
self.assertTrue("exclude" in result)
self.assertTrue("rename_logfile" in result)
self.assertTrue("redaction_rules" in result)
attrs = result["attributes"]
self.assertTrue(isinstance(attrs, JsonObject))
self.assertEquals(2, len(attrs))
self.assertTrue("parser" in attrs)
self.assertTrue("service" in attrs)
self.assertEquals("accessLog", attrs["parser"])
self.assertEquals("memcache", attrs["service"])
sampling = result["sampling_rules"]
self.assertTrue(isinstance(sampling, JsonArray))
self.assertEqual(2, len(sampling))
self.assertEqual("2-INFO", sampling[0]["match_expression"])
self.assertEqual(2, sampling[0]["sampling_rate"])
self.assertEqual("10-INFO", sampling[1]["match_expression"])
self.assertEqual(10, sampling[1]["sampling_rate"])
groupers = result["line_groupers"]
self.assertTrue(isinstance(groupers, JsonArray))
self.assertEqual(4, len(groupers))
self.assertEqual("start1", groupers[0]["start"])
self.assertEqual("haltWith", groupers[0]["haltWith"])
self.assertEqual("start2", groupers[1]["start"])
self.assertEqual("haltBefore", groupers[1]["haltBefore"])
self.assertEqual("start3", groupers[2]["start"])
self.assertEqual("continuePast", groupers[2]["continuePast"])
self.assertEqual("start4", groupers[3]["start"])
self.assertEqual("continueThrough", groupers[3]["continueThrough"])
exclude = result["exclude"]
self.assertTrue(isinstance(exclude, JsonArray))
self.assertEqual(3, len(exclude))
self.assertEqual("exclude1", exclude[0])
self.assertEqual("exclude2", exclude[1])
self.assertEqual("exclude3", exclude[2])
rename = result["rename_logfile"]
self.assertTrue(isinstance(rename, JsonObject))
self.assertEqual("rename", rename["match"])
self.assertEqual("renamed", rename["replacement"])
redaction = result["redaction_rules"]
self.assertTrue(isinstance(exclude, JsonArray))
self.assertEqual(3, len(exclude))
self.assertEqual("redacted1", redaction[0]["match_expression"])
self.assertEqual("redacted2", redaction[1]["match_expression"])
self.assertEqual("replaced2", redaction[1]["replacement"])
self.assertEqual("redacted3", redaction[2]["match_expression"])
self.assertEqual("replaced3", redaction[2]["replacement"])
self.assertEqual("salt3", redaction[2]["hash_salt"])
def test_full_config(self):
annotations = {
"log.config.scalyr.com/path": "/some/path.log",
"log.config.scalyr.com/attributes.parser": "accessLog",
"log.config.scalyr.com/attributes.service": "memcache",
"log.config.scalyr.com/sampling_rules.10.match_expression":
"10-INFO",
"log.config.scalyr.com/sampling_rules.10.sampling_rate": 10,
"log.config.scalyr.com/sampling_rules.2.match_expression":
"2-INFO",
"log.config.scalyr.com/sampling_rules.2.sampling_rate": 2,
"log.config.scalyr.com/line_groupers.4.start": "start4",
"log.config.scalyr.com/line_groupers.4.continueThrough":
"continueThrough",
"log.config.scalyr.com/line_groupers.3.start": "start3",
"log.config.scalyr.com/line_groupers.3.continuePast":
"continuePast",
"log.config.scalyr.com/line_groupers.2.start": "start2",
"log.config.scalyr.com/line_groupers.2.haltBefore": "haltBefore",
"log.config.scalyr.com/line_groupers.1.start": "start1",
"log.config.scalyr.com/line_groupers.1.haltWith": "haltWith",
"log.config.scalyr.com/exclude.1": "exclude1",
"log.config.scalyr.com/exclude.2": "exclude2",
"log.config.scalyr.com/exclude.3": "exclude3",
"log.config.scalyr.com/rename_logfile.match": "rename",
"log.config.scalyr.com/rename_logfile.replacement": "renamed",
"log.config.scalyr.com/redaction_rules.1.match_expression":
"redacted1",
"log.config.scalyr.com/redaction_rules.2.match_expression":
"redacted2",
"log.config.scalyr.com/redaction_rules.2.replacement": "replaced2",
"log.config.scalyr.com/redaction_rules.3.match_expression":
"redacted3",
"log.config.scalyr.com/redaction_rules.3.replacement": "replaced3",
"log.config.scalyr.com/redaction_rules.3.hash_salt": "salt3",
}
result = annotation_config.process_annotations(annotations)
self.assertEquals(7, len(result))
self.assertTrue('path' in result)
self.assertTrue('attributes' in result)
self.assertTrue('sampling_rules' in result)
self.assertTrue('line_groupers' in result)
self.assertTrue('exclude' in result)
self.assertTrue('rename_logfile' in result)
self.assertTrue('redaction_rules' in result)
attrs = result['attributes']
self.assertTrue(isinstance(attrs, JsonObject))
self.assertEquals(2, len(attrs))
self.assertTrue('parser' in attrs)
self.assertTrue('service' in attrs)
self.assertEquals('accessLog', attrs['parser'])
self.assertEquals('memcache', attrs['service'])
sampling = result['sampling_rules']
self.assertTrue(isinstance(sampling, JsonArray))
self.assertEqual(2, len(sampling))
self.assertEqual('2-INFO', sampling[0]['match_expression'])
self.assertEqual(2, sampling[0]['sampling_rate'])
self.assertEqual('10-INFO', sampling[1]['match_expression'])
self.assertEqual(10, sampling[1]['sampling_rate'])
groupers = result['line_groupers']
self.assertTrue(isinstance(groupers, JsonArray))
self.assertEqual(4, len(groupers))
self.assertEqual('start1', groupers[0]['start'])
self.assertEqual('haltWith', groupers[0]['haltWith'])
self.assertEqual('start2', groupers[1]['start'])
self.assertEqual('haltBefore', groupers[1]['haltBefore'])
self.assertEqual('start3', groupers[2]['start'])
self.assertEqual('continuePast', groupers[2]['continuePast'])
self.assertEqual('start4', groupers[3]['start'])
self.assertEqual('continueThrough', groupers[3]['continueThrough'])
exclude = result['exclude']
self.assertTrue(isinstance(exclude, JsonArray))
self.assertEqual(3, len(exclude))
self.assertEqual('exclude1', exclude[0])
self.assertEqual('exclude2', exclude[1])
self.assertEqual('exclude3', exclude[2])
rename = result['rename_logfile']
self.assertTrue(isinstance(rename, JsonObject))
self.assertEqual('rename', rename['match'])
self.assertEqual('renamed', rename['replacement'])
redaction = result['redaction_rules']
self.assertTrue(isinstance(exclude, JsonArray))
self.assertEqual(3, len(exclude))
self.assertEqual('redacted1', redaction[0]['match_expression'])
self.assertEqual('redacted2', redaction[1]['match_expression'])
self.assertEqual('replaced2', redaction[1]['replacement'])
self.assertEqual('redacted3', redaction[2]['match_expression'])
self.assertEqual('replaced3', redaction[2]['replacement'])
self.assertEqual('salt3', redaction[2]['hash_salt'])