def test_list_of_dicts(self): annotations = { "log.config.scalyr.com/10.match_expression": "fourth", "log.config.scalyr.com/10.sampling_rate": 4, "log.config.scalyr.com/2.match_expression": "third", "log.config.scalyr.com/2.sampling_rate": 3, "log.config.scalyr.com/0.match_expression": "first", "log.config.scalyr.com/0.sampling_rate": 1, "log.config.scalyr.com/1.match_expression": "second", "log.config.scalyr.com/1.sampling_rate": 2 } result = annotation_config.process_annotations(annotations) self.assertEquals(4, len(result)) self.assertTrue(isinstance(result, JsonArray)) self.assertEquals("first", result[0]['match_expression']) self.assertEquals(1, result[0]['sampling_rate']) self.assertEquals("second", result[1]['match_expression']) self.assertEquals(2, result[1]['sampling_rate']) self.assertEquals("third", result[2]['match_expression']) self.assertEquals(3, result[2]['sampling_rate']) self.assertEquals("fourth", result[3]['match_expression']) self.assertEquals(4, result[3]['sampling_rate'])
def test_dict_with_list(self): annotations = { "log.config.scalyr.com/rules.10.match_expression": "fourth", "log.config.scalyr.com/rules.10.sampling_rate": 4, "log.config.scalyr.com/rules.2.match_expression": "third", "log.config.scalyr.com/rules.2.sampling_rate": 3, "log.config.scalyr.com/rules.0.match_expression": "first", "log.config.scalyr.com/rules.0.sampling_rate": 1, "log.config.scalyr.com/rules.1.match_expression": "second", "log.config.scalyr.com/rules.1.sampling_rate": 2, } result = annotation_config.process_annotations(annotations) self.assertEquals(1, len(result)) result = result["rules"] self.assertEquals(4, len(result)) self.assertTrue(isinstance(result, JsonArray)) self.assertEquals("first", result[0]["match_expression"]) self.assertEquals(1, result[0]["sampling_rate"]) self.assertEquals("second", result[1]["match_expression"]) self.assertEquals(2, result[1]["sampling_rate"]) self.assertEquals("third", result[2]["match_expression"]) self.assertEquals(3, result[2]["sampling_rate"]) self.assertEquals("fourth", result[3]["match_expression"]) self.assertEquals(4, result[3]["sampling_rate"]) self.assertEquals(4, result[3]["sampling_rate"])
def test_dict_values(self): annotations = { "log.config.scalyr.com/attributes.parser": "accessLog", "log.config.scalyr.com/attributes.container": "my-container", "log.config.scalyr.com/attributes.amazing": "yes it is", "log.config.scalyr.com/rename_logfile.match": "/var/log/(.*).log", "log.config.scalyr.com/rename_logfile.replacement": "/scalyr/\\1.log", } result = annotation_config.process_annotations(annotations) self.assertEquals(2, len(result)) self.assertTrue('attributes' in result) self.assertTrue('rename_logfile' in result) attrs = result['attributes'] self.assertEquals(3, len(attrs)) self.assertTrue('parser' in attrs) self.assertTrue('container' in attrs) self.assertTrue('amazing' in attrs) self.assertEquals("accessLog", attrs['parser']) self.assertEquals("my-container", attrs['container']) self.assertEquals("yes it is", attrs['amazing'])
def test_list_of_dicts(self): annotations = { "log.config.scalyr.com/10.match_expression": "fourth", "log.config.scalyr.com/10.sampling_rate": 4, "log.config.scalyr.com/2.match_expression": "third", "log.config.scalyr.com/2.sampling_rate": 3, "log.config.scalyr.com/0.match_expression": "first", "log.config.scalyr.com/0.sampling_rate": 1, "log.config.scalyr.com/1.match_expression": "second", "log.config.scalyr.com/1.sampling_rate": 2 } result = annotation_config.process_annotations( annotations ) self.assertEquals( 4, len( result ) ) self.assertTrue( isinstance( result, JsonArray ) ) self.assertEquals( "first", result[0]['match_expression'] ) self.assertEquals( 1, result[0]['sampling_rate'] ) self.assertEquals( "second", result[1]['match_expression'] ) self.assertEquals( 2, result[1]['sampling_rate'] ) self.assertEquals( "third", result[2]['match_expression'] ) self.assertEquals( 3, result[2]['sampling_rate'] ) self.assertEquals( "fourth", result[3]['match_expression'] ) self.assertEquals( 4, result[3]['sampling_rate'] )
def test_dict_values(self): annotations = { "log.config.scalyr.com/attributes.parser": "accessLog", "log.config.scalyr.com/attributes.container": "my-container", "log.config.scalyr.com/attributes.amazing": "yes it is", "log.config.scalyr.com/rename_logfile.match": "/var/log/(.*).log", "log.config.scalyr.com/rename_logfile.replacement": "/scalyr/\\1.log", } result = annotation_config.process_annotations( annotations ) self.assertEquals( 2, len( result ) ) self.assertTrue( 'attributes' in result ) self.assertTrue( 'rename_logfile' in result ) attrs = result['attributes'] self.assertEquals( 3, len( attrs ) ) self.assertTrue( 'parser' in attrs ) self.assertTrue( 'container' in attrs ) self.assertTrue( 'amazing' in attrs ) self.assertEquals( "accessLog", attrs['parser'] ) self.assertEquals( "my-container", attrs['container'] ) self.assertEquals( "yes it is", attrs['amazing'] )
def test_invalid_annotations( self ): annotations = { "some.other.value": 10, "not.a.scalyr.annotation": "no it's not", } result = process_annotations( annotations ) self.assertEquals( 0, len( result.keys() ) )
def test_mixed_list_and_dict_key(self): annotations = { "log.config.scalyr.com/test.2.match_expression": "third", "log.config.scalyr.com/test.2.sampling_rate": 3, "log.config.scalyr.com/test.bad.match_expression": "first", "log.config.scalyr.com/test.bad.sampling_rate": 1, "log.config.scalyr.com/test.1.match_expression": "second", "log.config.scalyr.com/test.1.sampling_rate": 2 } self.assertRaises(annotation_config.BadAnnotationConfig, lambda: annotation_config.process_annotations( annotations ) )
def test_no_scalyr_annotations(self): annotations = { "config.agent.not-scalyr.com/attributes.parser": "accessLog", "config.agent.not-scalyr.com/log_path": "/var/log/access.log", "config.agent.not-scalyr.com/sampling_rules.0.match_expression": "INFO", "config.agent.not-scalyr.com/sampling_rules.0.sampling_rate": 0.1, } result = annotation_config.process_annotations( annotations ) self.assertFalse( result )
def test_mixed_list_and_dict_key(self): annotations = { "log.config.scalyr.com/test.2.match_expression": "third", "log.config.scalyr.com/test.2.sampling_rate": 3, "log.config.scalyr.com/test.bad.match_expression": "first", "log.config.scalyr.com/test.bad.sampling_rate": 1, "log.config.scalyr.com/test.1.match_expression": "second", "log.config.scalyr.com/test.1.sampling_rate": 2 } self.assertRaises(annotation_config.BadAnnotationConfig, lambda: annotation_config.process_annotations( annotations ) )
def test_no_scalyr_annotations(self): annotations = { "config.agent.not-scalyr.com/attributes.parser": "accessLog", "config.agent.not-scalyr.com/log_path": "/var/log/access.log", "config.agent.not-scalyr.com/sampling_rules.0.match_expression": "INFO", "config.agent.not-scalyr.com/sampling_rules.0.sampling_rate": 0.1, } result = annotation_config.process_annotations( annotations ) self.assertFalse( result )
def test_annotation_object( self ): annotations = { "log.config.scalyr.com/item1" : "item1", "log.config.scalyr.com/item2" : "item2", "log.config.scalyr.com/item3" : "item3" } result = process_annotations( annotations ) self.assertEquals( 3, len( result.keys() ) ) self.assertEquals( 'item1', result['item1'] ) self.assertEquals( 'item2', result['item2'] ) self.assertEquals( 'item3', result['item3'] )
def test_annotation_object(self): annotations = { "log.config.scalyr.com/item1": "item1", "log.config.scalyr.com/item2": "item2", "log.config.scalyr.com/item3": "item3", } result = process_annotations(annotations) self.assertEquals(3, len(list(result.keys()))) self.assertEquals("item1", result["item1"]) self.assertEquals("item2", result["item2"]) self.assertEquals("item3", result["item3"])
def test_list_value(self): annotations = { "log.config.scalyr.com/3": "three", "log.config.scalyr.com/2": "two", "log.config.scalyr.com/1": "one", } result = annotation_config.process_annotations( annotations ) self.assertTrue( isinstance( result, JsonArray ) ) self.assertEqual( 3, len( result ) ) self.assertEqual( 'one', result[0] ) self.assertEqual( 'two', result[1] ) self.assertEqual( 'three', result[2] )
def test_list_value(self): annotations = { "log.config.scalyr.com/3": "three", "log.config.scalyr.com/2": "two", "log.config.scalyr.com/1": "one", } result = annotation_config.process_annotations(annotations) self.assertTrue(isinstance(result, JsonArray)) self.assertEqual(3, len(result)) self.assertEqual('one', result[0]) self.assertEqual('two', result[1]) self.assertEqual('three', result[2])
def test_annotation_keys_with_hyphens( self ): annotations = { "com.scalyr.config.log.item1-1.element-1" : "item1 element 1", "com.scalyr.config.log.item1-2.element-2" : "item1 element 2", "com.scalyr.config.log.item1-3.element-3" : "item1 element 3", "com.scalyr.config.log.item2-1.element-1" : "item2 element 1", "com.scalyr.config.log.item2-2.element-2" : "item2 element 2", "com.scalyr.config.log.item2-3.element-3" : "item2 element 3", } result = process_annotations( annotations, annotation_prefix_re=re.compile( '^(com\.scalyr\.config\.log\.)(.+)' ), hyphens_as_underscores=True ) self.assertEquals( 'item1 element 1', result['item1_1']['element_1'] ) self.assertEquals( 'item1 element 2', result['item1_2']['element_2'] ) self.assertEquals( 'item1 element 3', result['item1_3']['element_3'] ) self.assertEquals( 'item2 element 1', result['item2_1']['element_1'] ) self.assertEquals( 'item2 element 2', result['item2_2']['element_2'] ) self.assertEquals( 'item2 element 3', result['item2_3']['element_3'] )
def test_plain_values(self): annotations = { "log.config.scalyr.com/someKey": "someValue", "log.config.scalyr.com/log_path": "/var/log/access.log", "log.config.scalyr.com/important_field": "important_value", } result = annotation_config.process_annotations( annotations ) self.assertEquals( 3, len( result ) ) self.assertTrue( 'someKey' in result ) self.assertTrue( 'log_path' in result ) self.assertTrue( 'important_field' in result ) self.assertEquals( "someValue", result['someKey'] ) self.assertEquals( "/var/log/access.log", result['log_path'] ) self.assertEquals( "important_value", result['important_field'] )
def test_plain_values(self): annotations = { "log.config.scalyr.com/someKey": "someValue", "log.config.scalyr.com/log_path": "/var/log/access.log", "log.config.scalyr.com/important_field": "important_value", } result = annotation_config.process_annotations(annotations) self.assertEquals(3, len(result)) self.assertTrue('someKey' in result) self.assertTrue('log_path' in result) self.assertTrue('important_field' in result) self.assertEquals("someValue", result['someKey']) self.assertEquals("/var/log/access.log", result['log_path']) self.assertEquals("important_value", result['important_field'])
def process_object(self, obj): """ Generate a PodInfo object from a JSON object @param pod: The JSON object returned as a response to querying a specific pod from the k8s API @return A PodInfo object """ result = {} metadata = obj.get('metadata', {}) spec = obj.get('spec', {}) labels = metadata.get('labels', {}) annotations = metadata.get('annotations', {}) owners = metadata.get('ownerReferences', []) pod_name = metadata.get("name", '') namespace = metadata.get("namespace", '') deployment_name = self._get_deployment_name_from_owners( owners, namespace) daemonset_name = self._get_daemonset_name_from_owners( owners, namespace) container_names = [] for container in spec.get('containers', []): container_names.append( container.get('name', 'invalid-container-name')) try: annotations = annotation_config.process_annotations(annotations) except BadAnnotationConfig, e: self._logger.warning( "Bad Annotation config for %s/%s. All annotations ignored. %s" % (namespace, pod_name, str(e)), limit_once_per_x_secs=300, limit_key='bad-annotation-config-%s' % info.uid) annotations = JsonObject()
def test_list_of_lists(self): annotations = { "log.config.scalyr.com/2.10.match_expression": "fourth", "log.config.scalyr.com/2.10.sampling_rate": 4, "log.config.scalyr.com/2.2.match_expression": "third", "log.config.scalyr.com/2.2.sampling_rate": 3, "log.config.scalyr.com/1.0.match_expression": "first", "log.config.scalyr.com/1.0.sampling_rate": 1, "log.config.scalyr.com/1.1.match_expression": "second", "log.config.scalyr.com/1.1.sampling_rate": 2 } #import pdb; pdb.set_trace() result = annotation_config.process_annotations( annotations ) self.assertEquals( 2, len( result ) ) first = result[0] self.assertEquals( 2, len( first ) ) self.assertTrue( isinstance( first, JsonArray ) ) self.assertEquals( "first", first[0]['match_expression'] ) self.assertEquals( 1, first[0]['sampling_rate'] ) self.assertEquals( "second", first[1]['match_expression'] ) self.assertEquals( 2, first[1]['sampling_rate'] ) second = result[1] self.assertEquals( 2, len( second ) ) self.assertTrue( isinstance( second, JsonArray ) ) self.assertEquals( "third", second[0]['match_expression'] ) self.assertEquals( 3, second[0]['sampling_rate'] ) self.assertEquals( "fourth", second[1]['match_expression'] ) self.assertEquals( 4, second[1]['sampling_rate'] )
def test_list_of_lists(self): annotations = { "log.config.scalyr.com/2.10.match_expression": "fourth", "log.config.scalyr.com/2.10.sampling_rate": 4, "log.config.scalyr.com/2.2.match_expression": "third", "log.config.scalyr.com/2.2.sampling_rate": 3, "log.config.scalyr.com/1.0.match_expression": "first", "log.config.scalyr.com/1.0.sampling_rate": 1, "log.config.scalyr.com/1.1.match_expression": "second", "log.config.scalyr.com/1.1.sampling_rate": 2 } #import pdb; pdb.set_trace() result = annotation_config.process_annotations(annotations) self.assertEquals(2, len(result)) first = result[0] self.assertEquals(2, len(first)) self.assertTrue(isinstance(first, JsonArray)) self.assertEquals("first", first[0]['match_expression']) self.assertEquals(1, first[0]['sampling_rate']) self.assertEquals("second", first[1]['match_expression']) self.assertEquals(2, first[1]['sampling_rate']) second = result[1] self.assertEquals(2, len(second)) self.assertTrue(isinstance(second, JsonArray)) self.assertEquals("third", second[0]['match_expression']) self.assertEquals(3, second[0]['sampling_rate']) self.assertEquals("fourth", second[1]['match_expression']) self.assertEquals(4, second[1]['sampling_rate'])
def test_annotation_nested_object(self): annotations = { "log.config.scalyr.com/item1.nest1": "item1 nest1", "log.config.scalyr.com/item1.nest2": "item1 nest2", "log.config.scalyr.com/item1.nest3": "item1 nest3", "log.config.scalyr.com/item2.nest1": "item2 nest1", "log.config.scalyr.com/item2.nest2": "item2 nest2", "log.config.scalyr.com/item2.nest3": "item2 nest3", "log.config.scalyr.com/item2.nest4": "item2 nest4", } result = process_annotations(annotations) self.assertEquals(2, len(list(result.keys()))) self.assertEquals(3, len(result["item1"])) self.assertEquals(4, len(result["item2"])) self.assertEquals("item1 nest1", result["item1"]["nest1"]) self.assertEquals("item1 nest2", result["item1"]["nest2"]) self.assertEquals("item1 nest3", result["item1"]["nest3"]) self.assertEquals("item2 nest1", result["item2"]["nest1"]) self.assertEquals("item2 nest2", result["item2"]["nest2"]) self.assertEquals("item2 nest3", result["item2"]["nest3"]) self.assertEquals("item2 nest4", result["item2"]["nest4"])
def test_annotation_nested_object( self ): annotations = { "log.config.scalyr.com/item1.nest1" : "item1 nest1", "log.config.scalyr.com/item1.nest2" : "item1 nest2", "log.config.scalyr.com/item1.nest3" : "item1 nest3", "log.config.scalyr.com/item2.nest1" : "item2 nest1", "log.config.scalyr.com/item2.nest2" : "item2 nest2", "log.config.scalyr.com/item2.nest3" : "item2 nest3", "log.config.scalyr.com/item2.nest4" : "item2 nest4" } result = process_annotations( annotations ) self.assertEquals( 2, len( result.keys() ) ) self.assertEquals( 3, len( result['item1'] ) ) self.assertEquals( 4, len( result['item2'] ) ) self.assertEquals( 'item1 nest1', result['item1']['nest1'] ) self.assertEquals( 'item1 nest2', result['item1']['nest2'] ) self.assertEquals( 'item1 nest3', result['item1']['nest3'] ) self.assertEquals( 'item2 nest1', result['item2']['nest1'] ) self.assertEquals( 'item2 nest2', result['item2']['nest2'] ) self.assertEquals( 'item2 nest3', result['item2']['nest3'] ) self.assertEquals( 'item2 nest4', result['item2']['nest4'] )
def test_annotation_array( self ): annotations = { "log.config.scalyr.com/item1.20" : "item1 element 2", "log.config.scalyr.com/item1.0" : "item1 element 0", "log.config.scalyr.com/item1.10" : "item1 element 1", "log.config.scalyr.com/item2.0" : "item2 element 0", "log.config.scalyr.com/item2.1" : "item2 element 1", "log.config.scalyr.com/item2.2" : "item2 element 2", "log.config.scalyr.com/item2.3" : "item2 element 3", } result = process_annotations( annotations ) self.assertEquals( 2, len( result.keys() ) ) self.assertEquals( 3, len( result['item1'] ) ) self.assertEquals( 4, len( result['item2'] ) ) self.assertEquals( 'item1 element 0', result['item1'][0] ) self.assertEquals( 'item1 element 1', result['item1'][1] ) self.assertEquals( 'item1 element 2', result['item1'][2] ) self.assertEquals( 'item2 element 0', result['item2'][0] ) self.assertEquals( 'item2 element 1', result['item2'][1] ) self.assertEquals( 'item2 element 2', result['item2'][2] ) self.assertEquals( 'item2 element 3', result['item2'][3] )
def test_full_config(self): annotations = { "log.config.scalyr.com/path": "/some/path.log", "log.config.scalyr.com/attributes.parser": "accessLog", "log.config.scalyr.com/attributes.service": "memcache", "log.config.scalyr.com/sampling_rules.10.match_expression": "10-INFO", "log.config.scalyr.com/sampling_rules.10.sampling_rate": 10, "log.config.scalyr.com/sampling_rules.2.match_expression": "2-INFO", "log.config.scalyr.com/sampling_rules.2.sampling_rate": 2, "log.config.scalyr.com/line_groupers.4.start": "start4", "log.config.scalyr.com/line_groupers.4.continueThrough": "continueThrough", "log.config.scalyr.com/line_groupers.3.start": "start3", "log.config.scalyr.com/line_groupers.3.continuePast": "continuePast", "log.config.scalyr.com/line_groupers.2.start": "start2", "log.config.scalyr.com/line_groupers.2.haltBefore": "haltBefore", "log.config.scalyr.com/line_groupers.1.start": "start1", "log.config.scalyr.com/line_groupers.1.haltWith": "haltWith", "log.config.scalyr.com/exclude.1": "exclude1", "log.config.scalyr.com/exclude.2": "exclude2", "log.config.scalyr.com/exclude.3": "exclude3", "log.config.scalyr.com/rename_logfile.match": "rename", "log.config.scalyr.com/rename_logfile.replacement": "renamed", "log.config.scalyr.com/redaction_rules.1.match_expression": "redacted1", "log.config.scalyr.com/redaction_rules.2.match_expression": "redacted2", "log.config.scalyr.com/redaction_rules.2.replacement": "replaced2", "log.config.scalyr.com/redaction_rules.3.match_expression": "redacted3", "log.config.scalyr.com/redaction_rules.3.replacement": "replaced3", "log.config.scalyr.com/redaction_rules.3.hash_salt": "salt3", } result = annotation_config.process_annotations( annotations ) self.assertEquals( 7, len( result ) ) self.assertTrue( 'path' in result ) self.assertTrue( 'attributes' in result ) self.assertTrue( 'sampling_rules' in result ) self.assertTrue( 'line_groupers' in result ) self.assertTrue( 'exclude' in result ) self.assertTrue( 'rename_logfile' in result ) self.assertTrue( 'redaction_rules' in result ) attrs = result['attributes'] self.assertTrue( isinstance( attrs, JsonObject ) ) self.assertEquals( 2, len( attrs ) ) self.assertTrue( 'parser' in attrs ) self.assertTrue( 'service' in attrs ) self.assertEquals( 'accessLog', attrs['parser'] ) self.assertEquals( 'memcache', attrs['service'] ) sampling = result['sampling_rules'] self.assertTrue( isinstance( sampling, JsonArray ) ) self.assertEqual( 2, len( sampling ) ) self.assertEqual( '2-INFO', sampling[0]['match_expression'] ) self.assertEqual( 2, sampling[0]['sampling_rate'] ) self.assertEqual( '10-INFO', sampling[1]['match_expression'] ) self.assertEqual( 10, sampling[1]['sampling_rate'] ) groupers = result['line_groupers'] self.assertTrue( isinstance( groupers, JsonArray ) ) self.assertEqual( 4, len( groupers ) ) self.assertEqual( 'start1', groupers[0]['start'] ) self.assertEqual( 'haltWith', groupers[0]['haltWith'] ) self.assertEqual( 'start2', groupers[1]['start'] ) self.assertEqual( 'haltBefore', groupers[1]['haltBefore'] ) self.assertEqual( 'start3', groupers[2]['start'] ) self.assertEqual( 'continuePast', groupers[2]['continuePast'] ) self.assertEqual( 'start4', groupers[3]['start'] ) self.assertEqual( 'continueThrough', groupers[3]['continueThrough'] ) exclude = result['exclude'] self.assertTrue( isinstance( exclude, JsonArray ) ) self.assertEqual( 3, len( exclude ) ) self.assertEqual( 'exclude1', exclude[0] ) self.assertEqual( 'exclude2', exclude[1] ) self.assertEqual( 'exclude3', exclude[2] ) rename = result['rename_logfile'] self.assertTrue( isinstance( rename, JsonObject ) ) self.assertEqual( 'rename', rename['match'] ) self.assertEqual( 'renamed', rename['replacement'] ) redaction = result['redaction_rules'] self.assertTrue( isinstance( exclude, JsonArray ) ) self.assertEqual( 3, len( exclude ) ) self.assertEqual( 'redacted1', redaction[0]['match_expression'] ) self.assertEqual( 'redacted2', redaction[1]['match_expression'] ) self.assertEqual( 'replaced2', redaction[1]['replacement'] ) self.assertEqual( 'redacted3', redaction[2]['match_expression'] ) self.assertEqual( 'replaced3', redaction[2]['replacement'] ) self.assertEqual( 'salt3', redaction[2]['hash_salt'] )
def test_full_config(self): annotations = { "log.config.scalyr.com/path": "/some/path.log", "log.config.scalyr.com/attributes.parser": "accessLog", "log.config.scalyr.com/attributes.service": "memcache", "log.config.scalyr.com/sampling_rules.10.match_expression": "10-INFO", "log.config.scalyr.com/sampling_rules.10.sampling_rate": 10, "log.config.scalyr.com/sampling_rules.2.match_expression": "2-INFO", "log.config.scalyr.com/sampling_rules.2.sampling_rate": 2, "log.config.scalyr.com/line_groupers.4.start": "start4", "log.config.scalyr.com/line_groupers.4.continueThrough": "continueThrough", "log.config.scalyr.com/line_groupers.3.start": "start3", "log.config.scalyr.com/line_groupers.3.continuePast": "continuePast", "log.config.scalyr.com/line_groupers.2.start": "start2", "log.config.scalyr.com/line_groupers.2.haltBefore": "haltBefore", "log.config.scalyr.com/line_groupers.1.start": "start1", "log.config.scalyr.com/line_groupers.1.haltWith": "haltWith", "log.config.scalyr.com/exclude.1": "exclude1", "log.config.scalyr.com/exclude.2": "exclude2", "log.config.scalyr.com/exclude.3": "exclude3", "log.config.scalyr.com/rename_logfile.match": "rename", "log.config.scalyr.com/rename_logfile.replacement": "renamed", "log.config.scalyr.com/redaction_rules.1.match_expression": "redacted1", "log.config.scalyr.com/redaction_rules.2.match_expression": "redacted2", "log.config.scalyr.com/redaction_rules.2.replacement": "replaced2", "log.config.scalyr.com/redaction_rules.3.match_expression": "redacted3", "log.config.scalyr.com/redaction_rules.3.replacement": "replaced3", "log.config.scalyr.com/redaction_rules.3.hash_salt": "salt3", } result = annotation_config.process_annotations(annotations) self.assertEquals(7, len(result)) self.assertTrue("path" in result) self.assertTrue("attributes" in result) self.assertTrue("sampling_rules" in result) self.assertTrue("line_groupers" in result) self.assertTrue("exclude" in result) self.assertTrue("rename_logfile" in result) self.assertTrue("redaction_rules" in result) attrs = result["attributes"] self.assertTrue(isinstance(attrs, JsonObject)) self.assertEquals(2, len(attrs)) self.assertTrue("parser" in attrs) self.assertTrue("service" in attrs) self.assertEquals("accessLog", attrs["parser"]) self.assertEquals("memcache", attrs["service"]) sampling = result["sampling_rules"] self.assertTrue(isinstance(sampling, JsonArray)) self.assertEqual(2, len(sampling)) self.assertEqual("2-INFO", sampling[0]["match_expression"]) self.assertEqual(2, sampling[0]["sampling_rate"]) self.assertEqual("10-INFO", sampling[1]["match_expression"]) self.assertEqual(10, sampling[1]["sampling_rate"]) groupers = result["line_groupers"] self.assertTrue(isinstance(groupers, JsonArray)) self.assertEqual(4, len(groupers)) self.assertEqual("start1", groupers[0]["start"]) self.assertEqual("haltWith", groupers[0]["haltWith"]) self.assertEqual("start2", groupers[1]["start"]) self.assertEqual("haltBefore", groupers[1]["haltBefore"]) self.assertEqual("start3", groupers[2]["start"]) self.assertEqual("continuePast", groupers[2]["continuePast"]) self.assertEqual("start4", groupers[3]["start"]) self.assertEqual("continueThrough", groupers[3]["continueThrough"]) exclude = result["exclude"] self.assertTrue(isinstance(exclude, JsonArray)) self.assertEqual(3, len(exclude)) self.assertEqual("exclude1", exclude[0]) self.assertEqual("exclude2", exclude[1]) self.assertEqual("exclude3", exclude[2]) rename = result["rename_logfile"] self.assertTrue(isinstance(rename, JsonObject)) self.assertEqual("rename", rename["match"]) self.assertEqual("renamed", rename["replacement"]) redaction = result["redaction_rules"] self.assertTrue(isinstance(exclude, JsonArray)) self.assertEqual(3, len(exclude)) self.assertEqual("redacted1", redaction[0]["match_expression"]) self.assertEqual("redacted2", redaction[1]["match_expression"]) self.assertEqual("replaced2", redaction[1]["replacement"]) self.assertEqual("redacted3", redaction[2]["match_expression"]) self.assertEqual("replaced3", redaction[2]["replacement"]) self.assertEqual("salt3", redaction[2]["hash_salt"])
def test_full_config(self): annotations = { "log.config.scalyr.com/path": "/some/path.log", "log.config.scalyr.com/attributes.parser": "accessLog", "log.config.scalyr.com/attributes.service": "memcache", "log.config.scalyr.com/sampling_rules.10.match_expression": "10-INFO", "log.config.scalyr.com/sampling_rules.10.sampling_rate": 10, "log.config.scalyr.com/sampling_rules.2.match_expression": "2-INFO", "log.config.scalyr.com/sampling_rules.2.sampling_rate": 2, "log.config.scalyr.com/line_groupers.4.start": "start4", "log.config.scalyr.com/line_groupers.4.continueThrough": "continueThrough", "log.config.scalyr.com/line_groupers.3.start": "start3", "log.config.scalyr.com/line_groupers.3.continuePast": "continuePast", "log.config.scalyr.com/line_groupers.2.start": "start2", "log.config.scalyr.com/line_groupers.2.haltBefore": "haltBefore", "log.config.scalyr.com/line_groupers.1.start": "start1", "log.config.scalyr.com/line_groupers.1.haltWith": "haltWith", "log.config.scalyr.com/exclude.1": "exclude1", "log.config.scalyr.com/exclude.2": "exclude2", "log.config.scalyr.com/exclude.3": "exclude3", "log.config.scalyr.com/rename_logfile.match": "rename", "log.config.scalyr.com/rename_logfile.replacement": "renamed", "log.config.scalyr.com/redaction_rules.1.match_expression": "redacted1", "log.config.scalyr.com/redaction_rules.2.match_expression": "redacted2", "log.config.scalyr.com/redaction_rules.2.replacement": "replaced2", "log.config.scalyr.com/redaction_rules.3.match_expression": "redacted3", "log.config.scalyr.com/redaction_rules.3.replacement": "replaced3", "log.config.scalyr.com/redaction_rules.3.hash_salt": "salt3", } result = annotation_config.process_annotations(annotations) self.assertEquals(7, len(result)) self.assertTrue('path' in result) self.assertTrue('attributes' in result) self.assertTrue('sampling_rules' in result) self.assertTrue('line_groupers' in result) self.assertTrue('exclude' in result) self.assertTrue('rename_logfile' in result) self.assertTrue('redaction_rules' in result) attrs = result['attributes'] self.assertTrue(isinstance(attrs, JsonObject)) self.assertEquals(2, len(attrs)) self.assertTrue('parser' in attrs) self.assertTrue('service' in attrs) self.assertEquals('accessLog', attrs['parser']) self.assertEquals('memcache', attrs['service']) sampling = result['sampling_rules'] self.assertTrue(isinstance(sampling, JsonArray)) self.assertEqual(2, len(sampling)) self.assertEqual('2-INFO', sampling[0]['match_expression']) self.assertEqual(2, sampling[0]['sampling_rate']) self.assertEqual('10-INFO', sampling[1]['match_expression']) self.assertEqual(10, sampling[1]['sampling_rate']) groupers = result['line_groupers'] self.assertTrue(isinstance(groupers, JsonArray)) self.assertEqual(4, len(groupers)) self.assertEqual('start1', groupers[0]['start']) self.assertEqual('haltWith', groupers[0]['haltWith']) self.assertEqual('start2', groupers[1]['start']) self.assertEqual('haltBefore', groupers[1]['haltBefore']) self.assertEqual('start3', groupers[2]['start']) self.assertEqual('continuePast', groupers[2]['continuePast']) self.assertEqual('start4', groupers[3]['start']) self.assertEqual('continueThrough', groupers[3]['continueThrough']) exclude = result['exclude'] self.assertTrue(isinstance(exclude, JsonArray)) self.assertEqual(3, len(exclude)) self.assertEqual('exclude1', exclude[0]) self.assertEqual('exclude2', exclude[1]) self.assertEqual('exclude3', exclude[2]) rename = result['rename_logfile'] self.assertTrue(isinstance(rename, JsonObject)) self.assertEqual('rename', rename['match']) self.assertEqual('renamed', rename['replacement']) redaction = result['redaction_rules'] self.assertTrue(isinstance(exclude, JsonArray)) self.assertEqual(3, len(exclude)) self.assertEqual('redacted1', redaction[0]['match_expression']) self.assertEqual('redacted2', redaction[1]['match_expression']) self.assertEqual('replaced2', redaction[1]['replacement']) self.assertEqual('redacted3', redaction[2]['match_expression']) self.assertEqual('replaced3', redaction[2]['replacement']) self.assertEqual('salt3', redaction[2]['hash_salt'])