def test_normalize_policies_1(): policies = EntitiesSet({ EntityWrapper( policy( name='policy-1', entitlements=['entitlement1', 'entitlement2', 'entitlement3'])), EntityWrapper( policy(name='policy-2', entitlements=[ 'entitlement1', 'entitlement2', 'entitlement3', ])), }) entitlements = EntitiesSet() policies_set, conflicts = resolve_entities( policies, [(entitlements, 'entitlements')]) assert conflicts == { 'policy-1': { 'entitlements': frozenset({'entitlement1', 'entitlement2', 'entitlement3'}) }, 'policy-2': { 'entitlements': frozenset({'entitlement1', 'entitlement2', 'entitlement3'}) } }
def test_dependencies_3(): """ several dependencies (even nested) See EntityDep5 for details """ api = load_test_open_api_spec() EntityDep1 = api.entities['EntityDep1'].cls EntityDep5 = api.entities['EntityDep5'].cls EntityDep5_Obj1 = api.entities['EntityDep5_Obj1'].cls EntityDep5_Obj1_Obj2 = api.entities['EntityDep5_Obj1_Obj2'].cls deps1 = EntitiesSet({ EntityWrapper(EntityDep1(id='d11', name='dep11')), EntityWrapper(EntityDep1(id='d12', name='dep12')), EntityWrapper(EntityDep1(id='d13', name='dep13')), }) data = {'id': 'd51', 'name': 'dep51', 'obj1': {'obj2': {'dep1': 'dep11'}}} deps5 = EntitiesSet( {EntityWrapper(K8S_LOADER.load(data, None, EntityDep5))}) deps5_resolved, conflicts = resolve_entities(deps5, [(deps1, 'obj1.obj2.dep1')]) assert conflicts is None assert deps5_resolved.entities == { EntityWrapper( EntityDep5( id='d51', name='dep51', obj1=EntityDep5_Obj1(obj2=EntityDep5_Obj1_Obj2(dep1='d11')))) }
def test_compare_entity_with_secrets_with_metadata_4(): EntityTest2 = load_test_open_api_spec( reload=True, k8s_get_secret=_k8s_get_secret).entities['EntityTest2'].cls data_1 = { 'fieldOne': { 'type': 'k8s/secret', 'name': 'secret-storage-1', 'key': 'field-one' }, 'fieldTwo': 'this is write only', 'fieldThree': 'this is a field', } data_2 = { 'fieldOne': { 'type': 'k8s/secret', 'name': 'secret-storage-1', 'key': 'field-one' }, 'fieldTwo': 'this is write only', 'fieldThree': 'this is a field', 'created': '2020-09-10T12:20:14Z', 'updated': '2020-09-20T12:20:14Z' } appgate_metadata = { 'generation': 2, 'latestGeneration': 3, 'creationTimestamp': '2020-09-10T10:20:14Z', 'modificationTimestamp': '2020-09-20T12:19:14Z', } e1 = EntityWrapper(K8S_LOADER.load(data_1, appgate_metadata, EntityTest2)) e2 = EntityWrapper(APPGATE_LOADER.load(data_2, None, EntityTest2)) assert e1 == e2
def test_normalize_entitlements_1(): entitlements = EntitiesSet({ EntityWrapper( entitlement(name='entitlement-1', conditions=['condition1', 'condition2', 'condition3'])), EntityWrapper( entitlement(name='entitlement-2', conditions=[ 'condition1', 'condition2', 'condition3', ])), }) conditions = EntitiesSet() entitlements_set, conflicts = resolve_entities( entitlements, [(conditions, 'conditions')]) assert entitlements_set.entities == entitlements.entities assert conflicts == { 'entitlement-1': { 'conditions': frozenset({'condition1', 'condition2', 'condition3'}) }, 'entitlement-2': { 'conditions': frozenset({'condition1', 'condition2', 'condition3'}) }, }
def test_compare_entity_with_secrets(): """ If we are missing metadata frm k8s we should always update the entity """ EntityTest2 = load_test_open_api_spec( reload=True, k8s_get_secret=_k8s_get_secret).entities['EntityTest2'].cls data_1 = { 'fieldOne': { 'type': 'k8s/secret', 'name': 'secret-storage-1', 'key': 'field-one' }, 'fieldTwo': 'this is write only', 'fieldThree': 'this is a field', } data_2 = { 'fieldOne': { 'type': 'k8s/secret', 'name': 'secret-storage-1', 'key': 'field-one' }, 'fieldTwo': 'this is write only', 'fieldThree': 'this is a field', 'created': '2020-09-10T12:20:14Z', 'updated': '2020-09-10T12:20:14Z' } e1 = EntityWrapper(K8S_LOADER.load(data_1, None, EntityTest2)) e2 = EntityWrapper(APPGATE_LOADER.load(data_2, None, EntityTest2)) assert e1 != e2
def test_compare_plan_entity_pem(): EntityCert = load_test_open_api_spec( secrets_key=None, reload=True).entities['EntityCert'].cls appgate_data = { 'name': 'c1', 'fieldOne': PEM_TEST, 'fieldTwo': { 'version': 1, 'serial': '3578', 'issuer': join_string(ISSUER), 'subject': join_string(SUBJECT), 'validFrom': '2012-08-22T05:26:54.000Z', 'validTo': '2017-08-21T05:26:54.000Z', 'fingerprint': 'Xw+1FmWBquZKEBwVg7G+vnToFKkeeooUuh6DXXj26ec=', 'certificate': join_string(CERTIFICATE_FIELD), 'subjectPublicKey': join_string(PUBKEY_FIELD), } } k8s_data = {'name': 'c1', 'fieldOne': PEM2} current_entities = EntitiesSet( {EntityWrapper(APPGATE_LOADER.load(appgate_data, None, EntityCert))}) new_e = K8S_LOADER.load(k8s_data, None, EntityCert) expected_entities = EntitiesSet({EntityWrapper(new_e)}) plan = compare_entities(current_entities, expected_entities, BUILTIN_TAGS) assert plan.modify.entities == frozenset({EntityWrapper(new_e)}) assert plan.modifications_diff == { 'c1': [ '--- \n', '+++ \n', '@@ -3,10 +3,10 @@\n', ' "fieldTwo": {\n', '- "version": 1,\n', '- "serial": "3578",\n', '+ "version": 3,\n', '+ "serial": "0",\n', ' "issuer": "[email protected], CN=Frank4DD Web CA, OU=WebCert Support, O=Frank4DD, L=Chuo-ku, ST=Tokyo, C=JP",\n', ' "subject": "CN=www.example.com, O=Frank4DD, ST=Tokyo, C=JP",\n', '- "validFrom": "2012-08-22T05:26:54.000Z",\n', '- "validTo": "2017-08-21T05:26:54.000Z",\n', '- "fingerprint": "Xw+1FmWBquZKEBwVg7G+vnToFKkeeooUuh6DXXj26ec=",\n', '- "certificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNFakNDQVhzQ0FnMzZNQTBHQ1NxR1NJYjNEUUVCQlFVQU1JR2JNUXN3Q1FZRFZRUUdF' 'd0pLVURFT01Bd0cKQTFVRUNCTUZWRzlyZVc4eEVEQU9CZ05WQkFjVEIwTm9kVzh0YTNVeEVUQVBCZ05WQkFvVENFWnlZVzVyTkVSRQpNUmd3RmdZRFZRUUxFdzlYWldKRFpYS' 'jBJRk4xY0hCdmNuUXhHREFXQmdOVkJBTVREMFp5WVc1ck5FUkVJRmRsCllpQkRRVEVqTUNFR0NTcUdTSWIzRFFFSkFSWVVjM1Z3Y0c5eWRFQm1jbUZ1YXpSa1pDNWpiMjB3SG' 'hjTk1USXcKT0RJeU1EVXlOalUwV2hjTk1UY3dPREl4TURVeU5qVTBXakJLTVFzd0NRWURWUVFHRXdKS1VERU9NQXdHQTFVRQpDQXdGVkc5cmVXOHhFVEFQQmdOVkJBb01DRVp' '5WVc1ck5FUkVNUmd3RmdZRFZRUUREQTkzZDNjdVpYaGhiWEJzClpTNWpiMjB3WERBTkJna3Foa2lHOXcwQkFRRUZBQU5MQURCSUFrRUFtL3hta0htRVFydXJFLzByZS9qZUZS' 'TGwKOFpQakJvcDd1TEhobmlhN2xRRy81ekR0WklVQzNSVnBxRFN3QnV3L05Ud2VHeXVQK284QUc5OEh4cXhUQndJRApBUUFCTUEwR0NTcUdTSWIzRFFFQkJRVUFBNEdCQUJTM' 'lRMdUJlVFBtY2FUYVVXL0xDQjJOWU95OEdNZHpSMW14CjhpQkl1Mkg2L0UydGlZM1JJZXZWMk9XNjFxWTIvWFJRZzdZUHh4M2ZmZVV1Z1g5RjRKL2lQbm51MXpBeHh5QnkKMl' 'ZndUt2NFNXalJGb1JrSWZJbEhYMHFWdmlNaFNsTnkyaW9GTHk3SmNQWmIrdjNmdERHeXdVcWNCaVZEb2VhMApIbitHbXhaQQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==",\n', '+ "validFrom": "1901-12-13T20:45:52.000Z",\n', '+ "validTo": "2038-01-19T03:14:07.000Z",\n', '+ "fingerprint": "6b7fb51b56acaf0a8f9b9a3f8ca6737cb97821ddb830106c9fc9a14b8bfdfa36",\n', '+ "certificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNHakNDQVlPZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRVUZBRENCbXpFTE1Ba0dB' 'MVVFQmhNQ1NsQXgKRGpBTUJnTlZCQWdUQlZSdmEzbHZNUkF3RGdZRFZRUUhFd2REYUhWdkxXdDFNUkV3RHdZRFZRUUtFd2hHY21GdQphelJFUkRFWU1CWUdBMVVFQ3hNUFYyV' 'mlRMlZ5ZENCVGRYQndiM0owTVJnd0ZnWURWUVFERXc5R2NtRnVhelJFClJDQlhaV0lnUTBFeEl6QWhCZ2txaGtpRzl3MEJDUUVXRkhOMWNIQnZjblJBWm5KaGJtczBaR1F1WT' 'I5dE1DSVkKRHpFNU1ERXhNakV6TWpBME5UVXlXaGdQTWpBek9EQXhNVGt3TXpFME1EZGFNRW94Q3pBSkJnTlZCQVlUQWtwUQpNUTR3REFZRFZRUUlEQVZVYjJ0NWJ6RVJNQTh' 'HQTFVRUNnd0lSbkpoYm1zMFJFUXhHREFXQmdOVkJBTU1EM2QzCmR5NWxlR0Z0Y0d4bExtTnZiVEJjTUEwR0NTcUdTSWIzRFFFQkFRVUFBMHNBTUVnQ1FRQ2IvR2FRZVlSQ3U2' 'c1QKL1N0NytONFZFdVh4aytNR2ludTRzZUdlSnJ1VkFiL25NTzFraFFMZEZXbW9OTEFHN0Q4MVBCNGJLNC82andBYgozd2ZHckZNSEFnTUJBQUV3RFFZSktvWklodmNOQVFFR' 'kJRQURnWUVBbnpkZVFCRzJjclhudlp5SGdDTDlkU25tCmxuYVhKSVRPLy8rRzU5dUN2REtiblgrQkt2WFh4WFFJYTdHbXR6WXV3M0xDL2pKSkwzMDdyL0NFQ1pyNnZWOUkKS0' 'huMjcreU90clBET3dURHRYeWFZT2FmOFY2ZmtTVk4zaUx4N3RiRVA2UjB1RUt4YVZhcU1aNzFlZDNTTzFPTAp3cTBqOEdrS1kvSy96bDJOd3pjPQotLS0tLUVORCBDRVJUSUZ' 'JQ0FURS0tLS0tCg==",\n', ' "subjectPublicKey": "MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAJv8ZpB5hEK7qxP9K3v43hUS5' 'fGT4waKe7ix4Z4mu5UBv+cw7WSFAt0Vaag0sAbsPzU8Hhsrj/qPABvfB8asUwcCAwEAAQ=="\n' ] }
def test_compare_plan_entity_bytes(): EntityTest3Appgate = load_test_open_api_spec( secrets_key=None, reload=True).entities['EntityTest3Appgate'].cls # fieldOne is writeOnly :: byte # fieldTwo is readOnly :: checksum of fieldOne # fieldThree is readOnly :: size of fieldOne e_data = { 'id': '6a01c585-c192-475b-b86f-0e632ada6769', # Current data always has ids 'name': 'entity1', 'fieldOne': None, 'fieldTwo': SHA256_FILE, 'fieldThree': 1563, } entities_current = EntitiesSet( {EntityWrapper(APPGATE_LOADER.load(e_data, None, EntityTest3Appgate))}) e_data = { 'name': 'entity1', 'fieldOne': BASE64_FILE_W0, 'fieldTwo': None, 'fieldThree': None, } e_metadata = {'uuid': '6a01c585-c192-475b-b86f-0e632ada6769'} entities_expected = EntitiesSet({ EntityWrapper(K8S_LOADER.load(e_data, e_metadata, EntityTest3Appgate)) }) plan = compare_entities(entities_current, entities_expected, BUILTIN_TAGS) assert plan.modify.entities == frozenset() assert plan.modifications_diff == {} assert compute_diff( list(entities_current.entities)[0], list(entities_expected.entities)[0]) == [] # Let's change the bytes e_data = { 'name': 'entity1', 'fieldOne': 'Some other content', 'fieldTwo': None, 'fieldThree': None, } new_e = K8S_LOADER.load(e_data, e_metadata, EntityTest3Appgate) entities_expected = EntitiesSet({EntityWrapper(new_e)}) plan = compare_entities(entities_current, entities_expected, BUILTIN_TAGS) assert plan.modify.entities == frozenset({EntityWrapper(new_e)}) assert plan.modifications_diff == { 'entity1': [ '--- \n', '+++ \n', '@@ -2,4 +2,4 @@\n', ' "name": "entity1",\n', '- "fieldTwo": "0d373afdccb82399b29ba0d6d1a282b4d10d7e70d948257e75c05999f0be9f3e",\n', '- "fieldThree": 1563\n', '+ "fieldTwo": "c8f4fc85b689f8f3a70e7024e2bb8c7c8f4f7f9ffd2a1a8d01fc8fba74d1af34",\n', '+ "fieldThree": 12\n', ' }' ] }
def test_normalize_policies_2(): policies = EntitiesSet({ EntityWrapper( policy( name='policy-1', entitlements=['entitlement1', 'entitlement2', 'entitlement3'])), EntityWrapper( policy(name='policy-2', entitlements=[ 'entitlement1', 'entitlement4', 'entitlement5', ])), }) entitlements = EntitiesSet({ EntityWrapper(entitlement(id='e1', name='entitlement1')), EntityWrapper(entitlement(id='e2', name='entitlement2')), EntityWrapper(entitlement(id='e3', name='entitlement3')), EntityWrapper(entitlement(id='e4', name='entitlement4')), EntityWrapper(entitlement(id='e5', name='entitlement5')), }) policies_set, conflicts = resolve_entities( policies, [(entitlements, 'entitlements')]) assert conflicts is None assert policies_set.entities == { EntityWrapper(policy(name='policy-1', entitlements=['e1', 'e2', 'e3'])), EntityWrapper( policy(name='policy-2', entitlements=[ 'e1', 'e4', 'e5', ])), }
def test_compare_policies_2(): current_policies = EntitiesSet({ EntityWrapper( Policy(id='id1', name='policy1', expression='expression-1')), EntityWrapper( Policy(id='id2', name='policy2', expression='expression-2')), EntityWrapper( Policy(id='id3', name='policy3', expression='expression-3')) }) expected_policies = EntitiesSet({ EntityWrapper( Policy(id='id1', name='policy1', expression='expression-1')), EntityWrapper( Policy(id='id2', name='policy2', expression='expression-2')), EntityWrapper( Policy(id='id3', name='policy3', expression='expression-3')) }) plan = compare_entities(current_policies, expected_policies, BUILTIN_TAGS) assert plan.modify.entities == set() assert plan.delete.entities == set() assert plan.create.entities == set() assert plan.share.entities == { EntityWrapper( Policy(id='id1', name='policy1', expression='expression-1')), EntityWrapper( Policy(id='id2', name='policy2', expression='expression-2')), EntityWrapper( Policy(id='id3', name='policy3', expression='expression-3')) } share_ids = [p.id for p in plan.share.entities] assert len(list(filter(None, share_ids))) == 3 share_ids.sort() assert share_ids == ['id1', 'id2', 'id3']
def test_normalize_entitlements_2(): entitlements = EntitiesSet({ EntityWrapper( entitlement(name='entitlement-1', conditions=['condition1', 'condition2', 'condition3'])), EntityWrapper( entitlement(name='entitlement-2', conditions=[ 'condition1', 'condition4', 'condition5', ])), }) conditions = EntitiesSet({ EntityWrapper(condition(id='c1', name='condition1')), EntityWrapper(condition(id='c2', name='condition2')), EntityWrapper(condition(id='c3', name='condition3')), EntityWrapper(condition(id='c4', name='condition4')), EntityWrapper(condition(id='c5', name='condition5')), }) entitlements_set, conflicts = resolve_entities( entitlements, [(conditions, 'conditions')]) assert entitlements_set.entities == { EntityWrapper( entitlement(name='entitlement-1', conditions=['c1', 'c2', 'c3'])), EntityWrapper( entitlement(name='entitlement-2', conditions=[ 'c1', 'c4', 'c5', ])), } assert conflicts is None
def resolve_entity(entity: Entity_T, field: str, names: Dict[str, EntityWrapper], ids: Dict[str, EntityWrapper], missing_dependencies: Dict[str, Dict[str, FrozenSet[str]]], reverse: bool = False) -> Optional[EntityWrapper]: new_dependencies = set() missing_dependencies_set = set() log.debug(f'Getting field %s in entity %s', field, entity.__class__.__name__) dependencies, rest_fields = get_field(entity, field.split('.')) if dependencies is None: raise Exception(f'Object {entity} has not field {field}.') is_iterable = isinstance(dependencies, frozenset) if not is_iterable: dependencies = frozenset({dependencies}) for dependency in dependencies: if type(dependency) not in PYTHON_TYPES and rest_fields: log.debug('dependency %s and rest_fields %s', dependency, rest_fields) resolve_entity(dependency, rest_fields, names, ids, missing_dependencies, reverse) elif dependency in ids: # dependency is an id if reverse: new_dependencies.add(ids[dependency].name) else: new_dependencies.add(dependency) elif dependency in names and names[dependency].id: # dependency is a name if reverse: new_dependencies.add(dependency) else: new_dependencies.add(names[dependency].id) else: missing_dependencies_set.add(dependency) if missing_dependencies_set: if entity.name not in missing_dependencies: missing_dependencies[entity.name] = {} missing_dependencies[entity.name][field] = frozenset( missing_dependencies_set) if new_dependencies: if is_iterable: return EntityWrapper( evolve_rec(entity, field.split('.'), frozenset(new_dependencies))) else: return EntityWrapper( evolve_rec(entity, field.split('.'), list(new_dependencies)[0])) return None
def test_dependencies_2(): """ Two dependencies EntityDep4 has an array field (deps1) with deps from EntityDep1 EntityDep4 has a string field (dep2) with deps from EntityDep2 """ api = load_test_open_api_spec() EntityDep1 = api.entities['EntityDep1'].cls EntityDep2 = api.entities['EntityDep2'].cls EntityDep4 = api.entities['EntityDep4'].cls deps1 = EntitiesSet({ EntityWrapper(EntityDep1(id='d11', name='dep11')), EntityWrapper(EntityDep1(id='d12', name='dep12')), EntityWrapper(EntityDep1(id='d13', name='dep13')), }) deps2 = EntitiesSet({ EntityWrapper(EntityDep2(id='d21', name='dep21')), EntityWrapper(EntityDep2(id='d22', name='dep22')), EntityWrapper(EntityDep2(id='d23', name='dep23')), }) # no conflicts deps4 = EntitiesSet({ EntityWrapper( EntityDep4(id='d31', name='dep31', deps1=frozenset({'dep11', 'dep12'}), dep2='dep23')) }) deps3_resolved, conflicts = resolve_entities(deps4, [(deps1, 'deps1'), (deps2, 'dep2')]) assert conflicts is None assert deps3_resolved.entities == { EntityWrapper( EntityDep4(id='d31', name='dep31', deps1=frozenset({'d11', 'd12'}), dep2='d23')) } # conflicts in field deps1 deps4 = EntitiesSet({ EntityWrapper( EntityDep4(id='d31', name='dep31', deps1=frozenset({'dep14', 'dep12'}), dep2='dep33')) }) deps3_resolved, conflicts = resolve_entities(deps4, [(deps1, 'deps1'), (deps2, 'dep2')]) assert conflicts == { 'dep31': { 'deps1': frozenset({'dep14'}), 'dep2': frozenset({'dep33'}) } }
def test_compare_policies_3(): current_policies = EntitiesSet({ EntityWrapper( Policy(id='id1', name='policy3', expression='expression-1')), EntityWrapper( Policy(id='id2', name='policy2', expression='expression-2')), EntityWrapper( Policy(id='id3', name='policy4', expression='expression-3')) }) expected_policies = EntitiesSet({ EntityWrapper(Policy(name='policy1', expression='expression-1')), EntityWrapper( Policy(id='id2', name='policy2', expression='expression-2')), EntityWrapper( Policy(id='id3', name='policy4', expression='expression-3')) }) plan = compare_entities(current_policies, expected_policies, BUILTIN_TAGS) assert plan.delete.entities == { EntityWrapper( Policy(id='id1', name='policy3', expression='expression-1')) } # test that the ids are propagated when modifying assert [p.id for p in plan.delete.entities] == ['id1'] assert plan.create.entities == { EntityWrapper(Policy(name='policy1', expression='expression-1')) } assert plan.modify.entities == set()
def test_normalize_entitlements_3(): entitlements = EntitiesSet({ EntityWrapper( entitlement(name='entitlement-1', conditions=['condition1', 'condition2', 'condition3'])), EntityWrapper( entitlement(name='entitlement-2', conditions=[ 'condition1', 'condition4', 'condition5', ])), }) conditions = EntitiesSet({ EntityWrapper(condition(id='id1', name='condition1')), EntityWrapper(condition(id='id2', name='condition2')), EntityWrapper(condition(id='id3', name='condition3')), EntityWrapper(condition(name='condition4')), EntityWrapper(condition(name='condition5')), }) entitlements_set, conflicts = resolve_entities( entitlements, [(conditions, 'conditions')]) assert conflicts == { 'entitlement-2': { 'conditions': frozenset({'condition4', 'condition5'}), } }
def entity_sync_generation(entity_wrapper: EntityWrapper) -> EntityWrapper: """ Syncs current generation to latest. """ entity = entity_wrapper.value appgate_metadata = evolve( entity.appgate_metadata, latest_generation=entity.appgate_metadata.current_generation) return EntityWrapper(evolve(entity, appgate_metadata=appgate_metadata))
def exclude_appgate_entities(entities: List[Entity_T], target_tags: Optional[FrozenSet[str]], exclude_tags: Optional[FrozenSet[str]]) -> Set[EntityWrapper]: """ Filter out entities according to target_tags and exclude_rags Returns the entities that are member of target_tags (all entities if None) but not member of exclude_tags """ return set(filter(lambda e: is_target(e, target_tags) and not has_tag(e, exclude_tags), [EntityWrapper(e) for e in entities]))
def test_normalize_policies_3(): policies = EntitiesSet({ EntityWrapper( policy( name='policy-1', entitlements=['entitlement1', 'entitlement2', 'entitlement3'])), EntityWrapper( policy(name='policy-2', entitlements=[ 'entitlement1', 'entitlement4', 'entitlement5', ])), }) entitlements = EntitiesSet({ EntityWrapper(entitlement(id='id1', name='entitlement1')), EntityWrapper(entitlement(id='id2', name='entitlement2')), EntityWrapper(entitlement(id='id3', name='entitlement3')), EntityWrapper(entitlement(name='entitlement5')) }) policies_set, conflicts = resolve_entities( policies, [(entitlements, 'entitlements')]) assert conflicts == { 'policy-2': { 'entitlements': frozenset({'entitlement4', 'entitlement5'}) } }
def test_compare_entities_updated_changed(): EntityTest2 = load_test_open_api_spec( reload=True, k8s_get_secret=_k8s_get_secret).entities['EntityTest2'].cls data_1 = { 'fieldOne': { 'type': 'k8s/secret', 'name': 'secret-storage-1', 'key': 'field-one' }, 'fieldTwo': 'this is write only', 'fieldThree': 'this is a field', } data_2 = { 'fieldOne': { 'type': 'k8s/secret', 'name': 'secret-storage-1', 'key': 'field-one' }, 'fieldTwo': 'this is write only', 'fieldThree': 'this is a field', 'created': '2020-09-10T12:20:14Z', 'updated': '2020-09-10T12:20:14Z' } appgate_metadata = { 'generation': 1, 'latestGeneration': 1, 'creationTimestamp': '2020-09-10T10:20:14Z', 'modificationTimestamp': '2020-09-16T12:20:14Z', } e1 = EntityWrapper(K8S_LOADER.load(data_1, appgate_metadata, EntityTest2)) e2 = EntityWrapper(APPGATE_LOADER.load(data_2, None, EntityTest2)) diff = compute_diff(e2, e1) assert diff == [ '--- \n', '+++ \n', '@@ -2,3 +2,3 @@\n', ' "fieldThree": "this is a field",\n', '- "updated": "2020-09-10T12:20:14.000Z"\n', '+ "updated": "2020-09-16T12:20:14.000Z"\n', ' }' ]
def test_compare_policies_1(): current_policies = EntitiesSet(set()) expected_policies = EntitiesSet({ EntityWrapper(Policy(name='policy1', expression='expression-1')), EntityWrapper(Policy(name='policy2', expression='expression-2')), EntityWrapper(Policy(name='policy3', expression='expression-3')) }) plan = compare_entities(current_policies, expected_policies, BUILTIN_TAGS) assert plan.create.entities == { EntityWrapper(Policy(name='policy1', expression='expression-1')), EntityWrapper(Policy(name='policy2', expression='expression-2')), EntityWrapper(Policy(name='policy3', expression='expression-3')) } assert plan.modify.entities == set() assert plan.delete.entities == set()
def test_dependencies_1(): """ One dependency EntityDep3 has an array field (deps1) with deps from EntityDep1 """ api = load_test_open_api_spec() EntityDep1 = api.entities['EntityDep1'].cls EntityDep3 = api.entities['EntityDep3'].cls deps1 = EntitiesSet({ EntityWrapper(EntityDep1(id='d11', name='dep11')), EntityWrapper(EntityDep1(id='d12', name='dep12')), EntityWrapper(EntityDep1(id='d13', name='dep13')), }) deps3 = EntitiesSet({ EntityWrapper( EntityDep3(id='d31', name='dep31', deps1=frozenset({'dep11', 'dep12'}))) }) # No conflits deps3_resolved, conflicts = resolve_entities(deps3, [(deps1, 'deps1')]) assert conflicts is None assert deps3_resolved.entities == { EntityWrapper( EntityDep3(id='d31', name='dep31', deps1=frozenset({'d11', 'd12'}))) } # Conflicts deps3 = EntitiesSet({ EntityWrapper( EntityDep3(id='d31', name='dep31', deps1=frozenset({'dep14', 'dep12'}))) }) deps3_resolved, conflicts = resolve_entities(deps3, [(deps1, 'deps1')]) assert conflicts == {'dep31': {'deps1': frozenset({'dep14'})}}
def test_filter_appgate_entities(): entities = [ Policy(id='id0', name='policy0', expression='expression-0', tags=frozenset({'tag1', 'tag2', 'tag3'})), Policy(id='id1', name='policy1', expression='expression-1', tags=frozenset({'tag1', 'tag4', 'tag5'})), Policy(id='id2', name='policy2', expression='expression-2', tags=frozenset({'tag3', 'tag6', 'tag7'})), Policy(id='id3', name='policy3', expression='expression-3', tags=frozenset({'tag4', 'tag9', 'tag10'})) ] r1 = exclude_appgate_entities(entities, target_tags=None, exclude_tags=None) assert frozenset(set(EntityWrapper(a) for a in entities)) == r1 r1 = exclude_appgate_entities(entities, target_tags=frozenset({'tag1'}), exclude_tags=None) assert r1 == frozenset( set( EntityWrapper(a) for a in [ Policy(id='id0', name='policy0', expression='expression-0', tags=frozenset({'tag1', 'tag2', 'tag3'})), Policy(id='id1', name='policy1', expression='expression-1', tags=frozenset({'tag1', 'tag4', 'tag5'})) ])) r1 = exclude_appgate_entities(entities, target_tags=frozenset({'tag1'}), exclude_tags=frozenset({'tag2'})) assert r1 == frozenset( set( EntityWrapper(a) for a in [ Policy(id='id1', name='policy1', expression='expression-1', tags=frozenset({'tag1', 'tag4', 'tag5'})) ])) r1 = exclude_appgate_entities(entities, target_tags=None, exclude_tags=frozenset({'tag2'})) assert r1 == frozenset( set( EntityWrapper(a) for a in [ Policy(id='id1', name='policy1', expression='expression-1', tags=frozenset({'tag1', 'tag4', 'tag5'})), Policy(id='id2', name='policy2', expression='expression-2', tags=frozenset({'tag3', 'tag6', 'tag7'})), Policy(id='id3', name='policy3', expression='expression-3', tags=frozenset({'tag4', 'tag9', 'tag10'})) ])) r1 = exclude_appgate_entities(entities, target_tags=None, exclude_tags=frozenset({'tag7'})) assert r1 == frozenset( set( EntityWrapper(a) for a in [ Policy(id='id0', name='policy0', expression='expression-0', tags=frozenset({'tag1', 'tag2', 'tag3'})), Policy(id='id1', name='policy1', expression='expression-1', tags=frozenset({'tag1', 'tag4', 'tag5'})), Policy(id='id3', name='policy3', expression='expression-3', tags=frozenset({'tag4', 'tag9', 'tag10'})) ])) r1 = exclude_appgate_entities(entities, target_tags=None, exclude_tags=frozenset( {'tag1', 'tag4', 'tag6', 'tag9'})) assert r1 == frozenset() r1 = exclude_appgate_entities(entities, target_tags=frozenset({'tag11', 'tag12'}), exclude_tags=None) assert r1 == frozenset()
def test_dependencies_4(): """ several dependencies (even nested) See EntityDep5 for details """ test_api_spec = load_test_open_api_spec() EntityDep1 = test_api_spec.entities['EntityDep1'].cls EntityDep2 = test_api_spec.entities['EntityDep2'].cls EntityDep3 = test_api_spec.entities['EntityDep3'].cls EntityDep4 = test_api_spec.entities['EntityDep4'].cls EntityDep6 = test_api_spec.entities['EntityDep6'].cls EntityDep6_Obj1 = test_api_spec.entities['EntityDep6_Obj1'].cls EntityDep6_Obj1_Obj2 = test_api_spec.entities['EntityDep6_Obj1_Obj2'].cls EntityDep6_Obj1_Obj2_Deps1 = test_api_spec.entities[ 'EntityDep6_Obj1_Obj2_Deps1'].cls deps1 = EntitiesSet({ EntityWrapper(EntityDep1(id='d11', name='dep11')), EntityWrapper(EntityDep1(id='d12', name='dep12')), EntityWrapper(EntityDep1(id='d13', name='dep13')), }) deps2 = EntitiesSet({ EntityWrapper(EntityDep2(id='d21', name='dep21')), EntityWrapper(EntityDep2(id='d22', name='dep22')), EntityWrapper(EntityDep2(id='d23', name='dep23')), }) deps3 = EntitiesSet({ EntityWrapper( EntityDep3(id='d31', name='dep31', deps1=frozenset({'dep11', 'dep12'}))), EntityWrapper( EntityDep3(id='d32', name='dep32', deps1=frozenset({'dep11', 'dep13'}))), EntityWrapper( EntityDep3(id='d33', name='dep33', deps1=frozenset({'dep12', 'dep13'}))), }) deps4 = EntitiesSet({ EntityWrapper( EntityDep4(id='d41', name='dep41', deps1=frozenset({'dep11', 'dep12'}), dep2='dep21')), EntityWrapper( EntityDep4(id='d42', name='dep42', deps1=frozenset({'dep11', 'dep13'}), dep2='dep22')), EntityWrapper( EntityDep4(id='d43', name='dep43', deps1=frozenset({'dep12', 'dep13'}), dep2='dep23')), }) # no conflicts data = { 'id': 'd61', 'name': 'dep61', 'deps4': ['dep41', 'dep42'], 'obj1': { 'dep3': 'dep31', 'obj2': { 'deps1': [ { 'dep1': 'dep11' }, { 'dep1': 'dep12' }, { 'dep1': 'dep13' }, ], 'deps2': ['dep21', 'dep22'] } } } deps6 = EntitiesSet( {EntityWrapper(K8S_LOADER.load(data, None, EntityDep6))}) appgate_state = AppgateState( entities_set={ 'EntityDep1': deps1, 'EntityDep2': deps2, 'EntityDep3': deps3, 'EntityDep4': deps4, 'EntityDep5': EntitiesSet(), 'EntityDep6': deps6, }) conflicts = resolve_appgate_state(appgate_state, test_api_spec) assert conflicts == {} assert appgate_state.entities_set['EntityDep1'].entities == { EntityWrapper(EntityDep1(id='d11', name='dep11')), EntityWrapper(EntityDep1(id='d12', name='dep12')), EntityWrapper(EntityDep1(id='d13', name='dep13')), } assert appgate_state.entities_set['EntityDep2'].entities == { EntityWrapper(EntityDep2(id='d21', name='dep21')), EntityWrapper(EntityDep2(id='d22', name='dep22')), EntityWrapper(EntityDep2(id='d23', name='dep23')), } assert appgate_state.entities_set['EntityDep3'].entities == { EntityWrapper( EntityDep3(id='d31', name='dep31', deps1=frozenset({'d11', 'd12'}))), EntityWrapper( EntityDep3(id='d32', name='dep32', deps1=frozenset({'d11', 'd13'}))), EntityWrapper( EntityDep3(id='d33', name='dep33', deps1=frozenset({'d12', 'd13'}))), } assert appgate_state.entities_set['EntityDep4'].entities == { EntityWrapper( EntityDep4(id='d41', name='dep41', deps1=frozenset({'d11', 'd12'}), dep2='d21')), EntityWrapper( EntityDep4(id='d42', name='dep42', deps1=frozenset({'d11', 'd13'}), dep2='d22')), EntityWrapper( EntityDep4(id='d43', name='dep43', deps1=frozenset({'d12', 'd13'}), dep2='d23')), } assert appgate_state.entities_set['EntityDep6'].entities == { EntityWrapper( EntityDep6(name='dep61', deps4=frozenset({'d42', 'd41'}), obj1=EntityDep6_Obj1( dep3='d31', obj2=EntityDep6_Obj1_Obj2(deps1=frozenset({ EntityDep6_Obj1_Obj2_Deps1(dep1='dep11'), EntityDep6_Obj1_Obj2_Deps1(dep1='dep12'), EntityDep6_Obj1_Obj2_Deps1(dep1='dep13'), }), deps2=frozenset( {'d21', 'd22'}))))) } # Test empty list in dependencies data = { 'id': 'd61', 'name': 'dep61', 'obj1': { 'dep3': 'dep31', 'obj2': { 'deps1': [], 'deps2': ['dep21', 'dep22'] } } } deps6 = EntitiesSet( {EntityWrapper(K8S_LOADER.load(data, None, EntityDep6))}) appgate_state = AppgateState( entities_set={ 'EntityDep1': deps1, 'EntityDep2': deps2, 'EntityDep3': deps3, 'EntityDep4': deps4, 'EntityDep5': EntitiesSet(), 'EntityDep6': deps6, }) conflicts = resolve_appgate_state(appgate_state, test_api_spec) assert conflicts == {} assert appgate_state.entities_set['EntityDep6'].entities == { EntityWrapper( EntityDep6(name='dep61', deps4=frozenset(), obj1=EntityDep6_Obj1(dep3='d31', obj2=EntityDep6_Obj1_Obj2( deps1=frozenset(), deps2=frozenset({'d21', 'd22'}))))) }
async def main_loop(queue: Queue, ctx: Context, k8s_configmap_client: K8SConfigMapClient) -> None: namespace = ctx.namespace log.info('[appgate-operator/%s] Main loop started:', namespace) log.info('[appgate-operator/%s] + namespace: %s', namespace, namespace) log.info('[appgate-operator/%s] + host: %s', namespace, ctx.controller) log.info('[appgate-operator/%s] + timeout: %s', namespace, ctx.timeout) log.info('[appgate-operator/%s] + dry-run: %s', namespace, ctx.dry_run_mode) log.info('[appgate-operator/%s] + cleanup: %s', namespace, ctx.cleanup_mode) log.info('[appgate-operator/%s] + two-way-sync: %s', namespace, ctx.two_way_sync) log.info('[appgate-operator/%s] Getting current state from controller', namespace) current_appgate_state = await get_current_appgate_state(ctx=ctx) if ctx.cleanup_mode: expected_appgate_state = AppgateState( {k: v.entities_with_tags(ctx.builtin_tags) for k, v in current_appgate_state.entities_set.items()}) else: expected_appgate_state = deepcopy(current_appgate_state) log.info('[appgate-operator/%s] Ready to get new events and compute a new plan', namespace) while True: try: event: AppgateEvent = await asyncio.wait_for(queue.get(), timeout=ctx.timeout) log.info('[appgate-operator/%s}] Event op: %s %s with name %s', namespace, event.op, str(type(event.entity)), event.entity.name) expected_appgate_state.with_entity(EntityWrapper(event.entity), event.op, current_appgate_state) except asyncio.exceptions.TimeoutError: # Log all entities in expected state log.info('[appgate-operator/%s] Expected entities:', namespace) for entity_type, xs in expected_appgate_state.entities_set.items(): for entity_name, e in xs.entities_by_name.items(): log.info('[appgate-operator/%s] %s: %s: %s', namespace, entity_type, entity_name, e.id) # Resolve entities now, in order # this will be the Topological sort total_conflicts = resolve_appgate_state(appgate_state=expected_appgate_state, reverse=False, api_spec=ctx.api_spec) if total_conflicts: log.error('[appgate-operator/%s] Found errors in expected state and plan can' ' not be applied.', namespace) entities_conflict_summary(conflicts=total_conflicts, namespace=namespace) log.info('[appgate-operator/%s] Waiting for more events that can fix the state.', namespace) continue if ctx.two_way_sync: # use current appgate state from controller instead of from memory current_appgate_state = await get_current_appgate_state(ctx=ctx) # Create a plan # Need to copy? # Now we use dicts so resolving update the contents of the keys plan = create_appgate_plan(current_appgate_state, expected_appgate_state, ctx.builtin_tags,) if plan.needs_apply: log.info('[appgate-operator/%s] No more events for a while, creating a plan', namespace) async with AsyncExitStack() as exit_stack: appgate_client = None if not ctx.dry_run_mode: if ctx.device_id is None: raise AppgateException('No device id specified') appgate_client = await exit_stack.enter_async_context(AppgateClient( controller=ctx.controller, user=ctx.user, password=ctx.password, provider=ctx.provider, device_id=ctx.device_id, version=ctx.api_spec.api_version, no_verify=ctx.no_verify, cafile=ctx.cafile)) else: log.warning('[appgate-operator/%s] Running in dry-mode, nothing will be created', namespace) new_plan = await appgate_plan_apply(appgate_plan=plan, namespace=namespace, entity_clients=generate_api_spec_clients( api_spec=ctx.api_spec, appgate_client=appgate_client) if appgate_client else {}, k8s_configmap_client=k8s_configmap_client, api_spec=ctx.api_spec) if len(new_plan.errors) > 0: log.error('[appgate-operator/%s] Found errors when applying plan:', namespace) for err in new_plan.errors: log.error('[appgate-operator/%s] Error %s:', namespace, err) sys.exit(1) if appgate_client: current_appgate_state = new_plan.appgate_state expected_appgate_state = expected_appgate_state.sync_generations() else: log.info('[appgate-operator/%s] Nothing changed! Keeping watching!', namespace)