def test_normalize_policies_1():
policies = EntitiesSet({
EntityWrapper(
policy(
name='policy-1',
entitlements=['entitlement1', 'entitlement2',
'entitlement3'])),
EntityWrapper(
policy(name='policy-2',
entitlements=[
'entitlement1',
'entitlement2',
'entitlement3',
])),
})
entitlements = EntitiesSet()
policies_set, conflicts = resolve_entities(
policies, [(entitlements, 'entitlements')])
assert conflicts == {
'policy-1': {
'entitlements':
frozenset({'entitlement1', 'entitlement2', 'entitlement3'})
},
'policy-2': {
'entitlements':
frozenset({'entitlement1', 'entitlement2', 'entitlement3'})
}
}
def test_normalize_entitlements_1():
entitlements = EntitiesSet({
EntityWrapper(
entitlement(name='entitlement-1',
conditions=['condition1', 'condition2',
'condition3'])),
EntityWrapper(
entitlement(name='entitlement-2',
conditions=[
'condition1',
'condition2',
'condition3',
])),
})
conditions = EntitiesSet()
entitlements_set, conflicts = resolve_entities(
entitlements, [(conditions, 'conditions')])
assert entitlements_set.entities == entitlements.entities
assert conflicts == {
'entitlement-1': {
'conditions': frozenset({'condition1', 'condition2', 'condition3'})
},
'entitlement-2': {
'conditions': frozenset({'condition1', 'condition2', 'condition3'})
},
}
def test_compare_entity_with_secrets_with_metadata_4():
EntityTest2 = load_test_open_api_spec(
reload=True,
k8s_get_secret=_k8s_get_secret).entities['EntityTest2'].cls
data_1 = {
'fieldOne': {
'type': 'k8s/secret',
'name': 'secret-storage-1',
'key': 'field-one'
},
'fieldTwo': 'this is write only',
'fieldThree': 'this is a field',
}
data_2 = {
'fieldOne': {
'type': 'k8s/secret',
'name': 'secret-storage-1',
'key': 'field-one'
},
'fieldTwo': 'this is write only',
'fieldThree': 'this is a field',
'created': '2020-09-10T12:20:14Z',
'updated': '2020-09-20T12:20:14Z'
}
appgate_metadata = {
'generation': 2,
'latestGeneration': 3,
'creationTimestamp': '2020-09-10T10:20:14Z',
'modificationTimestamp': '2020-09-20T12:19:14Z',
}
e1 = EntityWrapper(K8S_LOADER.load(data_1, appgate_metadata, EntityTest2))
e2 = EntityWrapper(APPGATE_LOADER.load(data_2, None, EntityTest2))
assert e1 == e2
def test_dependencies_3():
"""
several dependencies (even nested)
See EntityDep5 for details
"""
api = load_test_open_api_spec()
EntityDep1 = api.entities['EntityDep1'].cls
EntityDep5 = api.entities['EntityDep5'].cls
EntityDep5_Obj1 = api.entities['EntityDep5_Obj1'].cls
EntityDep5_Obj1_Obj2 = api.entities['EntityDep5_Obj1_Obj2'].cls
deps1 = EntitiesSet({
EntityWrapper(EntityDep1(id='d11', name='dep11')),
EntityWrapper(EntityDep1(id='d12', name='dep12')),
EntityWrapper(EntityDep1(id='d13', name='dep13')),
})
data = {'id': 'd51', 'name': 'dep51', 'obj1': {'obj2': {'dep1': 'dep11'}}}
deps5 = EntitiesSet(
{EntityWrapper(K8S_LOADER.load(data, None, EntityDep5))})
deps5_resolved, conflicts = resolve_entities(deps5,
[(deps1, 'obj1.obj2.dep1')])
assert conflicts is None
assert deps5_resolved.entities == {
EntityWrapper(
EntityDep5(
id='d51',
name='dep51',
obj1=EntityDep5_Obj1(obj2=EntityDep5_Obj1_Obj2(dep1='d11'))))
}
def test_compare_entity_with_secrets():
"""
If we are missing metadata frm k8s we should always update the entity
"""
EntityTest2 = load_test_open_api_spec(
reload=True,
k8s_get_secret=_k8s_get_secret).entities['EntityTest2'].cls
data_1 = {
'fieldOne': {
'type': 'k8s/secret',
'name': 'secret-storage-1',
'key': 'field-one'
},
'fieldTwo': 'this is write only',
'fieldThree': 'this is a field',
}
data_2 = {
'fieldOne': {
'type': 'k8s/secret',
'name': 'secret-storage-1',
'key': 'field-one'
},
'fieldTwo': 'this is write only',
'fieldThree': 'this is a field',
'created': '2020-09-10T12:20:14Z',
'updated': '2020-09-10T12:20:14Z'
}
e1 = EntityWrapper(K8S_LOADER.load(data_1, None, EntityTest2))
e2 = EntityWrapper(APPGATE_LOADER.load(data_2, None, EntityTest2))
assert e1 != e2
def test_compare_plan_entity_pem():
EntityCert = load_test_open_api_spec(
secrets_key=None, reload=True).entities['EntityCert'].cls
appgate_data = {
'name': 'c1',
'fieldOne': PEM_TEST,
'fieldTwo': {
'version': 1,
'serial': '3578',
'issuer': join_string(ISSUER),
'subject': join_string(SUBJECT),
'validFrom': '2012-08-22T05:26:54.000Z',
'validTo': '2017-08-21T05:26:54.000Z',
'fingerprint': 'Xw+1FmWBquZKEBwVg7G+vnToFKkeeooUuh6DXXj26ec=',
'certificate': join_string(CERTIFICATE_FIELD),
'subjectPublicKey': join_string(PUBKEY_FIELD),
}
}
k8s_data = {'name': 'c1', 'fieldOne': PEM2}
current_entities = EntitiesSet(
{EntityWrapper(APPGATE_LOADER.load(appgate_data, None, EntityCert))})
new_e = K8S_LOADER.load(k8s_data, None, EntityCert)
expected_entities = EntitiesSet({EntityWrapper(new_e)})
plan = compare_entities(current_entities, expected_entities, BUILTIN_TAGS)
assert plan.modify.entities == frozenset({EntityWrapper(new_e)})
assert plan.modifications_diff == {
'c1': [
'--- \n', '+++ \n', '@@ -3,10 +3,10 @@\n', ' "fieldTwo": {\n',
'- "version": 1,\n', '- "serial": "3578",\n',
'+ "version": 3,\n', '+ "serial": "0",\n',
' "issuer": "[email protected], CN=Frank4DD Web CA, OU=WebCert Support, O=Frank4DD, L=Chuo-ku, ST=Tokyo, C=JP",\n',
' "subject": "CN=www.example.com, O=Frank4DD, ST=Tokyo, C=JP",\n',
'- "validFrom": "2012-08-22T05:26:54.000Z",\n',
'- "validTo": "2017-08-21T05:26:54.000Z",\n',
'- "fingerprint": "Xw+1FmWBquZKEBwVg7G+vnToFKkeeooUuh6DXXj26ec=",\n',
'- "certificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNFakNDQVhzQ0FnMzZNQTBHQ1NxR1NJYjNEUUVCQlFVQU1JR2JNUXN3Q1FZRFZRUUdF'
'd0pLVURFT01Bd0cKQTFVRUNCTUZWRzlyZVc4eEVEQU9CZ05WQkFjVEIwTm9kVzh0YTNVeEVUQVBCZ05WQkFvVENFWnlZVzVyTkVSRQpNUmd3RmdZRFZRUUxFdzlYWldKRFpYS'
'jBJRk4xY0hCdmNuUXhHREFXQmdOVkJBTVREMFp5WVc1ck5FUkVJRmRsCllpQkRRVEVqTUNFR0NTcUdTSWIzRFFFSkFSWVVjM1Z3Y0c5eWRFQm1jbUZ1YXpSa1pDNWpiMjB3SG'
'hjTk1USXcKT0RJeU1EVXlOalUwV2hjTk1UY3dPREl4TURVeU5qVTBXakJLTVFzd0NRWURWUVFHRXdKS1VERU9NQXdHQTFVRQpDQXdGVkc5cmVXOHhFVEFQQmdOVkJBb01DRVp'
'5WVc1ck5FUkVNUmd3RmdZRFZRUUREQTkzZDNjdVpYaGhiWEJzClpTNWpiMjB3WERBTkJna3Foa2lHOXcwQkFRRUZBQU5MQURCSUFrRUFtL3hta0htRVFydXJFLzByZS9qZUZS'
'TGwKOFpQakJvcDd1TEhobmlhN2xRRy81ekR0WklVQzNSVnBxRFN3QnV3L05Ud2VHeXVQK284QUc5OEh4cXhUQndJRApBUUFCTUEwR0NTcUdTSWIzRFFFQkJRVUFBNEdCQUJTM'
'lRMdUJlVFBtY2FUYVVXL0xDQjJOWU95OEdNZHpSMW14CjhpQkl1Mkg2L0UydGlZM1JJZXZWMk9XNjFxWTIvWFJRZzdZUHh4M2ZmZVV1Z1g5RjRKL2lQbm51MXpBeHh5QnkKMl'
'ZndUt2NFNXalJGb1JrSWZJbEhYMHFWdmlNaFNsTnkyaW9GTHk3SmNQWmIrdjNmdERHeXdVcWNCaVZEb2VhMApIbitHbXhaQQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==",\n',
'+ "validFrom": "1901-12-13T20:45:52.000Z",\n',
'+ "validTo": "2038-01-19T03:14:07.000Z",\n',
'+ "fingerprint": "6b7fb51b56acaf0a8f9b9a3f8ca6737cb97821ddb830106c9fc9a14b8bfdfa36",\n',
'+ "certificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNHakNDQVlPZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRVUZBRENCbXpFTE1Ba0dB'
'MVVFQmhNQ1NsQXgKRGpBTUJnTlZCQWdUQlZSdmEzbHZNUkF3RGdZRFZRUUhFd2REYUhWdkxXdDFNUkV3RHdZRFZRUUtFd2hHY21GdQphelJFUkRFWU1CWUdBMVVFQ3hNUFYyV'
'mlRMlZ5ZENCVGRYQndiM0owTVJnd0ZnWURWUVFERXc5R2NtRnVhelJFClJDQlhaV0lnUTBFeEl6QWhCZ2txaGtpRzl3MEJDUUVXRkhOMWNIQnZjblJBWm5KaGJtczBaR1F1WT'
'I5dE1DSVkKRHpFNU1ERXhNakV6TWpBME5UVXlXaGdQTWpBek9EQXhNVGt3TXpFME1EZGFNRW94Q3pBSkJnTlZCQVlUQWtwUQpNUTR3REFZRFZRUUlEQVZVYjJ0NWJ6RVJNQTh'
'HQTFVRUNnd0lSbkpoYm1zMFJFUXhHREFXQmdOVkJBTU1EM2QzCmR5NWxlR0Z0Y0d4bExtTnZiVEJjTUEwR0NTcUdTSWIzRFFFQkFRVUFBMHNBTUVnQ1FRQ2IvR2FRZVlSQ3U2'
'c1QKL1N0NytONFZFdVh4aytNR2ludTRzZUdlSnJ1VkFiL25NTzFraFFMZEZXbW9OTEFHN0Q4MVBCNGJLNC82andBYgozd2ZHckZNSEFnTUJBQUV3RFFZSktvWklodmNOQVFFR'
'kJRQURnWUVBbnpkZVFCRzJjclhudlp5SGdDTDlkU25tCmxuYVhKSVRPLy8rRzU5dUN2REtiblgrQkt2WFh4WFFJYTdHbXR6WXV3M0xDL2pKSkwzMDdyL0NFQ1pyNnZWOUkKS0'
'huMjcreU90clBET3dURHRYeWFZT2FmOFY2ZmtTVk4zaUx4N3RiRVA2UjB1RUt4YVZhcU1aNzFlZDNTTzFPTAp3cTBqOEdrS1kvSy96bDJOd3pjPQotLS0tLUVORCBDRVJUSUZ'
'JQ0FURS0tLS0tCg==",\n',
' "subjectPublicKey": "MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAJv8ZpB5hEK7qxP9K3v43hUS5'
'fGT4waKe7ix4Z4mu5UBv+cw7WSFAt0Vaag0sAbsPzU8Hhsrj/qPABvfB8asUwcCAwEAAQ=="\n'
]
}
def test_compare_plan_entity_bytes():
EntityTest3Appgate = load_test_open_api_spec(
secrets_key=None, reload=True).entities['EntityTest3Appgate'].cls
# fieldOne is writeOnly :: byte
# fieldTwo is readOnly :: checksum of fieldOne
# fieldThree is readOnly :: size of fieldOne
e_data = {
'id':
'6a01c585-c192-475b-b86f-0e632ada6769', # Current data always has ids
'name': 'entity1',
'fieldOne': None,
'fieldTwo': SHA256_FILE,
'fieldThree': 1563,
}
entities_current = EntitiesSet(
{EntityWrapper(APPGATE_LOADER.load(e_data, None, EntityTest3Appgate))})
e_data = {
'name': 'entity1',
'fieldOne': BASE64_FILE_W0,
'fieldTwo': None,
'fieldThree': None,
}
e_metadata = {'uuid': '6a01c585-c192-475b-b86f-0e632ada6769'}
entities_expected = EntitiesSet({
EntityWrapper(K8S_LOADER.load(e_data, e_metadata, EntityTest3Appgate))
})
plan = compare_entities(entities_current, entities_expected, BUILTIN_TAGS)
assert plan.modify.entities == frozenset()
assert plan.modifications_diff == {}
assert compute_diff(
list(entities_current.entities)[0],
list(entities_expected.entities)[0]) == []
# Let's change the bytes
e_data = {
'name': 'entity1',
'fieldOne': 'Some other content',
'fieldTwo': None,
'fieldThree': None,
}
new_e = K8S_LOADER.load(e_data, e_metadata, EntityTest3Appgate)
entities_expected = EntitiesSet({EntityWrapper(new_e)})
plan = compare_entities(entities_current, entities_expected, BUILTIN_TAGS)
assert plan.modify.entities == frozenset({EntityWrapper(new_e)})
assert plan.modifications_diff == {
'entity1': [
'--- \n', '+++ \n', '@@ -2,4 +2,4 @@\n',
' "name": "entity1",\n',
'- "fieldTwo": "0d373afdccb82399b29ba0d6d1a282b4d10d7e70d948257e75c05999f0be9f3e",\n',
'- "fieldThree": 1563\n',
'+ "fieldTwo": "c8f4fc85b689f8f3a70e7024e2bb8c7c8f4f7f9ffd2a1a8d01fc8fba74d1af34",\n',
'+ "fieldThree": 12\n', ' }'
]
}
def test_normalize_entitlements_2():
entitlements = EntitiesSet({
EntityWrapper(
entitlement(name='entitlement-1',
conditions=['condition1', 'condition2',
'condition3'])),
EntityWrapper(
entitlement(name='entitlement-2',
conditions=[
'condition1',
'condition4',
'condition5',
])),
})
conditions = EntitiesSet({
EntityWrapper(condition(id='c1', name='condition1')),
EntityWrapper(condition(id='c2', name='condition2')),
EntityWrapper(condition(id='c3', name='condition3')),
EntityWrapper(condition(id='c4', name='condition4')),
EntityWrapper(condition(id='c5', name='condition5')),
})
entitlements_set, conflicts = resolve_entities(
entitlements, [(conditions, 'conditions')])
assert entitlements_set.entities == {
EntityWrapper(
entitlement(name='entitlement-1', conditions=['c1', 'c2', 'c3'])),
EntityWrapper(
entitlement(name='entitlement-2', conditions=[
'c1',
'c4',
'c5',
])),
}
assert conflicts is None
def test_compare_policies_2():
current_policies = EntitiesSet({
EntityWrapper(
Policy(id='id1', name='policy1', expression='expression-1')),
EntityWrapper(
Policy(id='id2', name='policy2', expression='expression-2')),
EntityWrapper(
Policy(id='id3', name='policy3', expression='expression-3'))
})
expected_policies = EntitiesSet({
EntityWrapper(
Policy(id='id1', name='policy1', expression='expression-1')),
EntityWrapper(
Policy(id='id2', name='policy2', expression='expression-2')),
EntityWrapper(
Policy(id='id3', name='policy3', expression='expression-3'))
})
plan = compare_entities(current_policies, expected_policies, BUILTIN_TAGS)
assert plan.modify.entities == set()
assert plan.delete.entities == set()
assert plan.create.entities == set()
assert plan.share.entities == {
EntityWrapper(
Policy(id='id1', name='policy1', expression='expression-1')),
EntityWrapper(
Policy(id='id2', name='policy2', expression='expression-2')),
EntityWrapper(
Policy(id='id3', name='policy3', expression='expression-3'))
}
share_ids = [p.id for p in plan.share.entities]
assert len(list(filter(None, share_ids))) == 3
share_ids.sort()
assert share_ids == ['id1', 'id2', 'id3']
def test_normalize_policies_2():
policies = EntitiesSet({
EntityWrapper(
policy(
name='policy-1',
entitlements=['entitlement1', 'entitlement2',
'entitlement3'])),
EntityWrapper(
policy(name='policy-2',
entitlements=[
'entitlement1',
'entitlement4',
'entitlement5',
])),
})
entitlements = EntitiesSet({
EntityWrapper(entitlement(id='e1', name='entitlement1')),
EntityWrapper(entitlement(id='e2', name='entitlement2')),
EntityWrapper(entitlement(id='e3', name='entitlement3')),
EntityWrapper(entitlement(id='e4', name='entitlement4')),
EntityWrapper(entitlement(id='e5', name='entitlement5')),
})
policies_set, conflicts = resolve_entities(
policies, [(entitlements, 'entitlements')])
assert conflicts is None
assert policies_set.entities == {
EntityWrapper(policy(name='policy-1', entitlements=['e1', 'e2',
'e3'])),
EntityWrapper(
policy(name='policy-2', entitlements=[
'e1',
'e4',
'e5',
])),
}
def test_dependencies_2():
"""
Two dependencies
EntityDep4 has an array field (deps1) with deps from EntityDep1
EntityDep4 has a string field (dep2) with deps from EntityDep2
"""
api = load_test_open_api_spec()
EntityDep1 = api.entities['EntityDep1'].cls
EntityDep2 = api.entities['EntityDep2'].cls
EntityDep4 = api.entities['EntityDep4'].cls
deps1 = EntitiesSet({
EntityWrapper(EntityDep1(id='d11', name='dep11')),
EntityWrapper(EntityDep1(id='d12', name='dep12')),
EntityWrapper(EntityDep1(id='d13', name='dep13')),
})
deps2 = EntitiesSet({
EntityWrapper(EntityDep2(id='d21', name='dep21')),
EntityWrapper(EntityDep2(id='d22', name='dep22')),
EntityWrapper(EntityDep2(id='d23', name='dep23')),
})
# no conflicts
deps4 = EntitiesSet({
EntityWrapper(
EntityDep4(id='d31',
name='dep31',
deps1=frozenset({'dep11', 'dep12'}),
dep2='dep23'))
})
deps3_resolved, conflicts = resolve_entities(deps4, [(deps1, 'deps1'),
(deps2, 'dep2')])
assert conflicts is None
assert deps3_resolved.entities == {
EntityWrapper(
EntityDep4(id='d31',
name='dep31',
deps1=frozenset({'d11', 'd12'}),
dep2='d23'))
}
# conflicts in field deps1
deps4 = EntitiesSet({
EntityWrapper(
EntityDep4(id='d31',
name='dep31',
deps1=frozenset({'dep14', 'dep12'}),
dep2='dep33'))
})
deps3_resolved, conflicts = resolve_entities(deps4, [(deps1, 'deps1'),
(deps2, 'dep2')])
assert conflicts == {
'dep31': {
'deps1': frozenset({'dep14'}),
'dep2': frozenset({'dep33'})
}
}
def resolve_entity(entity: Entity_T,
field: str,
names: Dict[str, EntityWrapper],
ids: Dict[str, EntityWrapper],
missing_dependencies: Dict[str, Dict[str, FrozenSet[str]]],
reverse: bool = False) -> Optional[EntityWrapper]:
new_dependencies = set()
missing_dependencies_set = set()
log.debug(f'Getting field %s in entity %s', field,
entity.__class__.__name__)
dependencies, rest_fields = get_field(entity, field.split('.'))
if dependencies is None:
raise Exception(f'Object {entity} has not field {field}.')
is_iterable = isinstance(dependencies, frozenset)
if not is_iterable:
dependencies = frozenset({dependencies})
for dependency in dependencies:
if type(dependency) not in PYTHON_TYPES and rest_fields:
log.debug('dependency %s and rest_fields %s', dependency,
rest_fields)
resolve_entity(dependency, rest_fields, names, ids,
missing_dependencies, reverse)
elif dependency in ids:
# dependency is an id
if reverse:
new_dependencies.add(ids[dependency].name)
else:
new_dependencies.add(dependency)
elif dependency in names and names[dependency].id:
# dependency is a name
if reverse:
new_dependencies.add(dependency)
else:
new_dependencies.add(names[dependency].id)
else:
missing_dependencies_set.add(dependency)
if missing_dependencies_set:
if entity.name not in missing_dependencies:
missing_dependencies[entity.name] = {}
missing_dependencies[entity.name][field] = frozenset(
missing_dependencies_set)
if new_dependencies:
if is_iterable:
return EntityWrapper(
evolve_rec(entity, field.split('.'),
frozenset(new_dependencies)))
else:
return EntityWrapper(
evolve_rec(entity, field.split('.'),
list(new_dependencies)[0]))
return None
def test_compare_policies_3():
current_policies = EntitiesSet({
EntityWrapper(
Policy(id='id1', name='policy3', expression='expression-1')),
EntityWrapper(
Policy(id='id2', name='policy2', expression='expression-2')),
EntityWrapper(
Policy(id='id3', name='policy4', expression='expression-3'))
})
expected_policies = EntitiesSet({
EntityWrapper(Policy(name='policy1', expression='expression-1')),
EntityWrapper(
Policy(id='id2', name='policy2', expression='expression-2')),
EntityWrapper(
Policy(id='id3', name='policy4', expression='expression-3'))
})
plan = compare_entities(current_policies, expected_policies, BUILTIN_TAGS)
assert plan.delete.entities == {
EntityWrapper(
Policy(id='id1', name='policy3', expression='expression-1'))
}
# test that the ids are propagated when modifying
assert [p.id for p in plan.delete.entities] == ['id1']
assert plan.create.entities == {
EntityWrapper(Policy(name='policy1', expression='expression-1'))
}
assert plan.modify.entities == set()
def test_normalize_entitlements_3():
entitlements = EntitiesSet({
EntityWrapper(
entitlement(name='entitlement-1',
conditions=['condition1', 'condition2',
'condition3'])),
EntityWrapper(
entitlement(name='entitlement-2',
conditions=[
'condition1',
'condition4',
'condition5',
])),
})
conditions = EntitiesSet({
EntityWrapper(condition(id='id1', name='condition1')),
EntityWrapper(condition(id='id2', name='condition2')),
EntityWrapper(condition(id='id3', name='condition3')),
EntityWrapper(condition(name='condition4')),
EntityWrapper(condition(name='condition5')),
})
entitlements_set, conflicts = resolve_entities(
entitlements, [(conditions, 'conditions')])
assert conflicts == {
'entitlement-2': {
'conditions': frozenset({'condition4', 'condition5'}),
}
}
def modify(self, entity: EntityWrapper) -> None:
if entity.name not in self.entities_by_name:
# Not yet in the system, register it with its own id
return self.add(entity)
# All the entities expect the one being modified
self.entities = {e for e in self.entities if e.name != entity.name}
# Replace always the id with the one registered in the system
self.entities.add(
entity.with_id(id=self.entities_by_name[entity.name].id))
def entity_sync_generation(entity_wrapper: EntityWrapper) -> EntityWrapper:
"""
Syncs current generation to latest.
"""
entity = entity_wrapper.value
appgate_metadata = evolve(
entity.appgate_metadata,
latest_generation=entity.appgate_metadata.current_generation)
return EntityWrapper(evolve(entity, appgate_metadata=appgate_metadata))
def exclude_appgate_entities(entities: List[Entity_T], target_tags: Optional[FrozenSet[str]],
exclude_tags: Optional[FrozenSet[str]]) -> Set[EntityWrapper]:
"""
Filter out entities according to target_tags and exclude_rags
Returns the entities that are member of target_tags (all entities if None)
but not member of exclude_tags
"""
return set(filter(lambda e: is_target(e, target_tags) and not has_tag(e, exclude_tags),
[EntityWrapper(e) for e in entities]))
def test_compare_entities_updated_changed():
EntityTest2 = load_test_open_api_spec(
reload=True,
k8s_get_secret=_k8s_get_secret).entities['EntityTest2'].cls
data_1 = {
'fieldOne': {
'type': 'k8s/secret',
'name': 'secret-storage-1',
'key': 'field-one'
},
'fieldTwo': 'this is write only',
'fieldThree': 'this is a field',
}
data_2 = {
'fieldOne': {
'type': 'k8s/secret',
'name': 'secret-storage-1',
'key': 'field-one'
},
'fieldTwo': 'this is write only',
'fieldThree': 'this is a field',
'created': '2020-09-10T12:20:14Z',
'updated': '2020-09-10T12:20:14Z'
}
appgate_metadata = {
'generation': 1,
'latestGeneration': 1,
'creationTimestamp': '2020-09-10T10:20:14Z',
'modificationTimestamp': '2020-09-16T12:20:14Z',
}
e1 = EntityWrapper(K8S_LOADER.load(data_1, appgate_metadata, EntityTest2))
e2 = EntityWrapper(APPGATE_LOADER.load(data_2, None, EntityTest2))
diff = compute_diff(e2, e1)
assert diff == [
'--- \n', '+++ \n', '@@ -2,3 +2,3 @@\n',
' "fieldThree": "this is a field",\n',
'- "updated": "2020-09-10T12:20:14.000Z"\n',
'+ "updated": "2020-09-16T12:20:14.000Z"\n', ' }'
]
def test_normalize_policies_3():
policies = EntitiesSet({
EntityWrapper(
policy(
name='policy-1',
entitlements=['entitlement1', 'entitlement2',
'entitlement3'])),
EntityWrapper(
policy(name='policy-2',
entitlements=[
'entitlement1',
'entitlement4',
'entitlement5',
])),
})
entitlements = EntitiesSet({
EntityWrapper(entitlement(id='id1', name='entitlement1')),
EntityWrapper(entitlement(id='id2', name='entitlement2')),
EntityWrapper(entitlement(id='id3', name='entitlement3')),
EntityWrapper(entitlement(name='entitlement5'))
})
policies_set, conflicts = resolve_entities(
policies, [(entitlements, 'entitlements')])
assert conflicts == {
'policy-2': {
'entitlements': frozenset({'entitlement4', 'entitlement5'})
}
}
def entities_op(entity_set: EntitiesSet, entity: EntityWrapper,
op: Literal['ADDED', 'DELETED', 'MODIFIED'],
current_entities: EntitiesSet) -> None:
# Current state should always contain the real id!!
cached_entity = current_entities.entities_by_name.get(entity.name)
if cached_entity:
entity = entity.with_id(id=cached_entity.id)
if op == 'ADDED':
entity_set.add(entity)
elif op == 'DELETED':
entity_set.delete(entity)
elif op == 'MODIFIED':
entity_set.modify(entity)
def dump_entity(entity: EntityWrapper, entity_type: str) -> Dict[str, Any]:
r"""
name should match this regexp:
'[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*'
"""
entity_name = k8s_name(
entity.name) if has_name(entity) else k8s_name(entity_type)
entity_mt = getattr(entity.value, ENTITY_METADATA_ATTRIB_NAME, {})
singleton = entity_mt.get('singleton', False)
if not singleton:
entity.value = evolve(entity.value,
appgate_metadata=evolve(
entity.value.appgate_metadata,
uuid=entity.id))
return {
'apiVersion': f'{K8S_APPGATE_DOMAIN}/{K8S_APPGATE_VERSION}',
'kind': entity_type,
'metadata': {
'name':
entity_name if entity.is_singleton() else k8s_name(entity.name)
},
'spec': K8S_DUMPER.dump(entity.value)
}
def compute_diff(e1: EntityWrapper, e2: EntityWrapper) -> List[str]:
"""
Computes a list with differences between e1 and e2.
e1 is current entity
e2 is expected entity
"""
e1_dump = DIFF_DUMPER.dump(e1.value)
e2_dump = DIFF_DUMPER.dump(e2.value)
if e2.has_secrets() and e2.changed_generation():
e1_dump['generation'] = e2.value.appgate_metadata.latest_generation
e2_dump['generation'] = e2.value.appgate_metadata.current_generation
elif e2.has_secrets() and e2.updated(e1):
updated_field = getattr(e1.value, 'updated', None)
if updated_field:
e1_dump['updated'] = dump_datetime(updated_field)
e2_dump['updated'] = dump_datetime(
e2.value.appgate_metadata.modified)
diff = list(
difflib.unified_diff(json.dumps(e1_dump,
indent=4).splitlines(keepends=True),
json.dumps(e2_dump,
indent=4).splitlines(keepends=True),
n=1))
return diff
def test_compare_policies_1():
current_policies = EntitiesSet(set())
expected_policies = EntitiesSet({
EntityWrapper(Policy(name='policy1', expression='expression-1')),
EntityWrapper(Policy(name='policy2', expression='expression-2')),
EntityWrapper(Policy(name='policy3', expression='expression-3'))
})
plan = compare_entities(current_policies, expected_policies, BUILTIN_TAGS)
assert plan.create.entities == {
EntityWrapper(Policy(name='policy1', expression='expression-1')),
EntityWrapper(Policy(name='policy2', expression='expression-2')),
EntityWrapper(Policy(name='policy3', expression='expression-3'))
}
assert plan.modify.entities == set()
assert plan.delete.entities == set()
def test_dependencies_1():
"""
One dependency
EntityDep3 has an array field (deps1) with deps from EntityDep1
"""
api = load_test_open_api_spec()
EntityDep1 = api.entities['EntityDep1'].cls
EntityDep3 = api.entities['EntityDep3'].cls
deps1 = EntitiesSet({
EntityWrapper(EntityDep1(id='d11', name='dep11')),
EntityWrapper(EntityDep1(id='d12', name='dep12')),
EntityWrapper(EntityDep1(id='d13', name='dep13')),
})
deps3 = EntitiesSet({
EntityWrapper(
EntityDep3(id='d31',
name='dep31',
deps1=frozenset({'dep11', 'dep12'})))
})
# No conflits
deps3_resolved, conflicts = resolve_entities(deps3, [(deps1, 'deps1')])
assert conflicts is None
assert deps3_resolved.entities == {
EntityWrapper(
EntityDep3(id='d31', name='dep31', deps1=frozenset({'d11',
'd12'})))
}
# Conflicts
deps3 = EntitiesSet({
EntityWrapper(
EntityDep3(id='d31',
name='dep31',
deps1=frozenset({'dep14', 'dep12'})))
})
deps3_resolved, conflicts = resolve_entities(deps3, [(deps1, 'deps1')])
assert conflicts == {'dep31': {'deps1': frozenset({'dep14'})}}
def test_dependencies_4():
"""
several dependencies (even nested)
See EntityDep5 for details
"""
test_api_spec = load_test_open_api_spec()
EntityDep1 = test_api_spec.entities['EntityDep1'].cls
EntityDep2 = test_api_spec.entities['EntityDep2'].cls
EntityDep3 = test_api_spec.entities['EntityDep3'].cls
EntityDep4 = test_api_spec.entities['EntityDep4'].cls
EntityDep6 = test_api_spec.entities['EntityDep6'].cls
EntityDep6_Obj1 = test_api_spec.entities['EntityDep6_Obj1'].cls
EntityDep6_Obj1_Obj2 = test_api_spec.entities['EntityDep6_Obj1_Obj2'].cls
EntityDep6_Obj1_Obj2_Deps1 = test_api_spec.entities[
'EntityDep6_Obj1_Obj2_Deps1'].cls
deps1 = EntitiesSet({
EntityWrapper(EntityDep1(id='d11', name='dep11')),
EntityWrapper(EntityDep1(id='d12', name='dep12')),
EntityWrapper(EntityDep1(id='d13', name='dep13')),
})
deps2 = EntitiesSet({
EntityWrapper(EntityDep2(id='d21', name='dep21')),
EntityWrapper(EntityDep2(id='d22', name='dep22')),
EntityWrapper(EntityDep2(id='d23', name='dep23')),
})
deps3 = EntitiesSet({
EntityWrapper(
EntityDep3(id='d31',
name='dep31',
deps1=frozenset({'dep11', 'dep12'}))),
EntityWrapper(
EntityDep3(id='d32',
name='dep32',
deps1=frozenset({'dep11', 'dep13'}))),
EntityWrapper(
EntityDep3(id='d33',
name='dep33',
deps1=frozenset({'dep12', 'dep13'}))),
})
deps4 = EntitiesSet({
EntityWrapper(
EntityDep4(id='d41',
name='dep41',
deps1=frozenset({'dep11', 'dep12'}),
dep2='dep21')),
EntityWrapper(
EntityDep4(id='d42',
name='dep42',
deps1=frozenset({'dep11', 'dep13'}),
dep2='dep22')),
EntityWrapper(
EntityDep4(id='d43',
name='dep43',
deps1=frozenset({'dep12', 'dep13'}),
dep2='dep23')),
})
# no conflicts
data = {
'id': 'd61',
'name': 'dep61',
'deps4': ['dep41', 'dep42'],
'obj1': {
'dep3': 'dep31',
'obj2': {
'deps1': [
{
'dep1': 'dep11'
},
{
'dep1': 'dep12'
},
{
'dep1': 'dep13'
},
],
'deps2': ['dep21', 'dep22']
}
}
}
deps6 = EntitiesSet(
{EntityWrapper(K8S_LOADER.load(data, None, EntityDep6))})
appgate_state = AppgateState(
entities_set={
'EntityDep1': deps1,
'EntityDep2': deps2,
'EntityDep3': deps3,
'EntityDep4': deps4,
'EntityDep5': EntitiesSet(),
'EntityDep6': deps6,
})
conflicts = resolve_appgate_state(appgate_state, test_api_spec)
assert conflicts == {}
assert appgate_state.entities_set['EntityDep1'].entities == {
EntityWrapper(EntityDep1(id='d11', name='dep11')),
EntityWrapper(EntityDep1(id='d12', name='dep12')),
EntityWrapper(EntityDep1(id='d13', name='dep13')),
}
assert appgate_state.entities_set['EntityDep2'].entities == {
EntityWrapper(EntityDep2(id='d21', name='dep21')),
EntityWrapper(EntityDep2(id='d22', name='dep22')),
EntityWrapper(EntityDep2(id='d23', name='dep23')),
}
assert appgate_state.entities_set['EntityDep3'].entities == {
EntityWrapper(
EntityDep3(id='d31', name='dep31', deps1=frozenset({'d11',
'd12'}))),
EntityWrapper(
EntityDep3(id='d32', name='dep32', deps1=frozenset({'d11',
'd13'}))),
EntityWrapper(
EntityDep3(id='d33', name='dep33', deps1=frozenset({'d12',
'd13'}))),
}
assert appgate_state.entities_set['EntityDep4'].entities == {
EntityWrapper(
EntityDep4(id='d41',
name='dep41',
deps1=frozenset({'d11', 'd12'}),
dep2='d21')),
EntityWrapper(
EntityDep4(id='d42',
name='dep42',
deps1=frozenset({'d11', 'd13'}),
dep2='d22')),
EntityWrapper(
EntityDep4(id='d43',
name='dep43',
deps1=frozenset({'d12', 'd13'}),
dep2='d23')),
}
assert appgate_state.entities_set['EntityDep6'].entities == {
EntityWrapper(
EntityDep6(name='dep61',
deps4=frozenset({'d42', 'd41'}),
obj1=EntityDep6_Obj1(
dep3='d31',
obj2=EntityDep6_Obj1_Obj2(deps1=frozenset({
EntityDep6_Obj1_Obj2_Deps1(dep1='dep11'),
EntityDep6_Obj1_Obj2_Deps1(dep1='dep12'),
EntityDep6_Obj1_Obj2_Deps1(dep1='dep13'),
}),
deps2=frozenset(
{'d21', 'd22'})))))
}
# Test empty list in dependencies
data = {
'id': 'd61',
'name': 'dep61',
'obj1': {
'dep3': 'dep31',
'obj2': {
'deps1': [],
'deps2': ['dep21', 'dep22']
}
}
}
deps6 = EntitiesSet(
{EntityWrapper(K8S_LOADER.load(data, None, EntityDep6))})
appgate_state = AppgateState(
entities_set={
'EntityDep1': deps1,
'EntityDep2': deps2,
'EntityDep3': deps3,
'EntityDep4': deps4,
'EntityDep5': EntitiesSet(),
'EntityDep6': deps6,
})
conflicts = resolve_appgate_state(appgate_state, test_api_spec)
assert conflicts == {}
assert appgate_state.entities_set['EntityDep6'].entities == {
EntityWrapper(
EntityDep6(name='dep61',
deps4=frozenset(),
obj1=EntityDep6_Obj1(dep3='d31',
obj2=EntityDep6_Obj1_Obj2(
deps1=frozenset(),
deps2=frozenset({'d21',
'd22'})))))
}
def test_filter_appgate_entities():
entities = [
Policy(id='id0',
name='policy0',
expression='expression-0',
tags=frozenset({'tag1', 'tag2', 'tag3'})),
Policy(id='id1',
name='policy1',
expression='expression-1',
tags=frozenset({'tag1', 'tag4', 'tag5'})),
Policy(id='id2',
name='policy2',
expression='expression-2',
tags=frozenset({'tag3', 'tag6', 'tag7'})),
Policy(id='id3',
name='policy3',
expression='expression-3',
tags=frozenset({'tag4', 'tag9', 'tag10'}))
]
r1 = exclude_appgate_entities(entities,
target_tags=None,
exclude_tags=None)
assert frozenset(set(EntityWrapper(a) for a in entities)) == r1
r1 = exclude_appgate_entities(entities,
target_tags=frozenset({'tag1'}),
exclude_tags=None)
assert r1 == frozenset(
set(
EntityWrapper(a) for a in [
Policy(id='id0',
name='policy0',
expression='expression-0',
tags=frozenset({'tag1', 'tag2', 'tag3'})),
Policy(id='id1',
name='policy1',
expression='expression-1',
tags=frozenset({'tag1', 'tag4', 'tag5'}))
]))
r1 = exclude_appgate_entities(entities,
target_tags=frozenset({'tag1'}),
exclude_tags=frozenset({'tag2'}))
assert r1 == frozenset(
set(
EntityWrapper(a) for a in [
Policy(id='id1',
name='policy1',
expression='expression-1',
tags=frozenset({'tag1', 'tag4', 'tag5'}))
]))
r1 = exclude_appgate_entities(entities,
target_tags=None,
exclude_tags=frozenset({'tag2'}))
assert r1 == frozenset(
set(
EntityWrapper(a) for a in [
Policy(id='id1',
name='policy1',
expression='expression-1',
tags=frozenset({'tag1', 'tag4', 'tag5'})),
Policy(id='id2',
name='policy2',
expression='expression-2',
tags=frozenset({'tag3', 'tag6', 'tag7'})),
Policy(id='id3',
name='policy3',
expression='expression-3',
tags=frozenset({'tag4', 'tag9', 'tag10'}))
]))
r1 = exclude_appgate_entities(entities,
target_tags=None,
exclude_tags=frozenset({'tag7'}))
assert r1 == frozenset(
set(
EntityWrapper(a) for a in [
Policy(id='id0',
name='policy0',
expression='expression-0',
tags=frozenset({'tag1', 'tag2', 'tag3'})),
Policy(id='id1',
name='policy1',
expression='expression-1',
tags=frozenset({'tag1', 'tag4', 'tag5'})),
Policy(id='id3',
name='policy3',
expression='expression-3',
tags=frozenset({'tag4', 'tag9', 'tag10'}))
]))
r1 = exclude_appgate_entities(entities,
target_tags=None,
exclude_tags=frozenset(
{'tag1', 'tag4', 'tag6', 'tag9'}))
assert r1 == frozenset()
r1 = exclude_appgate_entities(entities,
target_tags=frozenset({'tag11', 'tag12'}),
exclude_tags=None)
assert r1 == frozenset()
async def main_loop(queue: Queue, ctx: Context, k8s_configmap_client: K8SConfigMapClient) -> None:
namespace = ctx.namespace
log.info('[appgate-operator/%s] Main loop started:', namespace)
log.info('[appgate-operator/%s] + namespace: %s', namespace, namespace)
log.info('[appgate-operator/%s] + host: %s', namespace, ctx.controller)
log.info('[appgate-operator/%s] + timeout: %s', namespace, ctx.timeout)
log.info('[appgate-operator/%s] + dry-run: %s', namespace, ctx.dry_run_mode)
log.info('[appgate-operator/%s] + cleanup: %s', namespace, ctx.cleanup_mode)
log.info('[appgate-operator/%s] + two-way-sync: %s', namespace, ctx.two_way_sync)
log.info('[appgate-operator/%s] Getting current state from controller',
namespace)
current_appgate_state = await get_current_appgate_state(ctx=ctx)
if ctx.cleanup_mode:
expected_appgate_state = AppgateState(
{k: v.entities_with_tags(ctx.builtin_tags) for k, v in current_appgate_state.entities_set.items()})
else:
expected_appgate_state = deepcopy(current_appgate_state)
log.info('[appgate-operator/%s] Ready to get new events and compute a new plan',
namespace)
while True:
try:
event: AppgateEvent = await asyncio.wait_for(queue.get(), timeout=ctx.timeout)
log.info('[appgate-operator/%s}] Event op: %s %s with name %s', namespace,
event.op, str(type(event.entity)), event.entity.name)
expected_appgate_state.with_entity(EntityWrapper(event.entity), event.op, current_appgate_state)
except asyncio.exceptions.TimeoutError:
# Log all entities in expected state
log.info('[appgate-operator/%s] Expected entities:', namespace)
for entity_type, xs in expected_appgate_state.entities_set.items():
for entity_name, e in xs.entities_by_name.items():
log.info('[appgate-operator/%s] %s: %s: %s', namespace, entity_type, entity_name,
e.id)
# Resolve entities now, in order
# this will be the Topological sort
total_conflicts = resolve_appgate_state(appgate_state=expected_appgate_state,
reverse=False,
api_spec=ctx.api_spec)
if total_conflicts:
log.error('[appgate-operator/%s] Found errors in expected state and plan can'
' not be applied.', namespace)
entities_conflict_summary(conflicts=total_conflicts, namespace=namespace)
log.info('[appgate-operator/%s] Waiting for more events that can fix the state.',
namespace)
continue
if ctx.two_way_sync:
# use current appgate state from controller instead of from memory
current_appgate_state = await get_current_appgate_state(ctx=ctx)
# Create a plan
# Need to copy?
# Now we use dicts so resolving update the contents of the keys
plan = create_appgate_plan(current_appgate_state, expected_appgate_state,
ctx.builtin_tags,)
if plan.needs_apply:
log.info('[appgate-operator/%s] No more events for a while, creating a plan',
namespace)
async with AsyncExitStack() as exit_stack:
appgate_client = None
if not ctx.dry_run_mode:
if ctx.device_id is None:
raise AppgateException('No device id specified')
appgate_client = await exit_stack.enter_async_context(AppgateClient(
controller=ctx.controller,
user=ctx.user, password=ctx.password, provider=ctx.provider,
device_id=ctx.device_id,
version=ctx.api_spec.api_version, no_verify=ctx.no_verify,
cafile=ctx.cafile))
else:
log.warning('[appgate-operator/%s] Running in dry-mode, nothing will be created',
namespace)
new_plan = await appgate_plan_apply(appgate_plan=plan, namespace=namespace,
entity_clients=generate_api_spec_clients(
api_spec=ctx.api_spec,
appgate_client=appgate_client)
if appgate_client else {},
k8s_configmap_client=k8s_configmap_client,
api_spec=ctx.api_spec)
if len(new_plan.errors) > 0:
log.error('[appgate-operator/%s] Found errors when applying plan:', namespace)
for err in new_plan.errors:
log.error('[appgate-operator/%s] Error %s:', namespace, err)
sys.exit(1)
if appgate_client:
current_appgate_state = new_plan.appgate_state
expected_appgate_state = expected_appgate_state.sync_generations()
else:
log.info('[appgate-operator/%s] Nothing changed! Keeping watching!', namespace)