def resolve_appgate_state( appgate_state: AppgateState, api_spec: APISpec, reverse: bool = False) -> Dict[str, Dict[str, FrozenSet[str]]]: entities = api_spec.entities entities_sorted = api_spec.entities_sorted total_conflicts = {} log.info('[appgate-state] Validating expected state entities') log.debug('[appgate-state] Resolving dependencies in order: %s', entities_sorted) for entity_name in entities_sorted: for entity_dependency in entities[entity_name].dependencies: log.debug('[appgate-state] Checking dependencies %s for %s.%s', entity_dependency.dependencies, entity_name, entity_dependency.field_path) deps_tuple = [] e1 = appgate_state.entities_set[entity_name] for d in entity_dependency.dependencies: deps_tuple.append(( appgate_state.entities_set[d], entity_dependency.field_path, )) new_e1, conflicts = resolve_entities(e1, deps_tuple, reverse) if conflicts: total_conflicts.update(conflicts) appgate_state.entities_set[entity_name] = new_e1 return total_conflicts
def read_entity_generation(self, key: str) -> Optional[LatestEntityGeneration]: entry = self.get_entity_generation(key) log.info( '[k8s-configmap-client/%s/%s] Reading entity generation %s: %s', self.name, self.namespace, key, dump_latest_entity_generation(entry) if entry else 'not found') return entry
async def init(self) -> None: log.info('[k8s-configmap-client/%s/%s] Initializing config-map %s', self.name, self.namespace, self.name) configmap = await asyncio.to_thread( # type: ignore self._v1.read_namespaced_config_map, name=self.name, namespace=self.namespace) self._configmap_mt = configmap.metadata self._data = configmap.data or {}
async def put(self, entity: Entity_T) -> Optional[Entity_T]: log.info('[appgate-client/%s] PUT %s [%s]', entity.__class__.__name__, entity.name, entity.id) path = f'{self.path}/{entity.id}' if self.singleton: path = self.path data = await self._client.put(path, body=self.dump(entity)) if not data: return None return self.load(data) # type: ignore
async def post(self, entity: Entity_T) -> Optional[Entity_T]: log.info('[appgate-client/%s] POST %s [%s]', entity.__class__.__name__, entity.name, entity.id) body = self.dump(entity) body['id'] = entity.id data = await self._client.post(self.path, body=body) if not data: log.error( '[aggpate-client] POST %s :: Expecting a response but we got empty data', self.path) return None return self.load(data) # type: ignore
async def appgate_plan_apply(appgate_plan: AppgatePlan, namespace: str, entity_clients: Dict[str, EntityClient], k8s_configmap_client: K8SConfigMapClient, api_spec: APISpec) -> AppgatePlan: log.info('[appgate-operator/%s] AppgatePlan Summary:', namespace) entities_plan = { k: await plan_apply(v, namespace=namespace, entity_client=entity_clients.get(k), k8s_configmap_client=k8s_configmap_client) for k, v in appgate_plan.ordered_entities_plan(api_spec) } return AppgatePlan(entities_plan=entities_plan)
async def delete_entity_generation( self, key: str) -> Optional[LatestEntityGeneration]: if not self._configmap_mt: await self.init() entry_key = self._entry_key(key) if entry_key not in self._data: return None entry = self.get_entity_generation(key) del self._data[entry_key] log.info('[k8s-configmap-client/%s/%s] Deleting entity generation %s', self.name, self.namespace, key) self._configmap_mt = await self._delete_key(entry_key) return entry
async def ensure_device_id(self) -> str: """ Try to get the device id from the config map. If that fails, generate one and store it in the configmap. """ try: return self._data[self._device_id_key()] except KeyError: device_id = str(uuid.uuid4()) self._data[self._device_id_key()] = device_id log.info('[k8s-configmap-client/%s/%s] Saving device id: %s', self.name, self.namespace, device_id) self._configmap_mt = await self._update_key('device-id', device_id) return device_id
def parse_files(spec_entities: Dict[str, str], spec_directory: Optional[Path] = None, spec_file: str = 'api_specs.yml', k8s_get_secret: Optional[Callable[[str, str], str]] = None, secrets_key: Optional[str] = None) -> APISpec: parser_context = ParserContext(spec_entities=spec_entities, spec_api_path=spec_directory \ or Path(SPEC_DIR), secrets_key=secrets_key, k8s_get_secret=k8s_get_secret) parser = Parser(parser_context, spec_file) # First parse those paths we are interested in for path, v in parser.data['paths'].items(): if not parser_context.get_entity_name(path): continue entity_name = spec_entities[path] log.info('Generating entity %s for path %s', entity_name, path) keys = ['requestBody', 'content', 'application/json', 'schema'] # Check if path returns a singleton or a list of entities get_schema = parser.get_keys(keys=[ 'paths', path, 'get', 'responses', '200', 'content', 'application/json', 'schema' ]) if isinstance(get_schema, dict) and is_compound(get_schema): # TODO: when data.items is a compound method the references are not resolved. parsed_schema = parser.parse_all_of(get_schema['allOf']) elif isinstance(get_schema, dict): parsed_schema = get_schema else: parsed_schema = {} singleton = not all( map(lambda f: f in parsed_schema.get('properties', {}), LIST_PROPERTIES)) parser.parse_definition(entity_name=entity_name, keys=[['paths', path] + ['post'] + keys, ['paths', path] + ['put'] + keys], singleton=singleton) # Now parse the API version api_version_str = parser.get_keys(['info', 'version']) if not api_version_str: raise OpenApiParserException('Unable to find Appgate API version') try: api_version = api_version_str.split(' ')[2] except IndexError: raise OpenApiParserException('Unable to find Appgate API version') return APISpec(entities=parser_context.entities, api_version=api_version)
async def update_entity_generation( self, key: str, generation: Optional[int]) -> Optional[LatestEntityGeneration]: if not self._configmap_mt: await self.init() prev_entry = self.get_entity_generation( key) or LatestEntityGeneration() entry = LatestEntityGeneration( generation=generation or (prev_entry.generation + 1), modified=datetime.datetime.now().astimezone()) gen = dump_latest_entity_generation(entry) entry_key = self._entry_key(key) self._data[entry_key] = gen log.info( '[k8s-configmap-client/%s/%s] Updating entity generation %s -> %s', self.name, self.namespace, key, gen) self._configmap_mt = await self._update_key(entry_key, gen) return entry
async def plan_apply(plan: Plan, namespace: str, k8s_configmap_client: K8SConfigMapClient, entity_client: Optional[EntityClient] = None) -> Plan: errors = set() for e in plan.create.entities: log.info('[appgate-operator/%s] + %s %s [%s]', namespace, type(e.value), e.name, e.id) if entity_client: try: await entity_client.post(e.value) name = 'singleton' if e.value._entity_metadata.get( 'singleton', False) else e.name await k8s_configmap_client.update_entity_generation( key=entity_unique_id(e.value.__class__.__name__, name), generation=e.value.appgate_metadata.current_generation) except Exception as err: errors.add(f'{e.name} [{e.id}]: {str(err)}') for e in plan.modify.entities: log.info('[appgate-operator/%s] * %s %s [%s]', namespace, type(e.value), e.name, e.id) diff = plan.modifications_diff.get(e.name) if diff: log.info('[appgate-operator/%s] DIFF for %s:', namespace, e.name) for d in diff: log.info('%s', d.rstrip()) if entity_client: try: await entity_client.put(e.value) name = 'singleton' if e.value._entity_metadata.get( 'singleton', False) else e.name await k8s_configmap_client.update_entity_generation( key=entity_unique_id(e.value.__class__.__name__, name), generation=e.value.appgate_metadata.current_generation) except Exception as err: errors.add(f'{e.name} [{e.id}]: {str(err)}') for e in plan.delete.entities: log.info('[appgate-operator/%s] - %s %s [%s]', namespace, type(e.value), e.name, e.id) if entity_client: try: await entity_client.delete(e.id) name = 'singleton' if e.value._entity_metadata.get( 'singleton', False) else e.name await k8s_configmap_client.delete_entity_generation( entity_unique_id(e.value.__class__.__name__, name)) except Exception as err: errors.add(f'{e.name} [{e.id}]: {str(err)}') for e in plan.share.entities: log.debug('[appgate-operator/%s] = %s %s [%s]', namespace, type(e.value), e.name, e.id) has_errors = len(errors) > 0 return Plan(create=plan.create, share=plan.share, delete=plan.delete, modify=plan.modify, modifications_diff=plan.modifications_diff, errors=errors if has_errors else None)