def fetch_provider_vault_secret(path, version, name, labels, annotations, type, integration, integration_version): # get the fields from vault vault_client = VaultClient() raw_data = vault_client.read_all({'path': path, 'version': version}) # construct oc resource body = { "apiVersion": "v1", "kind": "Secret", "type": type, "metadata": { "name": name, "annotations": annotations } } if labels: body['metadata']['labels'] = labels if raw_data.items(): body['data'] = {} # populate data for k, v in raw_data.items(): if v == "": continue if k.lower().endswith(QONTRACT_BASE64_SUFFIX): k = k[:-len(QONTRACT_BASE64_SUFFIX)] elif v is not None: v = base64.b64encode(v.encode()).decode('utf-8') body['data'][k] = v try: return OR(body, integration, integration_version, error_details=path) except ConstructResourceError as e: raise FetchResourceError(str(e))
def check_alertmanager_config(data, path, alertmanager_config_key, decode_base64=False): try: config = data[alertmanager_config_key] except KeyError: e_msg = f"error validating alertmanager config in {path}: " \ f"missing key {alertmanager_config_key}" raise FetchResourceError(e_msg) if decode_base64: config = base64.b64decode(config).decode('utf-8') result = amtool.check_config(config) if not result: e_msg = f"error validating alertmanager config in {path}: {result}" raise FetchResourceError(e_msg)
def get_known_hosts(self, jh): known_hosts_path = jh['knownHosts'] gqlapi = gql.get_api() try: known_hosts = gqlapi.get_resource(known_hosts_path) except gql.GqlGetResourceError as e: raise FetchResourceError(str(e)) return known_hosts['content']
def collect_configs(self, configs): gqlapi = gql.get_api() instances = \ {c['instance']['name']: { 'serverUrl': c['instance']['serverUrl'], 'token': c['instance']['token'], 'delete_method': c['instance']['deleteMethod']} for c in configs} working_dirs = {} instance_urls = {} for name, data in instances.items(): token = data['token'] server_url = data['serverUrl'] wd = tempfile.mkdtemp() ini = JJB_INI if not self.print_only: ini = self.secret_reader.read(token) ini = ini.replace('"', '') ini = ini.replace('false', 'False') ini_file_path = '{}/{}.ini'.format(wd, name) with open(ini_file_path, 'w') as f: f.write(ini) f.write('\n') working_dirs[name] = wd instance_urls[name] = server_url self.sort(configs) for c in configs: instance_name = c['instance']['name'] config = c['config'] config_file_path = \ '{}/config.yaml'.format(working_dirs[instance_name]) if config: with open(config_file_path, 'a') as f: yaml.dump(yaml.load(config, Loader=yaml.FullLoader), f) f.write('\n') else: config_path = c['config_path'] # get config data try: config_resource = gqlapi.get_resource(config_path) config = config_resource['content'] except gql.GqlGetResourceError as e: raise FetchResourceError(str(e)) with open(config_file_path, 'a') as f: f.write(config) f.write('\n') self.instances = instances self.instance_urls = instance_urls self.working_dirs = working_dirs
def fetch_provider_resource(path, tfunc=None, tvars=None, validate_json=False): gqlapi = gql.get_api() # get resource data try: resource = gqlapi.get_resource(path) except gql.GqlGetResourceError as e: raise FetchResourceError(str(e)) content = resource['content'] if tfunc: content = tfunc(content, tvars) try: resource['body'] = anymarkup.parse(content, force_types=None) resource['body'].pop('$schema', None) except TypeError: body_type = type(resource['body']).__name__ e_msg = f"invalid resource type {body_type} found in path: {path}" raise FetchResourceError(e_msg) except anymarkup.AnyMarkupError: e_msg = f"Could not parse data. Skipping resource: {path}" raise FetchResourceError(e_msg) if validate_json: files = resource['body']['data'] for file_name, file_content in files.items(): try: json.loads(file_content) except ValueError: e_msg = f"invalid json in {path} under {file_name}" raise FetchResourceError(e_msg) try: return OR(resource['body'], QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION, error_details=path) except ConstructResourceError as e: raise FetchResourceError(str(e))
def print_diffs(self, io_dir): current_path = path.join(io_dir, 'jjb', 'current') current_files = self.get_files(current_path) if not current_files: raise FetchResourceError('current state not found for compare') desired_path = path.join(io_dir, 'jjb', 'desired') desired_files = self.get_files(desired_path) create = self.compare_files(desired_files, current_files) delete = self.compare_files(current_files, desired_files) common = self.compare_files(desired_files, current_files, in_op=True) self.print_diff(create, desired_path, 'create') self.print_diff(delete, current_path, 'delete') self.print_diff(common, desired_path, 'update')
def print_diffs(self, io_dir): compare_err_str = ('unable to find current state data for compare. ' 'If running in dry-run mode, first run with the ' '--no-compare option and use a config that points ' 'to unmodified source. Then run again without ' '--no-compare and use a config that points to ' 'a modified source') current_path = path.join(io_dir, 'jjb', 'current') current_files = self.get_files(current_path) if not current_files: raise FetchResourceError(compare_err_str) desired_path = path.join(io_dir, 'jjb', 'desired') desired_files = self.get_files(desired_path) create = self.compare_files(desired_files, current_files) delete = self.compare_files(current_files, desired_files) common = self.compare_files(desired_files, current_files, in_op=True) self.print_diff(create, desired_path, 'create') self.print_diff(delete, current_path, 'delete') self.print_diff(common, desired_path, 'update')
def fetch_provider_resource(path, tfunc=None, tvars=None, validate_json=False, validate_alertmanager_config=False, alertmanager_config_key='alertmanager.yaml', add_path_to_prom_rules=True): gqlapi = gql.get_api() # get resource data try: resource = gqlapi.get_resource(path) except gql.GqlGetResourceError as e: raise FetchResourceError(str(e)) content = resource['content'] if tfunc: content = tfunc(content, tvars) try: resource['body'] = anymarkup.parse(content, force_types=None) resource['body'].pop('$schema', None) except TypeError: body_type = type(resource['body']).__name__ e_msg = f"invalid resource type {body_type} found in path: {path}" raise FetchResourceError(e_msg) except anymarkup.AnyMarkupError: e_msg = f"Could not parse data. Skipping resource: {path}" raise FetchResourceError(e_msg) if validate_json: files = resource['body']['data'] for file_name, file_content in files.items(): try: json.loads(file_content) except ValueError: e_msg = f"invalid json in {path} under {file_name}" raise FetchResourceError(e_msg) if validate_alertmanager_config: decode_base64 = True if resource['body']['kind'] == 'Secret' else False check_alertmanager_config(resource['body']['data'], path, alertmanager_config_key, decode_base64) if add_path_to_prom_rules: body = resource['body'] if body['kind'] == 'PrometheusRule': try: groups = body['spec']['groups'] for group in groups: rules = group['rules'] for rule in rules: annotations = rule.get('annotations') if not annotations: continue # TODO(mafriedm): make this better rule['annotations']['html_url'] = \ f"{APP_INT_BASE_URL}/blob/master/resources{path}" except Exception: logging.warning('could not add html_url annotation to' + body['name']) try: return OR(resource['body'], QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION, error_details=path) except ConstructResourceError as e: raise FetchResourceError(str(e))