def check_alertmanager_config(data,
path,
alertmanager_config_key,
decode_base64=False):
try:
config = data[alertmanager_config_key]
except KeyError:
e_msg = (f"error validating alertmanager config in {path}: "
f"missing key {alertmanager_config_key}")
raise FetchResourceError(e_msg)
if decode_base64:
config = base64.b64decode(config).decode("utf-8")
result = amtool.check_config(config)
if not result:
e_msg = f"error validating alertmanager config in {path}: {result}"
raise FetchResourceError(e_msg)
def get_known_hosts(jh):
known_hosts_path = jh["knownHosts"]
gqlapi = gql.get_api()
try:
known_hosts = gqlapi.get_resource(known_hosts_path)
except gql.GqlGetResourceError as e:
raise FetchResourceError(str(e))
return known_hosts["content"]
def collect_configs(self, configs):
gqlapi = gql.get_api()
instances = {
c["instance"]["name"]: {
"serverUrl": c["instance"]["serverUrl"],
"token": c["instance"]["token"],
"delete_method": c["instance"]["deleteMethod"],
}
for c in configs
}
working_dirs = {}
instance_urls = {}
for name, data in instances.items():
token = data["token"]
server_url = data["serverUrl"]
wd = tempfile.mkdtemp()
ini = JJB_INI
if not self.print_only:
ini = self.secret_reader.read(token)
ini = ini.replace('"', "")
ini = ini.replace("false", "False")
ini_file_path = "{}/{}.ini".format(wd, name)
with open(ini_file_path, "w") as f:
f.write(ini)
f.write("\n")
working_dirs[name] = wd
instance_urls[name] = server_url
self.sort(configs)
for c in configs:
instance_name = c["instance"]["name"]
config = c["config"]
config_file_path = "{}/config.yaml".format(
working_dirs[instance_name])
if config:
with open(config_file_path, "a") as f:
yaml.dump(yaml.load(config, Loader=yaml.FullLoader), f)
f.write("\n")
else:
config_path = c["config_path"]
# get config data
try:
config_resource = gqlapi.get_resource(config_path)
config = config_resource["content"]
except gql.GqlGetResourceError as e:
raise FetchResourceError(str(e))
with open(config_file_path, "a") as f:
f.write(config)
f.write("\n")
self.instances = instances
self.instance_urls = instance_urls
self.working_dirs = working_dirs
def collect_configs(self, configs):
gqlapi = gql.get_api()
instances = \
{c['instance']['name']: {
'serverUrl': c['instance']['serverUrl'],
'token': c['instance']['token'],
'delete_method': c['instance']['deleteMethod']}
for c in configs}
working_dirs = {}
instance_urls = {}
for name, data in instances.items():
token = data['token']
server_url = data['serverUrl']
wd = tempfile.mkdtemp()
ini = JJB_INI
if not self.print_only:
ini = self.secret_reader.read(token)
ini = ini.replace('"', '')
ini = ini.replace('false', 'False')
ini_file_path = '{}/{}.ini'.format(wd, name)
with open(ini_file_path, 'w') as f:
f.write(ini)
f.write('\n')
working_dirs[name] = wd
instance_urls[name] = server_url
self.sort(configs)
for c in configs:
instance_name = c['instance']['name']
config = c['config']
config_file_path = \
'{}/config.yaml'.format(working_dirs[instance_name])
if config:
with open(config_file_path, 'a') as f:
yaml.dump(yaml.load(config, Loader=yaml.FullLoader), f)
f.write('\n')
else:
config_path = c['config_path']
# get config data
try:
config_resource = gqlapi.get_resource(config_path)
config = config_resource['content']
except gql.GqlGetResourceError as e:
raise FetchResourceError(str(e))
with open(config_file_path, 'a') as f:
f.write(config)
f.write('\n')
self.instances = instances
self.instance_urls = instance_urls
self.working_dirs = working_dirs
def fetch_provider_vault_secret(
path,
version,
name,
labels,
annotations,
type,
integration,
integration_version,
validate_alertmanager_config=False,
alertmanager_config_key="alertmanager.yaml",
):
# get the fields from vault
vault_client = VaultClient()
raw_data = vault_client.read_all({"path": path, "version": version})
if validate_alertmanager_config:
check_alertmanager_config(raw_data, path, alertmanager_config_key)
# construct oc resource
body = {
"apiVersion": "v1",
"kind": "Secret",
"type": type,
"metadata": {
"name": name,
"annotations": annotations
},
}
if labels:
body["metadata"]["labels"] = labels
if raw_data.items():
body["data"] = {}
# populate data
for k, v in raw_data.items():
if v == "":
continue
if k.lower().endswith(QONTRACT_BASE64_SUFFIX):
k = k[:-len(QONTRACT_BASE64_SUFFIX)]
v = v.replace("\n", "")
elif v is not None:
v = base64.b64encode(v.encode()).decode("utf-8")
body["data"][k] = v
try:
return OR(body, integration, integration_version, error_details=path)
except ConstructResourceError as e:
raise FetchResourceError(str(e))
def fetch_provider_resource(
path,
tfunc=None,
tvars=None,
validate_json=False,
validate_alertmanager_config=False,
alertmanager_config_key="alertmanager.yaml",
add_path_to_prom_rules=True,
):
gqlapi = gql.get_api()
# get resource data
try:
resource = gqlapi.get_resource(path)
except gql.GqlGetResourceError as e:
raise FetchResourceError(str(e))
content = resource["content"]
if tfunc:
content = tfunc(content, tvars)
try:
body = anymarkup.parse(content, force_types=None)
body.pop("$schema", None)
except TypeError:
body_type = type(body).__name__
e_msg = f"invalid resource type {body_type} found in path: {path}"
raise FetchResourceError(e_msg)
except anymarkup.AnyMarkupError:
e_msg = f"Could not parse data. Skipping resource: {path}"
raise FetchResourceError(e_msg)
if validate_json:
files = body["data"]
for file_name, file_content in files.items():
try:
json.loads(file_content)
except ValueError:
e_msg = f"invalid json in {path} under {file_name}"
raise FetchResourceError(e_msg)
if validate_alertmanager_config:
if body["kind"] == "Secret":
if "data" in body:
am_data = body["data"]
decode_base64 = True
elif "stringData" in body:
am_data = body["stringData"]
decode_base64 = False
else:
am_data = body["data"]
decode_base64 = False
check_alertmanager_config(am_data, path, alertmanager_config_key,
decode_base64)
if add_path_to_prom_rules:
if body["kind"] == "PrometheusRule":
try:
groups = body["spec"]["groups"]
for group in groups:
rules = group["rules"]
for rule in rules:
annotations = rule.get("annotations")
if not annotations:
continue
# TODO(mafriedm): make this better
rule["annotations"][
"html_url"] = f"{APP_INT_BASE_URL}/blob/master/resources{path}"
except Exception:
logging.warning("could not add html_url annotation to" +
body["name"])
try:
return OR(body,
QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION,
error_details=path)
except ConstructResourceError as e:
raise FetchResourceError(str(e))
def fetch_provider_resource(path,
tfunc=None,
tvars=None,
validate_json=False,
validate_alertmanager_config=False,
alertmanager_config_key='alertmanager.yaml',
add_path_to_prom_rules=True):
gqlapi = gql.get_api()
# get resource data
try:
resource = gqlapi.get_resource(path)
except gql.GqlGetResourceError as e:
raise FetchResourceError(str(e))
content = resource['content']
if tfunc:
content = tfunc(content, tvars)
try:
resource['body'] = anymarkup.parse(content, force_types=None)
resource['body'].pop('$schema', None)
except TypeError:
body_type = type(resource['body']).__name__
e_msg = f"invalid resource type {body_type} found in path: {path}"
raise FetchResourceError(e_msg)
except anymarkup.AnyMarkupError:
e_msg = f"Could not parse data. Skipping resource: {path}"
raise FetchResourceError(e_msg)
if validate_json:
files = resource['body']['data']
for file_name, file_content in files.items():
try:
json.loads(file_content)
except ValueError:
e_msg = f"invalid json in {path} under {file_name}"
raise FetchResourceError(e_msg)
if validate_alertmanager_config:
decode_base64 = True if resource['body']['kind'] == 'Secret' else False
check_alertmanager_config(resource['body']['data'], path,
alertmanager_config_key, decode_base64)
if add_path_to_prom_rules:
body = resource['body']
if body['kind'] == 'PrometheusRule':
try:
groups = body['spec']['groups']
for group in groups:
rules = group['rules']
for rule in rules:
annotations = rule.get('annotations')
if not annotations:
continue
# TODO(mafriedm): make this better
rule['annotations']['html_url'] = \
f"{APP_INT_BASE_URL}/blob/master/resources{path}"
except Exception:
logging.warning('could not add html_url annotation to' +
body['name'])
try:
return OR(resource['body'],
QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION,
error_details=path)
except ConstructResourceError as e:
raise FetchResourceError(str(e))