def fetch_provider_vault_secret(path, version, name, labels, annotations, type,
integration, integration_version):
# get the fields from vault
vault_client = VaultClient()
raw_data = vault_client.read_all({'path': path, 'version': version})
# construct oc resource
body = {
"apiVersion": "v1",
"kind": "Secret",
"type": type,
"metadata": {
"name": name,
"annotations": annotations
}
}
if labels:
body['metadata']['labels'] = labels
if raw_data.items():
body['data'] = {}
# populate data
for k, v in raw_data.items():
if v == "":
continue
if k.lower().endswith(QONTRACT_BASE64_SUFFIX):
k = k[:-len(QONTRACT_BASE64_SUFFIX)]
elif v is not None:
v = base64.b64encode(v.encode()).decode('utf-8')
body['data'][k] = v
try:
return OR(body, integration, integration_version, error_details=path)
except ConstructResourceError as e:
raise FetchResourceError(str(e))
def check_alertmanager_config(data,
path,
alertmanager_config_key,
decode_base64=False):
try:
config = data[alertmanager_config_key]
except KeyError:
e_msg = f"error validating alertmanager config in {path}: " \
f"missing key {alertmanager_config_key}"
raise FetchResourceError(e_msg)
if decode_base64:
config = base64.b64decode(config).decode('utf-8')
result = amtool.check_config(config)
if not result:
e_msg = f"error validating alertmanager config in {path}: {result}"
raise FetchResourceError(e_msg)
def get_known_hosts(self, jh):
known_hosts_path = jh['knownHosts']
gqlapi = gql.get_api()
try:
known_hosts = gqlapi.get_resource(known_hosts_path)
except gql.GqlGetResourceError as e:
raise FetchResourceError(str(e))
return known_hosts['content']
def collect_configs(self, configs):
gqlapi = gql.get_api()
instances = \
{c['instance']['name']: {
'serverUrl': c['instance']['serverUrl'],
'token': c['instance']['token'],
'delete_method': c['instance']['deleteMethod']}
for c in configs}
working_dirs = {}
instance_urls = {}
for name, data in instances.items():
token = data['token']
server_url = data['serverUrl']
wd = tempfile.mkdtemp()
ini = JJB_INI
if not self.print_only:
ini = self.secret_reader.read(token)
ini = ini.replace('"', '')
ini = ini.replace('false', 'False')
ini_file_path = '{}/{}.ini'.format(wd, name)
with open(ini_file_path, 'w') as f:
f.write(ini)
f.write('\n')
working_dirs[name] = wd
instance_urls[name] = server_url
self.sort(configs)
for c in configs:
instance_name = c['instance']['name']
config = c['config']
config_file_path = \
'{}/config.yaml'.format(working_dirs[instance_name])
if config:
with open(config_file_path, 'a') as f:
yaml.dump(yaml.load(config, Loader=yaml.FullLoader), f)
f.write('\n')
else:
config_path = c['config_path']
# get config data
try:
config_resource = gqlapi.get_resource(config_path)
config = config_resource['content']
except gql.GqlGetResourceError as e:
raise FetchResourceError(str(e))
with open(config_file_path, 'a') as f:
f.write(config)
f.write('\n')
self.instances = instances
self.instance_urls = instance_urls
self.working_dirs = working_dirs
def fetch_provider_resource(path, tfunc=None, tvars=None, validate_json=False):
gqlapi = gql.get_api()
# get resource data
try:
resource = gqlapi.get_resource(path)
except gql.GqlGetResourceError as e:
raise FetchResourceError(str(e))
content = resource['content']
if tfunc:
content = tfunc(content, tvars)
try:
resource['body'] = anymarkup.parse(content, force_types=None)
resource['body'].pop('$schema', None)
except TypeError:
body_type = type(resource['body']).__name__
e_msg = f"invalid resource type {body_type} found in path: {path}"
raise FetchResourceError(e_msg)
except anymarkup.AnyMarkupError:
e_msg = f"Could not parse data. Skipping resource: {path}"
raise FetchResourceError(e_msg)
if validate_json:
files = resource['body']['data']
for file_name, file_content in files.items():
try:
json.loads(file_content)
except ValueError:
e_msg = f"invalid json in {path} under {file_name}"
raise FetchResourceError(e_msg)
try:
return OR(resource['body'],
QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION,
error_details=path)
except ConstructResourceError as e:
raise FetchResourceError(str(e))
def print_diffs(self, io_dir):
current_path = path.join(io_dir, 'jjb', 'current')
current_files = self.get_files(current_path)
if not current_files:
raise FetchResourceError('current state not found for compare')
desired_path = path.join(io_dir, 'jjb', 'desired')
desired_files = self.get_files(desired_path)
create = self.compare_files(desired_files, current_files)
delete = self.compare_files(current_files, desired_files)
common = self.compare_files(desired_files, current_files, in_op=True)
self.print_diff(create, desired_path, 'create')
self.print_diff(delete, current_path, 'delete')
self.print_diff(common, desired_path, 'update')
def print_diffs(self, io_dir):
compare_err_str = ('unable to find current state data for compare. '
'If running in dry-run mode, first run with the '
'--no-compare option and use a config that points '
'to unmodified source. Then run again without '
'--no-compare and use a config that points to '
'a modified source')
current_path = path.join(io_dir, 'jjb', 'current')
current_files = self.get_files(current_path)
if not current_files:
raise FetchResourceError(compare_err_str)
desired_path = path.join(io_dir, 'jjb', 'desired')
desired_files = self.get_files(desired_path)
create = self.compare_files(desired_files, current_files)
delete = self.compare_files(current_files, desired_files)
common = self.compare_files(desired_files, current_files, in_op=True)
self.print_diff(create, desired_path, 'create')
self.print_diff(delete, current_path, 'delete')
self.print_diff(common, desired_path, 'update')
def fetch_provider_resource(path,
tfunc=None,
tvars=None,
validate_json=False,
validate_alertmanager_config=False,
alertmanager_config_key='alertmanager.yaml',
add_path_to_prom_rules=True):
gqlapi = gql.get_api()
# get resource data
try:
resource = gqlapi.get_resource(path)
except gql.GqlGetResourceError as e:
raise FetchResourceError(str(e))
content = resource['content']
if tfunc:
content = tfunc(content, tvars)
try:
resource['body'] = anymarkup.parse(content, force_types=None)
resource['body'].pop('$schema', None)
except TypeError:
body_type = type(resource['body']).__name__
e_msg = f"invalid resource type {body_type} found in path: {path}"
raise FetchResourceError(e_msg)
except anymarkup.AnyMarkupError:
e_msg = f"Could not parse data. Skipping resource: {path}"
raise FetchResourceError(e_msg)
if validate_json:
files = resource['body']['data']
for file_name, file_content in files.items():
try:
json.loads(file_content)
except ValueError:
e_msg = f"invalid json in {path} under {file_name}"
raise FetchResourceError(e_msg)
if validate_alertmanager_config:
decode_base64 = True if resource['body']['kind'] == 'Secret' else False
check_alertmanager_config(resource['body']['data'], path,
alertmanager_config_key, decode_base64)
if add_path_to_prom_rules:
body = resource['body']
if body['kind'] == 'PrometheusRule':
try:
groups = body['spec']['groups']
for group in groups:
rules = group['rules']
for rule in rules:
annotations = rule.get('annotations')
if not annotations:
continue
# TODO(mafriedm): make this better
rule['annotations']['html_url'] = \
f"{APP_INT_BASE_URL}/blob/master/resources{path}"
except Exception:
logging.warning('could not add html_url annotation to' +
body['name'])
try:
return OR(resource['body'],
QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION,
error_details=path)
except ConstructResourceError as e:
raise FetchResourceError(str(e))
