def fetch_desired_state(clusters, vault_input_path, settings):
desired_state = []
error = False
for cluster_info in clusters:
cluster = cluster_info['name']
auth = cluster_info['auth']
service = auth['service']
if service != 'github-org-team':
# currently not supported
continue
org = auth['org']
team = auth['team']
secret_path = f'{vault_input_path}/{QONTRACT_INTEGRATION}/' + \
f'{service}/{org}/{team}'
secret = {'path': secret_path}
try:
oauth_data = secret_reader.read_all(secret, settings=settings)
client_id = oauth_data['client-id']
client_secret = oauth_data['client-secret']
except Exception:
logging.error(f"unable to read secret in path {secret['path']}")
error = True
continue
item = {
'cluster': cluster,
'name': f'github-{org}',
'client_id': client_id,
'client_secret': client_secret,
'teams': [f'{org}/{team}']
}
desired_state.append(item)
return desired_state, error
def get_basic_auth(self, token, settings):
required_keys = ['username', 'password']
secret = secret_reader.read_all(token, settings)
ok = all(elem in secret.keys() for elem in required_keys)
if not ok:
raise KeyError('[{}] secret is missing required keys'.format(
self.project))
return (secret['username'], secret['password'])
def _initiate_image_auth(self, saas_file):
auth = saas_file.get('authentication') or {}
auth_image = auth.get('image') or {}
if auth_image:
creds = \
secret_reader.read_all(auth_image, settings=self.settings)
else:
creds = None
return creds
def __init__(self, dry_run, thread_pool_size):
self.dry_run = dry_run
self.thread_pool_size = thread_pool_size
self.settings = queries.get_app_interface_settings()
secret_content = secret_reader.read_all({'path': DASHDOTDB_SECRET},
settings=self.settings)
self.dashdotdb_url = secret_content['API_URL']
self.dashdotdb_token = secret_content['ACCESS_TOKEN']
def __init__(self, dry_run, thread_pool_size):
self.dry_run = dry_run
self.thread_pool_size = thread_pool_size
self.settings = queries.get_app_interface_settings()
secret_content = secret_reader.read_all({'path': DASHDOTDB_SECRET},
settings=self.settings)
self.dashdotdb_url = secret_content['url']
self.dashdotdb_user = secret_content['username']
self.dashdotdb_pass = secret_content['password']
def get_smtp_config(path, settings):
config = {}
required_keys = ('password', 'port', 'require_tls', 'server', 'username')
data = secret_reader.read_all({'path': path}, settings=settings)
try:
for k in required_keys:
config[k] = data[k]
except KeyError as e:
raise Exception(
"Missing expected SMTP config key in vault secret: {}".format(e))
return config
def _get_push_creds(self):
result = self.gqlapi.query(self.QUAY_ORG_CATALOG_QUERY)
creds = {}
for org_data in result['quay_orgs']:
push_secret = org_data['pushCredentials']
if push_secret is None:
continue
raw_data = secret_reader.read_all(push_secret,
settings=self.settings)
org = org_data['name']
creds[org] = f'{raw_data["user"]}:{raw_data["token"]}'
return creds
def _get_push_creds(self):
result = self.gqlapi.query(self.GCR_PROJECT_CATALOG_QUERY)
creds = {}
for project_data in result['projects']:
push_secret = project_data['pushCredentials']
if push_secret is None:
continue
raw_data = secret_reader.read_all(push_secret,
settings=self.settings)
project = project_data['name']
token = base64.b64decode(raw_data["token"]).decode()
creds[project] = f'{raw_data["user"]}:{token}'
return creds
def _initiate_image_auth(self, saas_file):
"""
This function initiates a dict required for image authentication.
This dict will be used as kwargs for sertoolbox's Image.
The image authentication secret specified in the saas file must
contain the 'user' and 'token' keys, and may optionally contain
a 'url' key specifying the image registry url to be passed to check
if an image should be checked using these credentials.
The function returns the keys extracted from the secret in the
structure expected by sretoolbox's Image:
'user' --> 'username'
'token' --> 'password'
'url' --> 'auth_server' (optional)
"""
auth = saas_file.get('authentication')
if not auth:
return {}
auth_image_secret = auth.get('image')
if not auth_image_secret:
return {}
creds = \
secret_reader.read_all(auth_image_secret, settings=self.settings)
required_keys = ['user', 'token']
ok = all(k in creds.keys() for k in required_keys)
if not ok:
logging.warning(
"the specified image authentication secret " +
f"found in path {auth_image_secret['path']} " +
f"does not contain all required keys: {required_keys}"
)
return {}
image_auth = {
'username': creds['user'],
'password': creds['token']
}
url = creds.get('url')
if url:
image_auth['auth_server']: url
return image_auth
def run(dry_run=False):
instance = queries.get_gitlab_instance()
settings = queries.get_app_interface_settings()
gl = GitLabApi(instance, settings=settings)
# Jira
repos = queries.get_repos_gitlab_jira(server=gl.server)
for repo in repos:
skip = False
repo_url = repo['url']
project = gl.get_project(repo_url=repo_url)
services = project.services
current_jira = services.get('jira')
desired_jira = repo['jira']
desired_jira_url = desired_jira['serverUrl']
desired_jira_crdentials = \
secret_reader.read_all(desired_jira['token'], settings=settings)
if current_jira.active:
properties = current_jira.properties
desired_jira_username = desired_jira_crdentials['username']
if properties['url'] == desired_jira_url \
and properties['username'] == desired_jira_username:
skip = True
if skip:
continue
logging.info(['update_jira', repo_url, desired_jira_url])
if not dry_run:
new_data = {
'active': True,
'url': desired_jira_url,
'username': desired_jira_crdentials['username'],
'password': desired_jira_crdentials['password'],
'commit_events': True,
'merge_requests_events': True,
'comment_on_event_enabled': False
}
services.update('jira', new_data=new_data)
def get_tf_secrets(self, account):
account_name = account['name']
automation_token = account['automationToken']
secret = secret_reader.read_all(automation_token,
settings=self.settings)
return (account_name, secret)
def process_sync_tasks(self):
eight_hours = 28800 # 60 * 60 * 8
is_deep_sync = self._is_deep_sync(interval=eight_hours)
summary = self.process_repos_query()
sync_tasks = defaultdict(list)
for org, data in summary.items():
for item in data:
image = Image(f'{item["server_url"]}/{org}/{item["name"]}')
mirror_url = item['mirror']['url']
username = None
password = None
mirror_creds = None
if item['mirror']['pullCredentials'] is not None:
pull_credentials = item['mirror']['pullCredentials']
raw_data = secret_reader.read_all(pull_credentials,
settings=self.settings)
username = raw_data["user"]
password = raw_data["token"]
mirror_creds = f'{username}:{password}'
image_mirror = Image(mirror_url, username=username,
password=password)
tags = item['mirror'].get('tags')
tags_exclude = item['mirror'].get('tagsExclude')
for tag in image_mirror:
if not self.sync_tag(tags=tags, tags_exclude=tags_exclude,
candidate=tag):
continue
upstream = image_mirror[tag]
downstream = image[tag]
if tag not in image:
_LOG.debug('Image %s and mirror %s are out off sync',
downstream, upstream)
sync_tasks[org].append({'mirror_url': str(upstream),
'mirror_creds': mirror_creds,
'image_url': str(downstream)})
continue
# Deep (slow) check only in non dry-run mode
if self.dry_run:
_LOG.debug('Image %s and mirror %s are in sync',
downstream, upstream)
continue
# Deep (slow) check only from time to time
if not is_deep_sync:
_LOG.debug('Image %s and mirror %s are in sync',
downstream, upstream)
continue
try:
if downstream == upstream:
_LOG.debug('Image %s and mirror %s are in sync',
downstream, upstream)
continue
except ImageComparisonError as details:
_LOG.error('[%s]', details)
continue
_LOG.debug('Image %s and mirror %s are out of sync',
downstream, upstream)
sync_tasks[org].append({'mirror_url': str(upstream),
'mirror_creds': mirror_creds,
'image_url': str(downstream)})
return sync_tasks
