def __init__(self, dry_run=False): self.dry_run = dry_run self.gqlapi = gql.get_api() settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=settings) self.skopeo_cli = Skopeo(dry_run) self.push_creds = self._get_push_creds()
def __init__(self, instance, dry_run): self.dry_run = dry_run self.instance = instance self.settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=self.settings) self.skopeo_cli = Skopeo(dry_run) self.error = False identifier = instance["identifier"] account = instance["account"] region = instance.get("region") self.aws_cli = AWSApi( thread_pool_size=1, accounts=[self._get_aws_account_info(account)], settings=self.settings, init_ecr_auth_tokens=True, ) self.aws_cli.map_ecr_resources() self.ecr_uri = self._get_image_uri( account=account, repository=identifier, ) if self.ecr_uri is None: self.error = True LOG.error(f"Could not find the ECR repository {identifier}") self.ecr_username, self.ecr_password = self._get_ecr_creds( account=account, region=region, ) self.ecr_auth = f"{self.ecr_username}:{self.ecr_password}" self.image_username = None self.image_password = None self.image_auth = None pull_secret = self.instance["mirror"]["pullCredentials"] if pull_secret is not None: raw_data = self.secret_reader.read_all(pull_secret) self.image_username = raw_data["user"] self.image_password = raw_data["token"] self.image_auth = f"{self.image_username}:{self.image_password}"
class QuayMirror: QUAY_ORG_CATALOG_QUERY = """ { quay_orgs: quay_orgs_v1 { name pushCredentials { path field } } } """ QUAY_REPOS_QUERY = """ { apps: apps_v1 { quayRepos { org { name } items { name mirror } } } } """ def __init__(self, dry_run=False): self.dry_run = dry_run self.gqlapi = gql.get_api() self.settings = queries.get_app_interface_settings() self.skopeo_cli = Skopeo(dry_run) self.push_creds = self._get_push_creds() def run(self): sync_tasks = self.process_sync_tasks() for org, data in sync_tasks.items(): for item in data: try: self.skopeo_cli.copy(src_image=item['mirror_url'], dst_image=item['image_url'], dest_creds=self.push_creds[org]) except SkopeoCmdError as details: _LOG.error('[%s]', details) def process_repos_query(self): result = self.gqlapi.query(self.QUAY_REPOS_QUERY) summary = defaultdict(list) for app in result['apps']: quay_repos = app.get('quayRepos') if quay_repos is None: continue for quay_repo in quay_repos: org = quay_repo['org']['name'] for item in quay_repo['items']: if item['mirror'] is None: continue summary[org].append({ 'name': item["name"], 'mirror': item['mirror'] }) return summary def process_sync_tasks(self): eight_hours = 28800 # 60 * 60 * 8 is_deep_sync = self._is_deep_sync(interval=eight_hours) summary = self.process_repos_query() sync_tasks = defaultdict(list) for org, data in summary.items(): for item in data: image = Image(f'quay.io/{org}/{item["name"]}') image_mirror = Image(item['mirror']) for tag in image_mirror: upstream = image_mirror[tag] downstream = image[tag] if tag not in image: _LOG.debug('Image %s and mirror %s are out off sync', downstream, upstream) sync_tasks[org].append({ 'mirror_url': str(upstream), 'image_url': str(downstream) }) continue # Deep (slow) check only in non dry-run mode if self.dry_run: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue # Deep (slow) check only from time to time if not is_deep_sync: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue try: if downstream == upstream: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue except ImageComparisonError as details: _LOG.error('[%s]', details) continue _LOG.debug('Image %s and mirror %s are out of sync', downstream, upstream) sync_tasks[org].append({ 'mirror_url': str(upstream), 'image_url': str(downstream) }) return sync_tasks def _is_deep_sync(self, interval): control_file_name = 'qontract-reconcile-quay-mirror.timestamp' control_file_path = os.path.join(tempfile.gettempdir(), control_file_name) try: with open(control_file_path, 'r') as file_obj: last_deep_sync = float(file_obj.read()) except FileNotFoundError: self._record_timestamp(control_file_path) return True next_deep_sync = last_deep_sync + interval if time.time() >= next_deep_sync: self._record_timestamp(control_file_path) return True return False @staticmethod def _record_timestamp(path): with open(path, 'w') as file_object: file_object.write(str(time.time())) def _get_push_creds(self): result = self.gqlapi.query(self.QUAY_ORG_CATALOG_QUERY) creds = {} for org_data in result['quay_orgs']: push_secret = org_data['pushCredentials'] if push_secret is None: continue raw_data = secret_reader.read_all(push_secret, settings=self.settings) org = org_data['name'] creds[org] = f'{raw_data["user"]}:{raw_data["token"]}' return creds
def __init__(self, dry_run=False): self.dry_run = dry_run self.skopeo_cli = Skopeo(dry_run) self.quay_api_store = get_quay_api_store()
class QuayMirrorOrg: def __init__(self, dry_run=False): self.dry_run = dry_run self.skopeo_cli = Skopeo(dry_run) self.quay_api_store = get_quay_api_store() def run(self): sync_tasks = self.process_sync_tasks() for org, data in sync_tasks.items(): for item in data: try: self.skopeo_cli.copy(src_image=item['mirror_url'], src_creds=item['mirror_creds'], dst_image=item['image_url'], dest_creds=self.get_push_creds(org)) except SkopeoCmdError as details: _LOG.error('[%s]', details) def process_org_mirrors(self, summary): """adds new keys to the summary dict with information about mirrored orgs It collects the list of repositories in the upstream org from an API call and not from App-Interface. :param summary: summary :type summary: dict :return: summary :rtype: dict """ for org_key, org_info in self.quay_api_store.items(): if not org_info.get('mirror'): continue quay_api = org_info['api'] upstream_org_key = org_info['mirror'] upstream_org = self.quay_api_store[upstream_org_key] upstream_quay_api = upstream_org['api'] username = upstream_org['push_token']['user'] token = upstream_org['push_token']['token'] repos = [item['name'] for item in quay_api.list_images()] for repo in upstream_quay_api.list_images(): if repo['name'] not in repos: continue server_url = upstream_org['url'] url = f"{server_url}/{org_key.org_name}/{repo['name']}" data = { 'name': repo['name'], 'mirror': { 'url': url, 'username': username, 'token': token, } } summary[org_key].append(data) return summary def process_sync_tasks(self): eight_hours = 28800 # 60 * 60 * 8 is_deep_sync = self._is_deep_sync(interval=eight_hours) summary = defaultdict(list) self.process_org_mirrors(summary) sync_tasks = defaultdict(list) for org_key, data in summary.items(): org = self.quay_api_store[org_key] org_name = org_key.org_name server_url = org['url'] username = org['push_token']['user'] password = org['push_token']['token'] for item in data: image = Image(f'{server_url}/{org_name}/{item["name"]}', username=username, password=password) mirror_url = item['mirror']['url'] mirror_username = None mirror_password = None mirror_creds = None if item['mirror'].get('username') and \ item['mirror'].get('token'): mirror_username = item['mirror']['username'] mirror_password = item['mirror']['token'] mirror_creds = f'{mirror_username}:{mirror_password}' image_mirror = Image(mirror_url, username=mirror_username, password=mirror_password) for tag in image_mirror: upstream = image_mirror[tag] downstream = image[tag] if tag not in image: _LOG.debug('Image %s and mirror %s are out of sync', downstream, upstream) task = {'mirror_url': str(upstream), 'mirror_creds': mirror_creds, 'image_url': str(downstream)} sync_tasks[org_key].append(task) continue # Deep (slow) check only in non dry-run mode if self.dry_run: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue # Deep (slow) check only from time to time if not is_deep_sync: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue try: if downstream == upstream: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue except ImageComparisonError as details: _LOG.error('[%s]', details) continue _LOG.debug('Image %s and mirror %s are out of sync', downstream, upstream) sync_tasks[org_key].append({'mirror_url': str(upstream), 'mirror_creds': mirror_creds, 'image_url': str(downstream)}) return sync_tasks def _is_deep_sync(self, interval): control_file_name = 'qontract-reconcile-quay-mirror-org.timestamp' control_file_path = os.path.join(tempfile.gettempdir(), control_file_name) try: with open(control_file_path, 'r') as file_obj: last_deep_sync = float(file_obj.read()) except FileNotFoundError: self._record_timestamp(control_file_path) return True next_deep_sync = last_deep_sync + interval if time.time() >= next_deep_sync: self._record_timestamp(control_file_path) return True return False @staticmethod def _record_timestamp(path): with open(path, 'w') as file_object: file_object.write(str(time.time())) def get_push_creds(self, org_key): """returns username and password for the given org :param org_key: org_key :type org_key: tuple(instance, org_name) :return: tuple containing username and password :rtype: tuple(str, str) """ push_token = self.quay_api_store[org_key]['push_token'] username = push_token['user'] password = push_token['token'] return f"{username}:{password}"
class QuayMirror: GCR_PROJECT_CATALOG_QUERY = """ { projects: gcp_projects_v1 { name pushCredentials { path field } } } """ GCR_REPOS_QUERY = """ { apps: apps_v1 { gcrRepos { project { name } items { name mirror { url pullCredentials { path field } tags tagsExclude } } } } } """ def __init__(self, dry_run=False): self.dry_run = dry_run self.gqlapi = gql.get_api() self.settings = queries.get_app_interface_settings() self.skopeo_cli = Skopeo(dry_run) self.push_creds = self._get_push_creds() def run(self): sync_tasks = self.process_sync_tasks() for org, data in sync_tasks.items(): for item in data: try: self.skopeo_cli.copy(src_image=item['mirror_url'], src_creds=item['mirror_creds'], dst_image=item['image_url'], dest_creds=self.push_creds[org]) except SkopeoCmdError as details: _LOG.error('[%s]', details) def process_repos_query(self): result = self.gqlapi.query(self.GCR_REPOS_QUERY) summary = defaultdict(list) for app in result['apps']: gcr_repos = app.get('gcrRepos') if gcr_repos is None: continue for gcr_repo in gcr_repos: project = gcr_repo['project']['name'] server_url = gcr_repo['project'].get('serverUrl') or 'gcr.io' for item in gcr_repo['items']: if item['mirror'] is None: continue summary[project].append({ 'name': item["name"], 'mirror': item['mirror'], 'server_url': server_url }) return summary def sync_tag(self, tags, tags_exclude, candidate): if tags is not None: for tag in tags: if re.match(tag, candidate): return True # When tags is defined, we don't look at # tags_exclude return False if tags_exclude is not None: for tag_exclude in tags_exclude: if re.match(tag_exclude, candidate): return False return True # Both tags and tags_exclude are None, so # tag must be synced return True def process_sync_tasks(self): eight_hours = 28800 # 60 * 60 * 8 is_deep_sync = self._is_deep_sync(interval=eight_hours) summary = self.process_repos_query() sync_tasks = defaultdict(list) for org, data in summary.items(): for item in data: image = Image(f'{item["server_url"]}/{org}/{item["name"]}') mirror_url = item['mirror']['url'] username = None password = None mirror_creds = None if item['mirror']['pullCredentials'] is not None: pull_credentials = item['mirror']['pullCredentials'] raw_data = secret_reader.read_all(pull_credentials, settings=self.settings) username = raw_data["user"] password = raw_data["token"] mirror_creds = f'{username}:{password}' image_mirror = Image(mirror_url, username=username, password=password) tags = item['mirror'].get('tags') tags_exclude = item['mirror'].get('tagsExclude') for tag in image_mirror: if not self.sync_tag(tags=tags, tags_exclude=tags_exclude, candidate=tag): continue upstream = image_mirror[tag] downstream = image[tag] if tag not in image: _LOG.debug('Image %s and mirror %s are out off sync', downstream, upstream) sync_tasks[org].append({ 'mirror_url': str(upstream), 'mirror_creds': mirror_creds, 'image_url': str(downstream) }) continue # Deep (slow) check only in non dry-run mode if self.dry_run: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue # Deep (slow) check only from time to time if not is_deep_sync: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue try: if downstream == upstream: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue except ImageComparisonError as details: _LOG.error('[%s]', details) continue _LOG.debug('Image %s and mirror %s are out of sync', downstream, upstream) sync_tasks[org].append({ 'mirror_url': str(upstream), 'mirror_creds': mirror_creds, 'image_url': str(downstream) }) return sync_tasks def _is_deep_sync(self, interval): control_file_name = 'qontract-reconcile-gcr-mirror.timestamp' control_file_path = os.path.join(tempfile.gettempdir(), control_file_name) try: with open(control_file_path, 'r') as file_obj: last_deep_sync = float(file_obj.read()) except FileNotFoundError: self._record_timestamp(control_file_path) return True next_deep_sync = last_deep_sync + interval if time.time() >= next_deep_sync: self._record_timestamp(control_file_path) return True return False @staticmethod def _record_timestamp(path): with open(path, 'w') as file_object: file_object.write(str(time.time())) def _get_push_creds(self): result = self.gqlapi.query(self.GCR_PROJECT_CATALOG_QUERY) creds = {} for project_data in result['projects']: push_secret = project_data['pushCredentials'] if push_secret is None: continue raw_data = secret_reader.read_all(push_secret, settings=self.settings) project = project_data['name'] token = base64.b64decode(raw_data["token"]).decode() creds[project] = f'{raw_data["user"]}:{token}' return creds
class QuayMirror: QUAY_ORG_CATALOG_QUERY = """ { quay_orgs: quay_orgs_v1 { name pushCredentials { path field } instance { name url } } } """ def __init__(self, dry_run=False): self.dry_run = dry_run self.gqlapi = gql.get_api() settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=settings) self.skopeo_cli = Skopeo(dry_run) self.push_creds = self._get_push_creds() def run(self): sync_tasks = self.process_sync_tasks() for org, data in sync_tasks.items(): for item in data: try: self.skopeo_cli.copy(src_image=item['mirror_url'], src_creds=item['mirror_creds'], dst_image=item['image_url'], dest_creds=self.push_creds[org]) except SkopeoCmdError as details: _LOG.error('[%s]', details) @staticmethod def process_repos_query(): apps = queries.get_quay_repos() summary = defaultdict(list) for app in apps: quay_repos = app.get('quayRepos') if quay_repos is None: continue for quay_repo in quay_repos: org = quay_repo['org']['name'] instance = quay_repo['org']['instance']['name'] server_url = quay_repo['org']['instance']['url'] for item in quay_repo['items']: if item['mirror'] is None: continue mirror_image = Image(item['mirror']['url']) if (mirror_image.registry == 'docker.io' and mirror_image.repository == 'library' and item['public']): _LOG.error("Image %s can't be mirrored to a public " "quay repository.", mirror_image) sys.exit(ExitCodes.ERROR) org_key = OrgKey(instance, org) summary[org_key].append({'name': item["name"], 'mirror': item['mirror'], 'server_url': server_url}) return summary @staticmethod def sync_tag(tags, tags_exclude, candidate): if tags is not None: for tag in tags: if re.match(tag, candidate): return True # When tags is defined, we don't look at # tags_exclude return False if tags_exclude is not None: for tag_exclude in tags_exclude: if re.match(tag_exclude, candidate): return False return True # Both tags and tags_exclude are None, so # tag must be synced return True def process_sync_tasks(self): eight_hours = 28800 # 60 * 60 * 8 is_deep_sync = self._is_deep_sync(interval=eight_hours) summary = self.process_repos_query() sync_tasks = defaultdict(list) for org_key, data in summary.items(): org = org_key.org_name for item in data: push_creds = self.push_creds[org_key].split(':') image = Image(f'{item["server_url"]}/{org}/{item["name"]}', username=push_creds[0], password=push_creds[1]) mirror_url = item['mirror']['url'] username = None password = None mirror_creds = None if item['mirror']['pullCredentials'] is not None: pull_credentials = item['mirror']['pullCredentials'] raw_data = self.secret_reader.read_all(pull_credentials) username = raw_data["user"] password = raw_data["token"] mirror_creds = f'{username}:{password}' image_mirror = Image(mirror_url, username=username, password=password) tags = item['mirror'].get('tags') tags_exclude = item['mirror'].get('tagsExclude') for tag in image_mirror: if not self.sync_tag(tags=tags, tags_exclude=tags_exclude, candidate=tag): continue upstream = image_mirror[tag] downstream = image[tag] if tag not in image: _LOG.debug('Image %s and mirror %s are out off sync', downstream, upstream) task = {'mirror_url': str(upstream), 'mirror_creds': mirror_creds, 'image_url': str(downstream)} sync_tasks[org_key].append(task) continue # Deep (slow) check only in non dry-run mode if self.dry_run: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue # Deep (slow) check only from time to time if not is_deep_sync: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue try: if downstream == upstream: _LOG.debug('Image %s and mirror %s are in sync', downstream, upstream) continue except ImageComparisonError as details: _LOG.error('[%s]', details) continue _LOG.debug('Image %s and mirror %s are out of sync', downstream, upstream) sync_tasks[org_key].append({'mirror_url': str(upstream), 'mirror_creds': mirror_creds, 'image_url': str(downstream)}) return sync_tasks def _is_deep_sync(self, interval): control_file_name = 'qontract-reconcile-quay-mirror.timestamp' control_file_path = os.path.join(tempfile.gettempdir(), control_file_name) try: with open(control_file_path, 'r') as file_obj: last_deep_sync = float(file_obj.read()) except FileNotFoundError: self._record_timestamp(control_file_path) return True next_deep_sync = last_deep_sync + interval if time.time() >= next_deep_sync: self._record_timestamp(control_file_path) return True return False @staticmethod def _record_timestamp(path): with open(path, 'w') as file_object: file_object.write(str(time.time())) def _get_push_creds(self): result = self.gqlapi.query(self.QUAY_ORG_CATALOG_QUERY) creds = {} for org_data in result['quay_orgs']: push_secret = org_data['pushCredentials'] if push_secret is None: continue raw_data = self.secret_reader.read_all(push_secret) org = org_data['name'] instance = org_data['instance']['name'] org_key = OrgKey(instance, org) creds[org_key] = f'{raw_data["user"]}:{raw_data["token"]}' return creds
class EcrMirror: def __init__(self, instance, dry_run): self.dry_run = dry_run self.instance = instance self.settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=self.settings) self.skopeo_cli = Skopeo(dry_run) self.error = False identifier = instance["identifier"] account = instance["account"] region = instance.get("region") self.aws_cli = AWSApi( thread_pool_size=1, accounts=[self._get_aws_account_info(account)], settings=self.settings, init_ecr_auth_tokens=True, ) self.aws_cli.map_ecr_resources() self.ecr_uri = self._get_image_uri( account=account, repository=identifier, ) if self.ecr_uri is None: self.error = True LOG.error(f"Could not find the ECR repository {identifier}") self.ecr_username, self.ecr_password = self._get_ecr_creds( account=account, region=region, ) self.ecr_auth = f"{self.ecr_username}:{self.ecr_password}" self.image_username = None self.image_password = None self.image_auth = None pull_secret = self.instance["mirror"]["pullCredentials"] if pull_secret is not None: raw_data = self.secret_reader.read_all(pull_secret) self.image_username = raw_data["user"] self.image_password = raw_data["token"] self.image_auth = f"{self.image_username}:{self.image_password}" def run(self): if self.error: return ecr_mirror = Image(self.ecr_uri, username=self.ecr_username, password=self.ecr_password) image = Image( self.instance["mirror"]["url"], username=self.image_username, password=self.image_password, ) LOG.debug("[checking %s -> %s]", image, ecr_mirror) for tag in image: if tag not in ecr_mirror: try: self.skopeo_cli.copy( src_image=image[tag], src_creds=self.image_auth, dst_image=ecr_mirror[tag], dest_creds=self.ecr_auth, ) except SkopeoCmdError as details: LOG.error("[%s]", details) def _get_ecr_creds(self, account, region): if region is None: region = self.aws_cli.accounts[account]["resourcesDefaultRegion"] auth_token = f"{account}/{region}" data = self.aws_cli.auth_tokens[auth_token] auth_data = data["authorizationData"][0] token = auth_data["authorizationToken"] password = base64.b64decode(token).decode("utf-8").split(":")[1] return "AWS", password def _get_image_uri(self, account, repository): for repo in self.aws_cli.resources[account]["ecr"]: if repo["repositoryName"] == repository: return repo["repositoryUri"] @staticmethod def _get_aws_account_info(account): for account_info in queries.get_aws_accounts(): if "name" not in account_info: continue if account_info["name"] != account: continue return account_info
class EcrMirror: def __init__(self, instance, dry_run): self.dry_run = dry_run self.instance = instance self.settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=self.settings) self.skopeo_cli = Skopeo(dry_run) self.error = False identifier = instance['identifier'] account = instance['account'] region = instance.get('region') self.aws_cli = AWSApi(thread_pool_size=1, accounts=[self._get_aws_account_info(account)], settings=self.settings, init_ecr_auth_tokens=True) self.aws_cli.map_ecr_resources() self.ecr_uri = self._get_image_uri( account=account, repository=identifier, ) if self.ecr_uri is None: self.error = True LOG.error(f"Could not find the ECR repository {identifier}") self.ecr_username, self.ecr_password = self._get_ecr_creds( account=account, region=region, ) self.ecr_auth = f'{self.ecr_username}:{self.ecr_password}' self.image_username = None self.image_password = None self.image_auth = None pull_secret = self.instance['mirror']['pullCredentials'] if pull_secret is not None: raw_data = self.secret_reader.read_all(pull_secret) self.image_username = raw_data["user"] self.image_password = raw_data["token"] self.image_auth = f'{self.image_username}:{self.image_password}' def run(self): if self.error: return ecr_mirror = Image(self.ecr_uri, username=self.ecr_username, password=self.ecr_password) image = Image(self.instance['mirror']['url'], username=self.image_username, password=self.image_password) LOG.debug('[checking %s -> %s]', image, ecr_mirror) for tag in image: if tag not in ecr_mirror: try: self.skopeo_cli.copy(src_image=image[tag], src_creds=self.image_auth, dst_image=ecr_mirror[tag], dest_creds=self.ecr_auth) except SkopeoCmdError as details: LOG.error('[%s]', details) def _get_ecr_creds(self, account, region): if region is None: region = self.aws_cli.accounts[account]['resourcesDefaultRegion'] auth_token = f'{account}/{region}' data = self.aws_cli.auth_tokens[auth_token] auth_data = data['authorizationData'][0] token = auth_data['authorizationToken'] password = base64.b64decode(token).decode('utf-8').split(':')[1] return 'AWS', password def _get_image_uri(self, account, repository): for repo in self.aws_cli.resources[account]['ecr']: if repo['repositoryName'] == repository: return repo['repositoryUri'] @staticmethod def _get_aws_account_info(account): for account_info in queries.get_aws_accounts(): if 'name' not in account_info: continue if account_info['name'] != account: continue return account_info