def retry_if_fails(command, max_retries=15): if max_retries < 0: return try: kubectl.check_output(command) except: time.sleep(5) retry_if_fails(command, max_retries=max_retries - 1)
def zk_set_url_scheme(scheme='http'): pod_name = kubectl.get('pods', '-l', 'app=provider-solr-solrcloud-zk', required=True)['items'][0]['metadata']['name'] kubectl.check_output( 'exec %s zkCli.sh set /clusterprops.json \'{"urlScheme":"%s"}\'' % (pod_name, scheme))
def zk_set_url_scheme(scheme='http', timeout=300): pod_name = kubectl.get('pods', '-l', 'app=provider-solr-solrcloud-zk', required=True)['items'][0]['metadata']['name'] try: kubectl.check_output('exec %s zkCli.sh set /clusterprops.json \'{"urlScheme":"%s"}\'' % (pod_name, scheme)) except Exception as e: print('Failed to connect ZooKeeper, retrying in 60 seconds') time.sleep(60) if timeout < 0: raise e zk_set_url_scheme(scheme=scheme, timeout=timeout-60)
def zk_list_configs(): pod_name = kubectl.get('pods', '-l', 'app=provider-solr-solrcloud-zk', required=True)['items'][0]['metadata']['name'] lines = list(kubectl.check_output(f'exec {pod_name} zkCli.sh ls /configs').decode().splitlines())[5:] if len(lines) == 1: return [name.strip() for name in lines[0][1:-1].split(',')] else: return []
def zk_get_config_file(config_name, config_file, output_filename): path = f'/configs/{config_name}{config_file}' # print(f'path={path}') pod_name = kubectl.get('pods', '-l', 'app=provider-solr-solrcloud-zk', required=True)['items'][0]['metadata']['name'] lines = list(kubectl.check_output(f'exec {pod_name} zkCli.sh get {path} 2>/dev/null').decode().splitlines())[5:] assert len(lines) > 0 os.makedirs(os.path.dirname(output_filename), exist_ok=True) with open(output_filename, 'w') as f: f.write('\n'.join(lines))
def reload(): deployment_app = _get_resource_labels(for_deployment=True)['app'] logs.info('Reloading pgbouncers...') for pod_name in kubectl.check_output( f'get pods -l app={deployment_app} --output=custom-columns=name:.metadata.name --no-headers' ).decode().splitlines(): kubectl.check_call( f'exec {pod_name} -- pgbouncer -q -u pgbouncer -d -R /var/local/pgbouncer/pgbouncer.ini' ) logs.info(f'{pod_name}: PgBouncer Reloaded')
def zk_list_config_files(config_name, config_files, base_path=''): path = f'/configs/{config_name}{base_path}' # print(f'path={path}') pod_name = kubectl.get('pods', '-l', 'app=provider-solr-solrcloud-zk', required=True)['items'][0]['metadata']['name'] lines = list(kubectl.check_output(f'exec {pod_name} zkCli.sh ls {path}').decode().splitlines())[5:] # print(f'lines={lines}') assert len(lines) == 1 num_files = 0 for name in lines[0][1:-1].split(','): name = name.strip() if not name: continue # print(f'name={name}') if zk_list_config_files(config_name, config_files, base_path=f'{base_path}/{name}') == 0: config_files.append(f'{base_path}/{name}') num_files += 1 return num_files
def setup_autoscaler(expander='random', min_nodes=1, max_nodes=10, zone='', node_pool='default-pool'): from ckan_cloud_operator.drivers.helm import driver as helm_driver from ckan_cloud_operator.providers import manager as providers_manager cloud_provider = get_provider_id() cluster_name = kubectl.check_output( 'config current-context').decode().replace('\n', '') if cloud_provider == 'gcloud': """GKE has built-in autoscaler""" from ckan_cloud_operator import gcloud gcloud.check_call( f'container clusters update {cluster_name} --enable-autoscaling --min-nodes {min_nodes} --max-nodes {max_nodes} --zone {zone} --node_pool {node_pool}' ) return values = { 'image.tag': 'v1.13.1', 'autoDiscovery.clusterName': cluster_name, 'extraArgs.balance-similar-node-groups': 'false', 'extraArgs.expander': expander, 'cloudProvider': cloud_provider, 'rbac.create': 'true' } if values['cloudProvider'] == 'aws': zone = providers_manager.get_provider('cluster').get_info().get('zone') while not zone: zone = input('Enter the AWS cluster region: ') values['awsRegion'] = zone.strip() helm_driver.deploy( tiller_namespace='kube-system', chart_repo='https://kubernetes-charts.storage.googleapis.com', chart_name='stable/cluster-autoscaler', chart_version='', release_name='cluster-autoscaler', namespace='kube-system', chart_repo_name='stable', values=values)
def get_node_names(): return kubectl.check_output('get nodes -o custom-columns=name:.metadata.name --no-headers').decode().splitlines()