def retry_if_fails(command, max_retries=15):
if max_retries < 0:
return
try:
kubectl.check_output(command)
except:
time.sleep(5)
retry_if_fails(command, max_retries=max_retries - 1)
def zk_set_url_scheme(scheme='http'):
pod_name = kubectl.get('pods',
'-l',
'app=provider-solr-solrcloud-zk',
required=True)['items'][0]['metadata']['name']
kubectl.check_output(
'exec %s zkCli.sh set /clusterprops.json \'{"urlScheme":"%s"}\'' %
(pod_name, scheme))
def zk_set_url_scheme(scheme='http', timeout=300):
pod_name = kubectl.get('pods', '-l', 'app=provider-solr-solrcloud-zk', required=True)['items'][0]['metadata']['name']
try:
kubectl.check_output('exec %s zkCli.sh set /clusterprops.json \'{"urlScheme":"%s"}\'' % (pod_name, scheme))
except Exception as e:
print('Failed to connect ZooKeeper, retrying in 60 seconds')
time.sleep(60)
if timeout < 0:
raise e
zk_set_url_scheme(scheme=scheme, timeout=timeout-60)
def zk_list_configs():
pod_name = kubectl.get('pods', '-l', 'app=provider-solr-solrcloud-zk', required=True)['items'][0]['metadata']['name']
lines = list(kubectl.check_output(f'exec {pod_name} zkCli.sh ls /configs').decode().splitlines())[5:]
if len(lines) == 1:
return [name.strip() for name in lines[0][1:-1].split(',')]
else:
return []
def zk_get_config_file(config_name, config_file, output_filename):
path = f'/configs/{config_name}{config_file}'
# print(f'path={path}')
pod_name = kubectl.get('pods', '-l', 'app=provider-solr-solrcloud-zk', required=True)['items'][0]['metadata']['name']
lines = list(kubectl.check_output(f'exec {pod_name} zkCli.sh get {path} 2>/dev/null').decode().splitlines())[5:]
assert len(lines) > 0
os.makedirs(os.path.dirname(output_filename), exist_ok=True)
with open(output_filename, 'w') as f:
f.write('\n'.join(lines))
def reload():
deployment_app = _get_resource_labels(for_deployment=True)['app']
logs.info('Reloading pgbouncers...')
for pod_name in kubectl.check_output(
f'get pods -l app={deployment_app} --output=custom-columns=name:.metadata.name --no-headers'
).decode().splitlines():
kubectl.check_call(
f'exec {pod_name} -- pgbouncer -q -u pgbouncer -d -R /var/local/pgbouncer/pgbouncer.ini'
)
logs.info(f'{pod_name}: PgBouncer Reloaded')
def zk_list_config_files(config_name, config_files, base_path=''):
path = f'/configs/{config_name}{base_path}'
# print(f'path={path}')
pod_name = kubectl.get('pods', '-l', 'app=provider-solr-solrcloud-zk', required=True)['items'][0]['metadata']['name']
lines = list(kubectl.check_output(f'exec {pod_name} zkCli.sh ls {path}').decode().splitlines())[5:]
# print(f'lines={lines}')
assert len(lines) == 1
num_files = 0
for name in lines[0][1:-1].split(','):
name = name.strip()
if not name: continue
# print(f'name={name}')
if zk_list_config_files(config_name, config_files, base_path=f'{base_path}/{name}') == 0:
config_files.append(f'{base_path}/{name}')
num_files += 1
return num_files
def setup_autoscaler(expander='random',
min_nodes=1,
max_nodes=10,
zone='',
node_pool='default-pool'):
from ckan_cloud_operator.drivers.helm import driver as helm_driver
from ckan_cloud_operator.providers import manager as providers_manager
cloud_provider = get_provider_id()
cluster_name = kubectl.check_output(
'config current-context').decode().replace('\n', '')
if cloud_provider == 'gcloud':
"""GKE has built-in autoscaler"""
from ckan_cloud_operator import gcloud
gcloud.check_call(
f'container clusters update {cluster_name} --enable-autoscaling --min-nodes {min_nodes} --max-nodes {max_nodes} --zone {zone} --node_pool {node_pool}'
)
return
values = {
'image.tag': 'v1.13.1',
'autoDiscovery.clusterName': cluster_name,
'extraArgs.balance-similar-node-groups': 'false',
'extraArgs.expander': expander,
'cloudProvider': cloud_provider,
'rbac.create': 'true'
}
if values['cloudProvider'] == 'aws':
zone = providers_manager.get_provider('cluster').get_info().get('zone')
while not zone:
zone = input('Enter the AWS cluster region: ')
values['awsRegion'] = zone.strip()
helm_driver.deploy(
tiller_namespace='kube-system',
chart_repo='https://kubernetes-charts.storage.googleapis.com',
chart_name='stable/cluster-autoscaler',
chart_version='',
release_name='cluster-autoscaler',
namespace='kube-system',
chart_repo_name='stable',
values=values)
def get_node_names():
return kubectl.check_output('get nodes -o custom-columns=name:.metadata.name --no-headers').decode().splitlines()
