def initialize(default_provider=None, interactive=False):
crds_manager.install_crd(CRD_SINGULAR, CRD_PLURAL, CRD_KIND)
if not default_provider:
default_provider = 'gcloud' if cluster_manager.get_provider_id(
) == 'gcloud' else 'rancher'
providers_manager.get_provider(
users_provider_submodule, required=False,
default=default_provider).initialize(interactive=interactive)
def initialize(log_kwargs=None, interactive=False, default_cluster_provider=None, skip_to=None):
if interactive and not skip_to:
logs.info('Starting interactive initialization of the operator on the following cluster:')
print_info(minimal=True)
input('Verify your are connected to the right cluster and press <RETURN> to continue')
logs.info(f'Creating operator namespace: {OPERATOR_NAMESPACE}', **(log_kwargs or {}))
subprocess.call(f'kubectl create ns {OPERATOR_NAMESPACE}', shell=True)
assert default_cluster_provider in ['gcloud', 'aws'], f'invalid cluster provider: {default_cluster_provider}'
from ckan_cloud_operator.labels import manager as labels_manager
from ckan_cloud_operator.crds import manager as crds_manager
from ckan_cloud_operator.providers.db import manager as db_manager
from ckan_cloud_operator.providers.ckan import manager as ckan_manager
from ckan_cloud_operator.providers.routers import manager as routers_manager
from ckan_cloud_operator.providers.solr import manager as solr_manager
from ckan_cloud_operator.providers.storage import manager as storage_manager
for component, func in (
('labels', lambda lk: labels_manager.initialize(log_kwargs=lk)),
('cluster', lambda lk: providers_manager.get_provider('cluster', default=default_cluster_provider).initialize(interactive=interactive)),
('crds', lambda lk: crds_manager.initialize(log_kwargs=lk)),
('db', lambda lk: db_manager.initialize(log_kwargs=lk, interactive=interactive, default_cluster_provider=default_cluster_provider)),
('routers', lambda lk: routers_manager.initialize(interactive=interactive)),
('solr', lambda lk: solr_manager.initialize(interactive=interactive)),
('storage', lambda lk: storage_manager.initialize(interactive=interactive)),
('ckan', lambda lk: ckan_manager.initialize(interactive=interactive)),
):
if not skip_to or skip_to == component:
skip_to = None
log_kwargs = {'cluster-init': component}
logs.info(f'Initializing', **log_kwargs)
func(log_kwargs)
def print_info(debug=False, minimal=False):
print(
yaml.dump([
dict(get_kubeconfig_info(),
nodes=get_node_names(),
operator_version=get_operator_version(verify=True))
],
default_flow_style=False))
if not minimal:
from ckan_cloud_operator.providers import manager as providers_manager
print(
yaml.dump([
providers_manager.get_provider('cluster').get_info(debug=debug)
],
default_flow_style=False))
if debug:
assert all([
subprocess.call(f'kubectl cluster-info', shell=True) == 0,
subprocess.call(
f'kubectl config get-contexts $(kubectl config current-context)',
shell=True) == 0,
subprocess.call(f'kubectl get nodes', shell=True) == 0
]), 'failed to get all debug info'
else:
return True
def initialize(log_kwargs=None, interactive=False):
"""Initialize / upgrade the db module and sub-modules"""
log_kwargs = log_kwargs or {}
logs.info(f'Initializing DB provider', **log_kwargs)
db_provider = providers_manager.get_provider(
db_provider_submodule, default=db_gcloudsql_provider_id)
db_provider.initialize(interactive=interactive)
if db_provider.is_private_ip():
logs.info('DB Uses a private ip, initializing the DB proxy')
db_proxy_manager.initialize()
def setup_autoscaler(expander='random',
min_nodes=1,
max_nodes=10,
zone='',
node_pool='default-pool'):
from ckan_cloud_operator.drivers.helm import driver as helm_driver
from ckan_cloud_operator.providers import manager as providers_manager
cloud_provider = get_provider_id()
cluster_name = kubectl.check_output(
'config current-context').decode().replace('\n', '')
if cloud_provider == 'gcloud':
"""GKE has built-in autoscaler"""
from ckan_cloud_operator import gcloud
gcloud.check_call(
f'container clusters update {cluster_name} --enable-autoscaling --min-nodes {min_nodes} --max-nodes {max_nodes} --zone {zone} --node_pool {node_pool}'
)
return
values = {
'image.tag': 'v1.13.1',
'autoDiscovery.clusterName': cluster_name,
'extraArgs.balance-similar-node-groups': 'false',
'extraArgs.expander': expander,
'cloudProvider': cloud_provider,
'rbac.create': 'true'
}
if values['cloudProvider'] == 'aws':
zone = providers_manager.get_provider('cluster').get_info().get('zone')
while not zone:
zone = input('Enter the AWS cluster region: ')
values['awsRegion'] = zone.strip()
helm_driver.deploy(
tiller_namespace='kube-system',
chart_repo='https://kubernetes-charts.storage.googleapis.com',
chart_name='stable/cluster-autoscaler',
chart_version='',
release_name='cluster-autoscaler',
namespace='kube-system',
chart_repo_name='stable',
values=values)
def initialize(log_kwargs=None,
interactive=False,
default_cluster_provider=None):
"""Initialize / upgrade the db module and sub-modules"""
if default_cluster_provider == 'aws':
default_provider = db_rds_provider_id
elif not default_cluster_provider or default_cluster_provider == 'gcloud':
default_provider = db_gcloudsql_provider_id
else:
raise NotImplementedError(
f'Unknown provider: {default_cluster_provider}')
log_kwargs = log_kwargs or {}
logs.info(f'Initializing DB provider', **log_kwargs)
db_provider = providers_manager.get_provider(db_provider_submodule,
default=default_provider)
db_provider.initialize(interactive=interactive)
if db_provider.is_private_ip():
logs.info('DB Uses a private ip, initializing the DB proxy')
db_proxy_manager.initialize()
def initialize(log_kwargs=None, interactive=False, default_cluster_provider=None, skip_to=None):
if interactive:
logs.info('Starting interactive initialization of the operator on the following cluster:')
print_info(minimal=True)
if sys.stdout.isatty():
input('Verify you are connected to the right cluster and press <RETURN> to continue')
if not skip_to:
logs.info(f'Creating operator namespace: {OPERATOR_NAMESPACE}', **(log_kwargs or {}))
subprocess.call(f'kubectl create ns {OPERATOR_NAMESPACE}', shell=True)
assert default_cluster_provider in ['gcloud', 'aws', 'azure', 'minikube'], f'invalid cluster provider: {default_cluster_provider}'
subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} create secret generic ckan-cloud-provider-cluster-{default_cluster_provider}', shell=True)
subprocess.call(f'kubectl -n {OPERATOR_NAMESPACE} create configmap operator-conf --from-literal=ckan-cloud-operator-image=viderum/ckan-cloud-operator:latest --from-literal=label-prefix={OPERATOR_NAMESPACE}', shell=True)
from ckan_cloud_operator.providers import manager as providers_manager
from ckan_cloud_operator.labels import manager as labels_manager
from ckan_cloud_operator.crds import manager as crds_manager
from ckan_cloud_operator.providers.db import manager as db_manager
from ckan_cloud_operator.providers.ckan import manager as ckan_manager
from ckan_cloud_operator.providers.routers import manager as routers_manager
from ckan_cloud_operator.providers.solr import manager as solr_manager
from ckan_cloud_operator.providers.storage import manager as storage_manager
from ckan_cloud_operator.providers.apps import manager as apps_manager
for component, func in (
('labels', lambda lk: labels_manager.initialize(log_kwargs=lk)),
('cluster', lambda lk: providers_manager.get_provider('cluster', default=default_cluster_provider).initialize(interactive=interactive)),
('crds', lambda lk: crds_manager.initialize(log_kwargs=lk)),
('db', lambda lk: db_manager.initialize(log_kwargs=lk, interactive=interactive, default_cluster_provider=default_cluster_provider)),
('routers', lambda lk: routers_manager.initialize(interactive=interactive)),
('solr', lambda lk: solr_manager.initialize(interactive=interactive)),
('storage', lambda lk: storage_manager.initialize(interactive=interactive)),
('ckan', lambda lk: ckan_manager.initialize(interactive=interactive)),
('apps', lambda lk: apps_manager.initialize(interactive=interactive)),
):
if not skip_to or skip_to == component:
skip_to = None
log_kwargs = {'cluster-init': component}
logs.info(f'Initializing', **log_kwargs)
func(log_kwargs)
def get_provider():
return providers_manager.get_provider('cluster')
def get_cluster_kubeconfig_spec():
return providers_manager.get_provider('cluster').get_cluster_kubeconfig_spec()
def get_cluster_name():
return providers_manager.get_provider('cluster').get_name()
def get_provider():
from ckan_cloud_operator.providers import manager as providers_manager
return providers_manager.get_provider('cluster')
def get_cluster_kubeconfig_spec():
from ckan_cloud_operator.providers import manager as providers_manager
return providers_manager.get_provider('cluster').get_cluster_kubeconfig_spec()
def update(name):
logs.info(f'Updating user: {name}')
providers_manager.get_provider(users_provider_submodule, verbose=True).update(get(name))
def get_provider():
return providers_manager.get_provider(db_provider_submodule)
def get_provider(default=None, verbose=False):
from ckan_cloud_operator.providers import manager as providers_manager
return providers_manager.get_provider(PROVIDER_SUBMODULE,
default=default,
verbose=verbose)
def get_provider(default=None, required=True):
return providers_manager.get_provider(PROVIDER_SUBMODULE,
default=default,
required=required)
def initialize(default_provider='gcloud'):
crds_manager.install_crd(CRD_SINGULAR, CRD_PLURAL, CRD_KIND)
providers_manager.get_provider(users_provider_submodule, required=False, default=default_provider).initialize()
def initialize_cluster(lk):
from ckan_cloud_operator.providers import manager as providers_manager
providers_manager.get_provider(
'cluster', default=default_cluster_provider).initialize(
interactive=interactive)
def get_provider(default=None, provider_id=None):
return providers_manager.get_provider(PROVIDER_SUBMODULE, default=default, provider_id=provider_id)
def get_provider(default=None, verbose=False):
return providers_manager.get_provider(PROVIDER_SUBMODULE,
default=default,
verbose=verbose)
def _get_provider(verbose=False):
return providers_manager.get_provider(users_provider_submodule, verbose=verbose)
def _gcloud():
return providers_manager.get_provider(cluster_provider_submodule)
