def main(args):
quiet = not args.verbose
print_topology_meta(args.topology)
if args.include_services and args.exclude_services:
raise ValueError(
'Cannot pass both --include-services and --exclude-services.')
image_prefix = '{}/{}/topology_hdp:hdp{}_ambari{}'.format(
args.registry, args.namespace or DEFAULT_NAMESPACE, args.hdp_version,
args.ambari_version)
primary_node_image = '{}_{}'.format(image_prefix, 'primary-node')
secondary_node_image = '{}_{}'.format(image_prefix, 'secondary-node')
clusterdock_config_host_dir = os.path.realpath(
os.path.expanduser(args.clusterdock_config_directory))
volumes = [{clusterdock_config_host_dir: CLUSTERDOCK_CLIENT_CONTAINER_DIR}]
primary_node = Node(hostname=args.primary_node[0],
group='primary',
volumes=volumes,
image=primary_node_image,
ports=[{
AMBARI_PORT: AMBARI_PORT
} if args.predictable else AMBARI_PORT])
secondary_nodes = [
Node(hostname=hostname,
group='secondary',
volumes=volumes,
image=secondary_node_image) for hostname in args.secondary_nodes
]
cluster = Cluster(primary_node, *secondary_nodes)
cluster.primary_node = primary_node
cluster.secondary_nodes = secondary_nodes
for node in cluster.nodes:
node.volumes.append({'/sys/fs/cgroup': '/sys/fs/cgroup'})
# do not use tempfile.mkdtemp, as systemd wont be able to bring services up when temp ends to be created in
# /var/tmp/ directory
node.volumes.append(['/run', '/run/lock'])
cluster.start(args.network)
hdp_version_tuple = version_tuple(args.hdp_version)
logger.debug('Starting PostgreSQL for Ambari server ...')
# Need this as init system in Docker misreports on postgres start initially
# Check https://github.com/docker-library/postgres/issues/146 for more
def condition():
primary_node.execute('service postgresql restart', quiet=quiet)
if '1 row' in primary_node.execute(
'PGPASSWORD=bigdata psql ambari '
'-U ambari -h localhost -c "select 1"',
quiet=quiet).output:
return True
wait_for_condition(condition=condition, time_between_checks=2)
def condition():
if 'running' in primary_node.execute('service postgresql status',
quiet=quiet).output:
return True
wait_for_condition(condition=condition)
time.sleep(
10
) # If images are set to start Ambari server/agents - give some time to recover the right status
_update_node_names(cluster, quiet=quiet)
# The HDP topology uses two pre-built images ('primary' and 'secondary'). If a cluster
# larger than 2 nodes is started, some modifications need to be done.
if len(secondary_nodes) > 1:
_remove_files(nodes=secondary_nodes[1:],
files=['/hadoop/hdfs/data/current/*'],
quiet=quiet)
logger.info('Starting Ambari server ...')
primary_node.execute('ambari-server start', quiet=quiet)
# Docker for Mac exposes ports that can be accessed only with ``localhost:<port>`` so
# use that instead of the hostname if the host name is ``moby``.
hostname = ('localhost'
if client.info().get('Name') == 'moby' else socket.getaddrinfo(
socket.gethostname(), 0, flags=socket.AI_CANONNAME)[0][3])
port = cluster.primary_node.host_ports.get(AMBARI_PORT)
server_url = 'http://{}:{}'.format(hostname, port)
logger.info('Ambari server is now reachable at %s', server_url)
logger.info('Starting Ambari agents ...')
for node in cluster:
logger.debug('Starting Ambari agent on %s ...', node.fqdn)
node.execute('ambari-agent start', quiet=quiet)
ambari = Ambari(server_url, username='******', password='******')
def condition(ambari, cluster):
cluster_hosts = {node.fqdn for node in cluster}
ambari_hosts = {host.host_name for host in ambari.hosts}
logger.debug('Cluster hosts: %s; Ambari hosts: %s', cluster_hosts,
ambari_hosts)
return cluster_hosts == ambari_hosts
wait_for_condition(condition=condition, condition_args=[ambari, cluster])
service_types_to_leave = (args.include_services.upper().split(',')
if args.include_services else [])
service_types_to_remove = (args.exclude_services.upper().split(',')
if args.exclude_services else [])
if service_types_to_leave or service_types_to_remove:
for service in list(ambari.clusters(DEFAULT_CLUSTER_NAME).services):
service_name = service.service_name.upper()
if (service_name in service_types_to_remove
or (service_types_to_leave
and service_name not in service_types_to_leave)):
logger.info('Removing cluster service (name = %s) ...',
service_name)
service.delete()
for node in secondary_nodes[1:]:
logger.info('Adding %s to cluster ...', node.fqdn)
ambari.clusters(DEFAULT_CLUSTER_NAME).hosts.create(node.fqdn)
secondary_node = ambari.clusters(DEFAULT_CLUSTER_NAME).hosts(
secondary_nodes[0].fqdn)
for component in secondary_node.components:
logger.debug('Adding component (%s) to cluster on host (%s) ...',
component.component_name, node.fqdn)
host_components = ambari.clusters(DEFAULT_CLUSTER_NAME).hosts(
node.fqdn).components
host_components.create(component.component_name).wait()
logger.debug('Installing all registered components on host (%s) ...',
node.fqdn)
ambari.clusters(DEFAULT_CLUSTER_NAME).hosts(
node.fqdn).components.install().wait()
logger.info('Waiting for all hosts to reach healthy state ...')
def condition(ambari):
health_report = ambari.clusters(DEFAULT_CLUSTER_NAME).health_report
logger.debug('Ambari cluster health report: %s ...', health_report)
return health_report.get('Host/host_state/HEALTHY') == len(
list(ambari.hosts))
wait_for_condition(condition=condition, condition_args=[ambari])
service_names = [
service['service_name'] for service in ambari.clusters(
DEFAULT_CLUSTER_NAME).services.to_dict()
]
if 'ATLAS' in service_names:
logger.info('Configuring Atlas required properties ...')
_configure_atlas(ambari,
args.hdp_version,
atlas_server_host=cluster.primary_node.fqdn)
if 'HIVE' in service_names:
primary_node.execute('touch /etc/hive/sys.db.created', quiet=quiet)
logger.info('Waiting for components to be ready ...')
def condition(ambari):
comps = ambari.clusters(
DEFAULT_CLUSTER_NAME).cluster.host_components.refresh()
for comp in comps:
if comp.state.upper() == 'UNKNOWN':
logger.debug('Not ready with component `%s` ...',
comp.component_name)
return False
else:
return True
wait_for_condition(condition=condition, condition_args=[ambari])
if not args.dont_start_cluster:
logger.info('Starting cluster services ...')
ambari.clusters(DEFAULT_CLUSTER_NAME).services.start().wait(
timeout=3600)
if 'HBASE' in service_names:
logger.info('Starting Thrift server ...')
if hdp_version_tuple <= (2, 0, 13, 0):
hbase_daemon_path = '/usr/lib/hbase/bin/hbase-daemon.sh'
else:
hbase_daemon_path = '/usr/hdp/current/hbase-master/bin/hbase-daemon.sh'
primary_node.execute('{} start thrift -p {} '
'--infoport {}'.format(
hbase_daemon_path,
HBASE_THRIFT_SERVER_PORT,
HBASE_THRIFT_SERVER_INFO_PORT),
quiet=quiet)
def main(args):
primary_node_image = "{0}/{1}/{2}:cdh-cm-primary-{3}".format(
args.registry, args.clusterdock_namespace, args.image_name,
args.version_string)
secondary_node_image = "{0}/{1}/{2}:cdh-cm-secondary-{3}".format(
args.registry, args.clusterdock_namespace, args.image_name,
args.version_string)
edge_node_image = "{0}/{1}/{2}:cdh-cm-edge-{3}".format(
args.registry, args.clusterdock_namespace, args.image_name,
args.version_string)
# Docker's API for healthcheck uses units of nanoseconds. Define a constant
# to make this more readable.
SECONDS = 1000000000
cm_server_healthcheck = {
'test':
'curl --silent --output /dev/null 127.0.0.1:{}'.format(CM_PORT),
'interval': 1 * SECONDS,
'timeout': 1 * SECONDS,
'retries': 1,
'start_period': 30 * SECONDS
}
primary_node = Node(hostname=args.primary_node[0],
group='primary',
image=primary_node_image,
ports=[{
CM_PORT: CM_PORT
}],
healthcheck=cm_server_healthcheck)
secondary_nodes = [
Node(hostname=hostname, group='secondary', image=secondary_node_image)
for hostname in args.secondary_nodes
]
edge_nodes = [
Node(hostname=hostname, group='edge', image=edge_node_image)
for hostname in args.edge_nodes
]
all_nodes = [primary_node] + secondary_nodes + edge_nodes
cluster = Cluster(*all_nodes)
cluster.primary_node = primary_node
secondary_node_group = NodeGroup(secondary_nodes)
edge_node_group = NodeGroup(edge_nodes)
cluster.start(args.network)
filesystem_fix_commands = [
'cp {0} {0}.1; umount {0}; mv -f {0}.1 {0}'.format(file_) for file_ in
['/etc/hosts', '/etc/resolv.conf', '/etc/hostname', '/etc/localtime']
]
cluster.execute("bash -c '{}'".format('; '.join(filesystem_fix_commands)))
# Use BSD tar instead of tar because it works bether with docker
cluster.execute("ln -fs /usr/bin/bsdtar /bin/tar")
_configure_cm_agents(cluster)
if args.change_hostfile:
update_hosts_file(cluster)
# The CDH topology uses two pre-built images ('primary' and 'secondary'). If a cluster
# larger than 2 nodes is started, some modifications need to be done to the nodes to
# prevent duplicate heartbeats and things like that.
if len(secondary_nodes) > 1:
_remove_files(
nodes=secondary_nodes[1:],
files=['/var/lib/cloudera-scm-agent/uuid', '/dfs*/dn/current/*'])
logger.info('Configuring Kerberos...')
cluster.primary_node.execute('/root/configure-kerberos.sh', quiet=True)
cluster.primary_node.execute('service krb5kdc start', quiet=True)
cluster.primary_node.execute('service kadmin start', quiet=True)
logger.info('Restarting Cloudera Manager agents ...')
# _restart_cm_agents(cluster)
logger.info('Waiting for Cloudera Manager server to come online ...')
_wait_for_cm_server(primary_node)
# Docker for Mac exposes ports that can be accessed only with ``localhost:<port>`` so
# use that instead of the hostname if the host name is ``moby``.
hostname = 'localhost' if client.info().get(
'Name') == 'moby' else socket.gethostname()
port = primary_node.host_ports.get(CM_PORT)
server_url = 'http://{}:{}'.format(hostname, port)
logger.info('Cloudera Manager server is now reachable at %s', server_url)
# The work we need to do through CM itself begins here...
deployment = ClouderaManagerDeployment(server_url)
deployment.stop_cm_service()
time.sleep(10)
logger.info('Starting krb5kdc and kadmin ...')
cluster.primary_node.execute('service krb5kdc start', quiet=True)
cluster.primary_node.execute('service kadmin start', quiet=True)
logger.info("Regenerating keytabs...")
regenerate_keytabs(cluster, primary_node, deployment)
logger.info("Adding hosts to cluster ...")
# Add all CM hosts to the cluster (i.e. only new hosts that weren't part of the original
# images).
all_host_ids = {}
for host in deployment.get_all_hosts():
all_host_ids[host['hostId']] = host['hostname']
for node in cluster:
if node.fqdn == host['hostname']:
node.host_id = host['hostId']
break
else:
raise Exception('Could not find CM host with hostname {}.'.format(
node.fqdn))
cluster_host_ids = {
host['hostId']
for host in deployment.get_cluster_hosts(
cluster_name=DEFAULT_CLUSTER_NAME)
}
host_ids_to_add = set(all_host_ids.keys()) - cluster_host_ids
if host_ids_to_add:
logger.debug(
'Adding %s to cluster %s ...', 'host{} ({})'.format(
's' if len(host_ids_to_add) > 1 else '',
', '.join(all_host_ids[host_id]
for host_id in host_ids_to_add)),
DEFAULT_CLUSTER_NAME)
deployment.add_cluster_hosts(cluster_name=DEFAULT_CLUSTER_NAME,
host_ids=host_ids_to_add)
_wait_for_activated_cdh_parcel(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME)
# create and Apply host templates
deployment.create_host_template(cluster_name='cluster',
host_template_name='secondary',
role_config_group_names=[
'hdfs-DATANODE-BASE',
'hbase-REGIONSERVER-BASE',
'yarn-NODEMANAGER-BASE'
])
deployment.create_host_template(cluster_name='cluster',
host_template_name='edgenode',
role_config_group_names=[
'hive-GATEWAY-BASE',
'hbase-GATEWAY-BASE',
'hdfs-GATEWAY-BASE',
'spark_on_yarn-GATEWAY-BASE'
])
deployment.apply_host_template(cluster_name=DEFAULT_CLUSTER_NAME,
host_template_name='secondary',
start_roles=False,
host_ids=host_ids_to_add)
deployment.apply_host_template(cluster_name=DEFAULT_CLUSTER_NAME,
host_template_name='edgenode',
start_roles=False,
host_ids=host_ids_to_add)
logger.info('Updating database configurations ...')
_update_database_configs(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
primary_node=primary_node)
# deployment.update_database_configs()
# deployment.update_hive_metastore_namenodes()
logger.info("Update KDC Config ")
deployment.update_cm_config({
'SECURITY_REALM': 'CLOUDERA',
'KDC_HOST': 'node-1.cluster',
'KRB_MANAGE_KRB5_CONF': 'true'
})
deployment.update_service_config(
service_name='hbase',
cluster_name=DEFAULT_CLUSTER_NAME,
configs={'hbase_superuser': '******'})
deployment.update_service_role_config_group_config(
service_name='hive',
cluster_name=DEFAULT_CLUSTER_NAME,
role_config_group_name='hive-HIVESERVER2-BASE',
configs={'hiveserver2_webui_port': '10009'})
logger.info("Importing Credentials..")
cluster.primary_node.execute(
"curl -XPOST -u admin:admin http://{0}:{1}/api/v14/cm/commands/importAdminCredentials?username=cloudera-scm/admin@CLOUDERA&password=cloudera"
.format(primary_node.fqdn, CM_PORT),
quiet=True)
logger.info("deploy cluster client config ...")
deployment.deploy_cluster_client_config(cluster_name=DEFAULT_CLUSTER_NAME)
logger.info("Configure for kerberos ...")
cluster.primary_node.execute(
"curl -XPOST -u admin:admin http://{0}:{1}/api/v14/cm/commands/configureForKerberos --data 'clustername={2}'"
.format(primary_node.fqdn, CM_PORT, DEFAULT_CLUSTER_NAME),
quiet=True)
logger.info("Creating keytab files ...")
cluster.execute('/root/create-keytab.sh', quiet=True)
logger.info('Deploying client config ...')
_deploy_client_config(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME)
if not args.dont_start_cluster:
logger.info('Starting cluster services ...')
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="zookeeper",
command="start")
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="hdfs",
command="start")
if not args.skip_accumulo:
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="accumulo16",
command="CreateHdfsDirCommand")
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="accumulo16",
command="CreateAccumuloUserDirCommand")
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="accumulo16",
command="AccumuloInitServiceCommand")
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="accumulo16",
command="start")
if not args.skip_yarn:
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="yarn",
command="start")
if not args.skip_hbase:
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="hbase",
command="start")
if not args.skip_flume:
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="flume",
command="start")
if not args.skip_spark:
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="spark_on_yarn",
command="start")
if not args.skip_sqoop:
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="sqoop",
command="start")
if not args.skip_hive:
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="hive",
command="start")
if not args.skip_oozie:
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="oozie",
command="start")
if not args.skip_hue:
_start_service_command(deployment=deployment,
cluster_name=DEFAULT_CLUSTER_NAME,
service_name="hue",
command="start")
logger.info('Starting CM services ...')
_start_cm_service(deployment=deployment)
logger.info("Setting up HDFS Homedir ...")
cluster.primary_node.execute(
"kinit -kt /var/run/cloudera-scm-agent/process/*-hdfs-NAMENODE/hdfs.keytab hdfs/node-1.cluster@CLOUDERA",
quiet=True)
cluster.primary_node.execute("hadoop fs -mkdir /user/cloudera-scm",
quiet=True)
cluster.primary_node.execute(
"hadoop fs -chown cloudera-scm:cloudera-scm /user/cloudera-scm",
quiet=True)
logger.info("Kinit cloudera-scm/admin ...")
cluster.execute('kinit -kt /root/cloudera-scm.keytab cloudera-scm/admin',
quiet=True)
logger.info("Executing post run script ...")
secondary_node_group.execute("/root/post_run.sh")
edge_node_group.execute("/root/post_run.sh")
def main(args):
if args.license_url and not args.license_credentials:
raise Exception(
'--license-credentials is a required argument if --license-url is provided.'
)
image_prefix = '{}/{}/clusterdock:mapr{}'.format(
args.registry, args.namespace or DEFAULT_NAMESPACE, args.mapr_version)
if args.mep_version:
image_prefix = '{}_mep{}'.format(image_prefix, args.mep_version)
primary_node_image = '{}_{}'.format(image_prefix, 'primary-node')
secondary_node_image = '{}_{}'.format(image_prefix, 'secondary-node')
node_disks = yaml.load(args.node_disks)
# MapR-FS needs each fileserver node to have a disk allocated for it, so fail fast if the
# node disks map is missing any nodes.
if set(args.primary_node + args.secondary_nodes) != set(node_disks):
raise Exception(
'Not all nodes are accounted for in the --node-disks dictionary')
primary_node = Node(
hostname=args.primary_node[0],
group='primary',
image=primary_node_image,
ports=[{
MCS_SERVER_PORT: MCS_SERVER_PORT
} if args.predictable else MCS_SERVER_PORT],
devices=node_disks.get(args.primary_node[0]),
# Secure cluster needs the ticket to execute rest of commands
# after cluster start.
environment=['MAPR_TICKETFILE_LOCATION=/opt/mapr/conf/mapruserticket']
if args.secure else [])
secondary_nodes = [
Node(hostname=hostname,
group='secondary',
image=secondary_node_image,
devices=node_disks.get(hostname))
for hostname in args.secondary_nodes
]
cluster = Cluster(primary_node, *secondary_nodes)
if args.secure:
secure_config_host_dir = os.path.expanduser(
args.secure_config_directory)
volumes = [{secure_config_host_dir: SECURE_CONFIG_CONTAINER_DIR}]
for node in cluster.nodes:
node.volumes.extend(volumes)
# MapR versions 6.0.0 onwards use CentOS 7 which needs following settings.
mapr_version_tuple = tuple(int(i) for i in args.mapr_version.split('.'))
if mapr_version_tuple >= EARLIEST_MAPR_VERSION_WITH_LICENSE_AND_CENTOS_7:
for node in cluster.nodes:
node.volumes.append({'/sys/fs/cgroup': '/sys/fs/cgroup'})
temp_dir_name = tempfile.mkdtemp()
logger.debug('Created temporary directory %s', temp_dir_name)
node.volumes.append({temp_dir_name: '/run'})
cluster.primary_node = primary_node
cluster.start(args.network, pull_images=args.always_pull)
logger.info('Generating new UUIDs ...')
cluster.execute('/opt/mapr/server/mruuidgen > /opt/mapr/hostid')
if not args.secure:
logger.info('Configuring the cluster ...')
for node in cluster:
configure_command = (
'/opt/mapr/server/configure.sh -C {0} -Z {0} -RM {0} -HS {0} '
'-u mapr -g mapr -D {1}'.format(
primary_node.fqdn,
','.join(node_disks.get(node.hostname))))
node.execute("bash -c '{}'".format(configure_command))
else:
logger.info('Configuring native security for the cluster ...')
configure_command = (
'/opt/mapr/server/configure.sh -secure -genkeys -C {0} -Z {0} -RM {0} -HS {0} '
'-u mapr -g mapr -D {1}'.format(
primary_node.fqdn,
','.join(node_disks.get(primary_node.hostname))))
source_files = [
'{}/{}'.format(MAPR_CONFIG_DIR, file) for file in SECURE_FILES
]
commands = [
configure_command,
'chmod 600 {}/{}'.format(MAPR_CONFIG_DIR, SSL_KEYSTORE_FILE),
'cp -f {src} {dest_dir}'.format(
src=' '.join(source_files),
dest_dir=SECURE_CONFIG_CONTAINER_DIR)
]
primary_node.execute(' && '.join(commands))
for node in secondary_nodes:
source_files = [
'{}/{}'.format(SECURE_CONFIG_CONTAINER_DIR, file)
for file in SECURE_FILES
]
configure_command = (
'/opt/mapr/server/configure.sh -secure -C {0} -Z {0} -RM {0} -HS {0} '
'-u mapr -g mapr -D {1}'.format(
primary_node.fqdn,
','.join(node_disks.get(node.hostname))))
commands = [
'cp -f {src} {dest_dir}'.format(src=' '.join(source_files),
dest_dir=MAPR_CONFIG_DIR),
configure_command
]
node.execute(' && '.join(commands))
logger.info('Waiting for MapR Control System server to come online ...')
def condition(address, port):
return socket().connect_ex((address, port)) == 0
def success(time):
logger.info('MapR Control System server is online after %s seconds.',
time)
def failure(timeout):
raise TimeoutError(
'Timed out after {} seconds waiting '
'for MapR Control System server to come online.'.format(timeout))
wait_for_condition(
condition=condition,
condition_args=[primary_node.ip_address, MCS_SERVER_PORT],
time_between_checks=3,
timeout=180,
success=success,
failure=failure)
mcs_server_host_port = primary_node.host_ports.get(MCS_SERVER_PORT)
logger.info('Creating /apps/spark directory on %s ...',
primary_node.hostname)
spark_directory_command = [
'hadoop fs -mkdir -p /apps/spark', 'hadoop fs -chmod 777 /apps/spark'
]
primary_node.execute("bash -c '{}'".format(
'; '.join(spark_directory_command)))
logger.info('Creating MapR sample Stream named /sample-stream on %s ...',
primary_node.hostname)
primary_node.execute('maprcli stream create -path /sample-stream '
'-produceperm p -consumeperm p -topicperm p')
if mapr_version_tuple >= EARLIEST_MAPR_VERSION_WITH_LICENSE_AND_CENTOS_7 and args.license_url:
license_commands = [
'curl --user {} {} > /tmp/lic'.format(args.license_credentials,
args.license_url),
'/opt/mapr/bin/maprcli license add -license /tmp/lic -is_file true',
'rm -rf /tmp/lic'
]
logger.info('Applying license ...')
primary_node.execute(' && '.join(license_commands))
if not args.dont_register_gateway:
logger.info('Registering gateway with the cluster ...')
register_gateway_commands = [
"cat /opt/mapr/conf/mapr-clusters.conf | egrep -o '^[^ ]* '"
' > /tmp/cluster-name',
'maprcli cluster gateway set -dstcluster $(cat '
'/tmp/cluster-name) -gateways {}'.format(primary_node.fqdn),
'rm /tmp/cluster-name'
]
primary_node.execute(' && '.join(register_gateway_commands))
logger.info(
'MapR Control System server is now accessible at https://%s:%s',
getfqdn(), mcs_server_host_port)
def main(args):
node_image = '{}/{}/clusterdock:greenplum{}'.format(
args.registry, args.namespace or DEFAULT_NAMESPACE,
args.greenplum_version)
volumes = [{'/sys/fs/cgroup': '/sys/fs/cgroup'}, {'/run': '/run/lock'}]
if args.predictable:
ports = [{
GREENPLUM_SQL_CLIENT_CONNECTION_PORT:
GREENPLUM_SQL_CLIENT_CONNECTION_PORT
}, {
GREENPLUM_SSH_HOST_PORT: GREENPLUM_SSH_CONTAINER_PORT
}, {
GREENPLUM_GPSS_LISTENER_PORT: GREENPLUM_GPSS_LISTENER_PORT
}, {
GREENPLUM_GPFDIST_SERVICE_PORT: GREENPLUM_GPFDIST_SERVICE_PORT
}]
else:
ports = [
GREENPLUM_SQL_CLIENT_CONNECTION_PORT, GREENPLUM_SSH_CONTAINER_PORT,
GREENPLUM_GPSS_LISTENER_PORT, GREENPLUM_GPFDIST_SERVICE_PORT
]
primary_node = Node(hostname=args.primary_node[0],
group='primary',
image=node_image,
name='greenplum_{}'.format(args.greenplum_version),
ports=ports,
volumes=volumes)
secondary_nodes = [
Node(hostname=hostname,
group='secondary',
image=node_image,
volumes=volumes) for hostname in args.secondary_nodes
]
nodes = [primary_node] + secondary_nodes
cluster = Cluster(*nodes)
cluster.primary_node = primary_node
cluster.start(args.network, pull_images=args.always_pull)
primary_node.put_file(HOST_FILE_PATH, '\n'.join(args.secondary_nodes))
primary_node.put_file(CONFIG_JSON_FILE_PATH, CONFIG_JSON)
commands = [
'source /usr/local/greenplum-db/greenplum_path.sh',
'chmod 755 /home/gpadmin/prepare.sh',
# Create segment hosts with 1 primary segment in each segment host.
'/home/gpadmin/prepare.sh -s {} -n 1'.format(len(args.secondary_nodes)
),
# Initialize Greenplum Database system using gpinitsystem_config file.
'gpinitsystem -a -c /home/gpadmin/gpinitsystem_config'
]
primary_node.execute(' && '.join(commands), user='******')
commands = [
'source /usr/local/greenplum-db/greenplum_path.sh',
# To allow access to Greenplum Database from every host, change pg_hba.conf file.
"echo 'host all all 0.0.0.0/0 trust' >> /home/gpadmin/master/gpseg-1/pg_hba.conf",
'export MASTER_DATA_DIRECTORY=/home/gpadmin/master/gpseg-1',
# Following will make sure the changes in pg_hba.conf take effect.
'/usr/local/greenplum-db/bin/gpstop -u',
'sudo ln -s /usr/local/greenplum-db-5.12.0/lib/libpq.so.5 /usr/lib64/libpq.so.5',
'sudo ln -s /usr/local/greenplum-db-5.12.0/lib/libssl.so.1.0.0 /usr/lib64/libssl.so.1.0.0',
'sudo ln -s /usr/local/greenplum-db-5.12.0/lib/libcrypto.so.1.0.0 /usr/lib64/libcrypto.so.1.0.0',
'sudo ln -s /usr/local/greenplum-db-5.12.0/lib/libcom_err.so.3 /usr/lib64/libcom_err.so.3',
# Create db and extension in it.
'/usr/local/greenplum-db/bin/createdb some_db',
"/usr/local/greenplum-db/bin/psql -d some_db -c 'CREATE EXTENSION gpss;'"
]
primary_node.execute(' && '.join(commands), user='******')
# Start Greenplum Stream Server in detached mode since it waits indefinitely for client job requests.
primary_node.execute(
'/usr/local/greenplum-db/bin/gpss /home/gpadmin/config.json',
user='******',
detach=True)
def main(args):
image_prefix = '{}/{}/topology_hdp:hdp{}_ambari{}'.format(
args.registry, args.namespace or DEFAULT_NAMESPACE, args.hdp_version,
args.ambari_version)
primary_node_image = '{}_{}'.format(image_prefix, 'primary-node')
secondary_node_image = '{}_{}'.format(image_prefix, 'secondary-node')
primary_node = Node(hostname=args.primary_node[0],
group='primary',
image=primary_node_image,
ports=[{
AMBARI_PORT: AMBARI_PORT
} if args.predictable else AMBARI_PORT])
secondary_nodes = [
Node(hostname=hostname, group='secondary', image=secondary_node_image)
for hostname in args.secondary_nodes
]
cluster = Cluster(primary_node, *secondary_nodes)
cluster.primary_node = primary_node
cluster.secondary_nodes = secondary_nodes
cluster.start(args.network)
logger.debug('Starting PostgreSQL for Ambari server ...')
primary_node.execute('service postgresql start', quiet=not args.verbose)
_update_node_names(cluster, quiet=not args.verbose)
# The HDP topology uses two pre-built images ('primary' and 'secondary'). If a cluster
# larger than 2 nodes is started, some modifications need to be done.
if len(secondary_nodes) > 1:
_remove_files(nodes=secondary_nodes[1:],
files=['/hadoop/hdfs/data/current/*'])
logger.info('Starting Ambari server ...')
primary_node.execute('ambari-server start', quiet=not args.verbose)
# Docker for Mac exposes ports that can be accessed only with ``localhost:<port>`` so
# use that instead of the hostname if the host name is ``moby``.
hostname = 'localhost' if client.info().get(
'Name') == 'moby' else socket.gethostname()
port = cluster.primary_node.host_ports.get(AMBARI_PORT)
server_url = 'http://{}:{}'.format(hostname, port)
logger.info('Ambari server is now reachable at %s', server_url)
logger.info('Starting Ambari agents ...')
for node in cluster:
logger.debug('Starting Ambari agent on %s ...', node.fqdn)
node.execute('ambari-agent start', quiet=not args.verbose)
ambari = Ambari(server_url, username='******', password='******')
def condition(ambari, cluster):
cluster_hosts = {node.fqdn for node in cluster}
ambari_hosts = {host.host_name for host in ambari.hosts}
logger.debug('Cluster hosts: %s; Ambari hosts: %s', cluster_hosts,
ambari_hosts)
return cluster_hosts == ambari_hosts
wait_for_condition(condition=condition, condition_args=[ambari, cluster])
for node in secondary_nodes[1:]:
logger.info('Adding %s to cluster ...', node.fqdn)
ambari.clusters('cluster').hosts.create(node.fqdn)
for component in ambari.clusters('cluster').hosts(
secondary_nodes[0].fqdn).components:
logger.debug('Adding component (%s) to cluster on host (%s) ...',
component.component_name, node.fqdn)
host_components = ambari.clusters('cluster').hosts(
node.fqdn).components
host_components.create(component.component_name).wait()
logger.debug('Installing all registered components on host (%s) ...',
node.fqdn)
ambari.clusters('cluster').hosts(node.fqdn).components.install().wait()
if not args.dont_start_cluster:
logger.debug(
'Waiting for all hosts to reach healthy state before starting cluster ...'
)
def condition(ambari):
health_report = ambari.clusters('cluster').health_report
logger.debug('Ambari cluster health report: %s ...', health_report)
return health_report.get('Host/host_state/HEALTHY') == len(
list(ambari.hosts))
wait_for_condition(condition=condition, condition_args=[ambari])
logger.info('Starting cluster services ...')
ambari.clusters('cluster').services.start().wait()