def manage():
if not bootstrap_resources():
# defer until resources are available, since charmhelpers, and thus
# the framework, are required (will require manual intervention)
return
import jujubigdata
import callbacks
zookeeper_reqs = ['vendor', 'packages', 'groups', 'users', 'dirs', 'ports']
dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml',
required_keys=zookeeper_reqs)
zookeeper = callbacks.Zookeeper(dist_config)
manager = charmframework.Manager([
{
'name':
'zookeeper',
'provides': [],
'requires': [
zookeeper.verify_resources,
],
'callbacks': [
callbacks.update_working_status,
zookeeper.install,
zookeeper.start,
callbacks.update_active_status,
],
'cleanup': [
zookeeper.stop,
zookeeper.cleanup,
],
},
])
manager.manage()
def manage():
if not bootstrap_resources():
# defer until resources are available, since charmhelpers, and thus
# the framework, are required (will require manual intervention)
return
import jujubigdata
import callbacks
kafka_reqs = ['vendor', 'packages', 'groups', 'users', 'dirs', 'ports']
dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml',
required_keys=kafka_reqs)
kafka = callbacks.Kafka(dist_config)
manager = charmframework.Manager([
{
'name': 'kafka',
'provides': [
jujubigdata.relations.Kafka(port=dist_config.port('kafka'))
],
'requires': [
kafka.verify_resources,
jujubigdata.relations.Zookeeper(),
callbacks.update_blocked_status, # not really a requirement, but best way to fit into framework
],
'callbacks': [
callbacks.update_working_status,
kafka.install,
kafka.configure_kafka,
kafka.restart,
charmframework.helpers.open_ports(
dist_config.exposed_ports('kafka')),
callbacks.update_active_status,
],
'cleanup': [
callbacks.clear_active_flag,
charmframework.helpers.close_ports(
dist_config.exposed_ports('kafka')),
kafka.stop,
kafka.cleanup,
callbacks.update_blocked_status,
],
},
])
manager.manage()
def manage():
if not bootstrap_resources():
# defer until resources are available, since charmhelpers, and thus
# the framework, are required (will require manual intervention)
return
import jujubigdata
import callbacks
flume_reqs = ['packages', 'groups', 'users', 'dirs']
dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml',
required_keys=flume_reqs)
flume = callbacks.Flume(dist_config)
manager = charmframework.Manager([
{
'name':
'flume',
'provides': [],
'requires': [
flume.verify_resources,
jujubigdata.relations.FlumeAgent(),
callbacks.
update_blocked_status, # not really a requirement, but best way to fit into framework
],
'callbacks': [
callbacks.update_working_status,
flume.install,
charmframework.helpers.render_template(
source='flume.conf.j2',
target=flume.dist_config.path('flume_conf') / 'flume.conf',
context={'dist_config': dist_config}),
flume.restart,
callbacks.update_active_status,
],
'cleanup': [
callbacks.clear_active_flag,
flume.stop,
flume.cleanup,
callbacks.update_blocked_status,
],
},
])
manager.manage()
def manage():
if not bootstrap_resources():
# defer until resources are available, since jujubigdata, and thus the
# classes needed for the requires blocks, (will be) managed by jujuresources
return
import jujubigdata
import callbacks
# list of keys required to be in the dist.yaml
yarn_reqs = [
'vendor', 'hadoop_version', 'packages', 'groups', 'users', 'dirs',
'ports'
]
dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml',
required_keys=yarn_reqs)
hadoop = jujubigdata.handlers.HadoopBase(dist_config)
yarn = jujubigdata.handlers.YARN(hadoop)
hdfs = jujubigdata.handlers.HDFS(hadoop)
port = dist_config.port('resourcemanager')
hs_http = dist_config.port('jh_webapp_http')
hs_ipc = dist_config.port('jobhistory')
nodemanagers = jujubigdata.relations.ResourceManagerMaster(
spec=hadoop.spec,
port=port,
historyserver_http=hs_http,
historyserver_ipc=hs_ipc)
clients = jujubigdata.relations.ResourceManager(spec=hadoop.spec,
port=port,
historyserver_http=hs_http,
historyserver_ipc=hs_ipc)
namenode = jujubigdata.relations.NameNode(spec=hadoop.client_spec)
manager = charmframework.Manager([
{
'name': 'hadoop-base',
'requires': [
hadoop.verify_conditional_resources,
],
'callbacks': [
hadoop.install,
callbacks.update_blocked_status,
],
},
{
'name':
'yarn-master',
'provides': [
nodemanagers,
clients,
],
'requires': [
hadoop.is_installed,
namenode,
jujubigdata.relations.NodeManager(optional=True),
],
'callbacks': [
# These callbacks will be executed once the Hadoop base packages
# are installed and HDFS is available. New items can be added
# to the end of this list and to hooks/callbacks.py to extend
# the functionality of this charm.
callbacks.update_working_status,
nodemanagers.register_connected_hosts,
clients.register_connected_hosts,
namenode.register_provided_hosts,
jujubigdata.utils.manage_etc_hosts,
hdfs.configure_client,
yarn.configure_resourcemanager,
yarn.configure_jobhistory,
yarn.register_slaves,
yarn.start_resourcemanager,
yarn.start_jobhistory,
charmframework.helpers.open_ports(
dist_config.exposed_ports('yarn-master')),
callbacks.update_active_status,
],
'cleanup': [
callbacks.clear_active_flag,
charmframework.helpers.close_ports(
dist_config.exposed_ports('yarn-master')),
yarn.stop_resourcemanager,
yarn.stop_jobhistory,
callbacks.update_blocked_status,
],
},
{
'name': 'ganglia',
'requires': [
hadoop.is_installed,
jujubigdata.relations.Ganglia,
],
'callbacks': [
callbacks.conf_ganglia_metrics,
],
'cleanup': [
callbacks.purge_ganglia_metrics,
],
},
])
manager.manage()
def manage():
if not bootstrap_resources():
# defer until resources are available, since jujubigdata, and thus the
# classes needed for the requires blocks, (will be) managed by jujuresources
return
import jujubigdata
import callbacks
# list of keys required to be in the dist.yaml
master_reqs = [
'vendor', 'hadoop_version', 'packages', 'groups', 'users', 'dirs',
'ports'
]
dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml',
required_keys=master_reqs)
hadoop = jujubigdata.handlers.HadoopBase(dist_config)
hdfs = jujubigdata.handlers.HDFS(hadoop)
hdfs_port = dist_config.port('namenode')
webhdfs_port = dist_config.port('nn_webapp_http')
secondary = jujubigdata.relations.NameNodeMaster(spec=hadoop.spec,
port=hdfs_port,
webhdfs_port=webhdfs_port,
relation_name='secondary')
datanodes = jujubigdata.relations.NameNodeMaster(spec=hadoop.spec,
port=hdfs_port,
webhdfs_port=webhdfs_port)
clients = jujubigdata.relations.NameNode(spec=hadoop.spec,
port=hdfs_port,
webhdfs_port=webhdfs_port)
manager = charmframework.Manager([
{
'name': 'hadoop-base',
'requires': [
hadoop.verify_conditional_resources,
],
'callbacks': [
hadoop.install,
],
},
{
'name':
'hdfs-master',
'provides': [
secondary,
datanodes,
clients,
],
'requires': [
hadoop.is_installed,
jujubigdata.relations.DataNode(optional=True),
jujubigdata.relations.SecondaryNameNode(optional=True),
],
'callbacks': [
# These callbacks will be executed once the Hadoop base packages
# are installed. New items can be added to the end of this list
# and to hooks/callbacks.py to extend the functionality of this
# charm.
callbacks.update_working_status,
datanodes.register_connected_hosts,
secondary.register_connected_hosts,
clients.register_connected_hosts,
jujubigdata.utils.manage_etc_hosts,
hdfs.configure_namenode,
hdfs.register_slaves,
hdfs.format_namenode,
hdfs.start_namenode,
hdfs.create_hdfs_dirs,
charmframework.helpers.open_ports(
dist_config.exposed_ports('hdfs-master')),
callbacks.update_active_status,
],
'cleanup': [
callbacks.clear_active_flag,
charmframework.helpers.close_ports(
dist_config.exposed_ports('hdfs-master')),
hdfs.stop_namenode,
callbacks.update_active_status,
],
},
{
'name': 'ganglia',
'requires': [
hadoop.is_installed,
jujubigdata.relations.Ganglia,
],
'callbacks': [
callbacks.conf_ganglia_metrics,
],
'cleanup': [callbacks.purge_ganglia_metrics],
},
])
manager.manage()
def manage():
if not bootstrap_resources():
# defer until resources are available, since jujubigdata, and thus the
# classes needed for the requires blocks, (will be) managed by jujuresources
return
import jujubigdata
import callbacks
# list of keys required to be in the dist.yaml
slave_reqs = [
'vendor', 'hadoop_version', 'packages', 'groups', 'users', 'dirs',
'ports'
]
dist_config = jujubigdata.utils.DistConfig(filename='dist.yaml',
required_keys=slave_reqs)
hadoop = jujubigdata.handlers.HadoopBase(dist_config)
hdfs = jujubigdata.handlers.HDFS(hadoop)
hdfs_relation = jujubigdata.relations.NameNodeMaster(spec=hadoop.spec)
yarn = jujubigdata.handlers.YARN(hadoop)
yarn_relation = jujubigdata.relations.ResourceManagerMaster(
spec=hadoop.spec)
manager = charmframework.Manager([
{
'name': 'hadoop-base',
'requires': [
hadoop.verify_conditional_resources,
],
'callbacks': [
hadoop.install,
callbacks.update_blocked_status,
],
},
{
'name':
'datanode',
'provides': [
jujubigdata.relations.DataNode(),
],
'requires': [
hadoop.is_installed,
hdfs_relation,
hdfs_relation.am_i_registered,
],
'callbacks': [
callbacks.update_working_status,
hdfs_relation.register_provided_hosts,
jujubigdata.utils.manage_etc_hosts,
hdfs_relation.install_ssh_keys,
hdfs.configure_datanode,
hdfs.start_datanode,
charmframework.helpers.open_ports(
dist_config.exposed_ports('compute-slave-hdfs')),
callbacks.update_active_status,
],
'cleanup': [
callbacks.clear_active_flag,
charmframework.helpers.close_ports(
dist_config.exposed_ports('compute-slave-hdfs')),
hdfs.stop_datanode,
callbacks.update_blocked_status,
],
},
{
'name':
'nodemanager',
'provides': [
jujubigdata.relations.NodeManager(),
],
'requires': [
hadoop.is_installed,
yarn_relation,
yarn_relation.am_i_registered,
],
'callbacks': [
callbacks.update_working_status,
yarn_relation.register_provided_hosts,
jujubigdata.utils.manage_etc_hosts,
yarn_relation.install_ssh_keys,
yarn.configure_nodemanager,
yarn.start_nodemanager,
charmframework.helpers.open_ports(
dist_config.exposed_ports('compute-slave-yarn')),
callbacks.update_active_status,
],
'cleanup': [
callbacks.clear_active_flag,
charmframework.helpers.close_ports(
dist_config.exposed_ports('compute-slave-yarn')),
yarn.stop_nodemanager,
callbacks.
update_active_status, # might still be active if HDFS-only
],
},
{
'name': 'ganglia',
'requires': [
hadoop.is_installed,
jujubigdata.relations.Ganglia,
],
'callbacks': [
callbacks.conf_ganglia_metrics,
],
'cleanup': [callbacks.purge_ganglia_metrics],
},
])
manager.manage()