def kibana(action=None): if action == 'start' or action == 'stop' or action == 'status': cmd=Popen(['service','httpd',action],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() Logger.info('Kibana action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err)) if action == 'start' or action == 'status': check_rc(cmd.returncode,stdout=out,stderr=err) if action == 'config' : import params File('/etc/httpd/conf.d/kibana.conf', content=Static('kibana.conf'), owner='apache', group='apache' ) File('/etc/kibana/conf/config.js', content=Template('config.js.j2'), owner="apache", group="apache", mode=0644) if action == 'install': os.symlink('/usr/lib/kibana','/var/www/kibana') os.chdir('/var/www/kibana') cmd=Popen(['gem','install','bundle']) cmd=Popen(['bundle','install'])
def kibana(action=None): if action == 'start' or action == 'stop' or action == 'status': cmd = Popen(['service', 'httpd', action], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() Logger.info('Kibana action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action, out, err)) if action == 'start' or action == 'status': check_rc(cmd.returncode, stdout=out, stderr=err) if action == 'config': import params File('/etc/httpd/conf.d/kibana.conf', content=StaticFile('kibana.conf'), owner='apache', group='apache') File('/etc/kibana/conf/config.js', content=Template('config.js.j2'), owner="apache", group="apache", mode=0644) if action == 'install': if not os.path.exists('/var/www/html/kibana'): os.symlink('/usr/lib/kibana', '/var/www/html/kibana') #os.chdir('/var/www/html/kibana') cmd = Popen(['gem', 'install', '--no-ri', '--no-rdoc', 'bundle'])
def snamenode(action=None, format=False): if action == "configure": import params Directory(params.fs_checkpoint_dir, recursive=True, mode=0755, owner=params.hdfs_user, group=params.user_group) File(params.exclude_file_path, content=Template("exclude_hosts_list.j2"), owner=params.hdfs_user, group=params.user_group) elif action == "start" or action == "stop" or action == "status": """ In this point, HDP code uses a much more complex execution, I assume it is for standarization porpuses and avoid using /etc/init.d """ executed = Popen(["service","hadoop-hdfs-secondarynamenode",action],stdout=PIPE,stderr=PIPE) out,err = executed.communicate() Logger.info("Secondary namenode:"+action) Logger.info(out) Logger.info(err) rc = executed.returncode check_rc(rc,out,err)
def elasticsearch(action=None): if action == 'start' or action == 'stop' or action == 'status': cmd=Popen(['service','elasticsearch',action],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() Logger.info('Elasticsearch action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err)) if action == 'start' or action == 'status': check_rc(cmd.returncode,stdout=out,stderr=err) if action == 'config' : import params File('/etc/security/limits.d/elasticsearch.conf', content=Template('limits.j2'), owner='root', group='root' ) Directory(params.path_data, recursive=True, owner='elasticsearch', group='elasticsearch' ) File('/etc/elasticsearch/elasticsearch.yml', content=Template('elasticsearch.j2'), owner="elasticsearch", group="elasticsearch", mode=0644) File('/etc//sysconfig/elasticsearch', content=Template('elasticsearch-env.j2'), owner="elasticsearch", group="elasticsearch", mode=0644)
def snamenode(action=None, format=False): if action == "configure": import params Directory(params.fs_checkpoint_dir, recursive=True, mode=0755, owner=params.hdfs_user, group=params.user_group) File(params.exclude_file_path, content=Template("exclude_hosts_list.j2"), owner=params.hdfs_user, group=params.user_group) elif action == "start" or action == "stop" or action == "status": """ In this point, HDP code uses a much more complex execution, I assume it is for standarization porpuses and avoid using /etc/init.d """ executed = Popen(["service", "hadoop-hdfs-secondarynamenode", action], stdout=PIPE, stderr=PIPE) out, err = executed.communicate() rc = executed.returncode check_rc(rc, out, err)
def resourcemanager(action=None): if action == "configure" or action == "start": yarn_conf.configure(service="resourcemanager") if action == "start" or action == "stop" or action == "status": cmd = Popen(["service","hadoop-yarn-resourcemanager",action],stdout=PIPE,stderr=PIPE) out,err = cmd.communicate() rc=cmd.returncode check_rc(rc,stdout=out,stderr=err)
def historyserver(action=None): if action == "configure" or action == "start": yarn_conf.configure(service="historyserver") if action == "start" or action == "stop" or action == "status": cmd = Popen(["service","hadoop-mapreduce-historyserver",action],stdout=PIPE,stderr=PIPE) out,err = cmd.communicate() rc=cmd.returncode check_rc(rc,stdout=out,stderr=err)
def resourcemanager(action=None): if action == "configure" or action == "start": yarn_conf.configure(service="resourcemanager") if action == "start" or action == "stop" or action == "status": cmd = Popen(["service", "hadoop-yarn-resourcemanager", action], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() rc = cmd.returncode check_rc(rc, stdout=out, stderr=err)
def historyserver(action=None): if action == "configure" or action == "start": yarn_conf.configure(service="historyserver") if action == "start" or action == "stop" or action == "status": cmd = Popen(["service", "hadoop-mapreduce-historyserver", action], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() rc = cmd.returncode check_rc(rc, stdout=out, stderr=err)
def hue(service=None,action=None): if action == "initdb": import params cmd=Popen(['/usr/lib/hue/build/env/bin/hue','syncdb','--noinput'],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() Logger.info("Hue installation: sync db") Logger.info(out) Logger.info(err) cmd=Popen(['/usr/lib/hue/build/env/bin/hue','migrate'],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() Logger.info("Hue installation: migrate tables") Logger.info(out) Logger.info(err) Logger.info("Hue installation: Installation of kafka-hue") os.chmod('/usr/lib/hue/tools/app_reg/app_reg.py',0755) os.chdir('/usr/lib/hue/apps') cmd=Popen(['../tools/app_reg/app_reg.py','--install','kafka','--relative-paths'],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() Logger.info(out) Logger.info(err) Logger.info("Hue installation: Installation of storm-hue") cmd=Popen(['../tools/app_reg/app_reg.py','--install','storm','--relative-paths'],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() Logger.info(out) Logger.info(err) if action == "config": import params #configurations = params.config['configurations']['spark'] File(format(params.hue_conf_dir+"hue.ini"), content=Template("hue.ini.j2"), owner="root", group="root" ) if action == "start" or action == "stop": cmd=Popen(['service','hue',action],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() rc=cmd.returncode Logger.info("Hue service %s: %s" % (action, cmd.returncode == 0)) if action == "status": from utils import check_rc cmd=Popen(['service','hue',action],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() rc=cmd.returncode Logger.info(rc) Logger.info("Hue service %s: %s" % (action, cmd.returncode == 0)) check_rc(rc,stdout=out,stderr=err)
def kafka(action): import utils # FIX: When status is checked params shouldn't be called if action != 'status': import params if action == 'config': utils.os_mkdir(params.log_dirs,owner='kafka') kafka_server_properties = "broker.id=" + str(params.kafka_id) + "\n" for param in params.kafka_conf.keys(): value=params.kafka_conf[param] if type(value)==bool: value=str(params.kafka_conf[param]).lower()+"\n" else: value=str(params.kafka_conf[param])+"\n" kafka_server_properties+=param+"="+value if params.has_ganglia_server: File('/etc/default/kafka.sh', content=StaticFile('kafka.profiles.d') ) File('/etc/kafka/conf/server.properties', content=kafka_server_properties, owner='kafka', group='kafka') if params.has_ganglia_server: File('/etc/jmxtrans/config/jmxtrans.config', content=Template('jmxtrans.j2') ) File('/etc/jmxtrans/config/KafkaMetrics.json', content=StaticFile('KafkaMetrics.json') ) elif action == 'start' or action == 'stop': executed = Popen(["service","kafka",action],stdout=PIPE,stderr=PIPE) out,err = executed.communicate() Logger.info("Kafka service:") Logger.info(action) Logger.info(out) Logger.info(err) if params.has_ganglia_server: executed2=Popen(["service","jmxtrans",action]) out,err = executed2.communicate() Logger.info("JMXtrans service") Logger.info(action) Logger.info(out) Logger.info(err) elif action == "status": executed = Popen(["service","kafka",action],stdout=PIPE,stderr=PIPE) out,err = executed.communicate() rc = executed.returncode utils.check_rc(rc,out,err)
def elasticsearch(action=None): if action == 'start' or action == 'stop' or action == 'status': cmd=Popen(['service','elasticsearch',action],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() Logger.info('Elasticsearch action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err)) if action == 'start' or action == 'status': check_rc(cmd.returncode,stdout=out,stderr=err) if action == 'config' : File('/etc/elasticsearch/elasticsearch.yml', content=Template('elasticsearch.j2'), owner="elasticsearch", group="elasticsearch", mode=0644)
def journalnode(action=None, format=False): if action == "configure": import params Directory(params.jn_edits_dir, recursive=True, owner=params.hdfs_user, group=params.user_group ) if action == "start" or action == "stop" or action == "status": executed = Popen(["service","hadoop-hdfs-journalnode",action],stdout=PIPE,stderr=PIPE) out,err = executed.communicate() if action == "status": rc = executed.returncode check_rc(rc,out,err)
def kibana(action=None): if action == 'start' or action == 'stop' or action == 'status': cmd = Popen(['service', 'kibana4', action], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() Logger.info('Kibana4 action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action, out, err)) if action == 'start' or action == 'status': check_rc(cmd.returncode, stdout=out, stderr=err) if action == 'config': import params Directory('/var/log/kibana', owner='kibana', group='kibana', mode=0550) File('/usr/lib/kibana4/config/kibana.yml', content=Template('kibana.j2'), owner='kibana', group='kibana')
def elasticsearch(action=None): if action == 'start' or action == 'stop' or action == 'status': cmd = Popen(['service', 'elasticsearch', action], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() Logger.info('Elasticsearch action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action, out, err)) if action == 'start' or action == 'status': check_rc(cmd.returncode, stdout=out, stderr=err) if action == 'config': File('/etc/elasticsearch/elasticsearch.yml', content=Template('elasticsearch.j2'), owner="elasticsearch", group="elasticsearch", mode=0644)
def cassandra(action=None): if action == 'start' or action == 'stop' or action == 'status': cmd=Popen(['service','cassandra',action],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() Logger.info('Cassandra action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err)) if action == 'start' or action == 'status': check_rc(cmd.returncode,stdout=out,stderr=err) if action == 'config': import params File('/etc/cassandra/conf/cassandra.yaml', content=Template('cassandra.j2'), owner='cassandra', group='cassandra' ) File('/etc/cassandra/conf/cassandra-rackdc.properties', content=Template('cassandra-rackdc.properties.j2'), owner='cassandra', group='cassandra' )
def kibana(action=None): if action == 'start' or action == 'stop' or action == 'status': cmd=Popen(['service','kibana4',action],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() Logger.info('Kibana4 action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err)) if action == 'start' or action == 'status': check_rc(cmd.returncode,stdout=out,stderr=err) if action == 'config' : import params Directory('/var/log/kibana', owner='kibana', group='kibana', mode=0550 ) File('/usr/lib/kibana4/config/kibana.yml', content=Template('kibana.j2'), owner='kibana', group='kibana' )
def zkfc(action=None, format=False): if action == "configure": File("/etc/monit.d/hadoop-hdfs-zkfc", content=StaticFile("monit.d_hadoop-hdfs-zkfc")) File("/etc/monit.conf", content=StaticFile("monit.conf")) #First check monit is running and start if not executed = Popen(["service","monit","status"]) out,err=executed.communicate() Logger.info("Checking monit status") Logger.info(out) Logger.info(err) rc = executed.returncode if rc > 0 or action=="configure": executed = Popen(["service","monit","restart"]) out,err=executed.communicate() Logger.info("Restarting monit") Logger.info(out) Logger.info(err) if action == "start": # AC: this avoids a race condition at restart time.sleep(30) executed = Popen(["monit",action,"hadoop-hdfs-zkfc","-v"],stdout=PIPE,stderr=PIPE) out,err = executed.communicate() Logger.info("Monit Hdfs-zkfc: "+action) Logger.info(out) Logger.info(err) rc = executed.returncode check_rc(rc,out,err) if action == "stop": executed = Popen(["monit",action,"hadoop-hdfs-zkfc","-v"],stdout=PIPE,stderr=PIPE) out,err = executed.communicate() Logger.info("Monit Hdfs-zkfc: "+action) Logger.info(out) Logger.info(err) rc = executed.returncode check_rc(rc,out,err) if action == "status": executed = Popen(["service","hadoop-hdfs-zkfc",action],stdout=PIPE,stderr=PIPE) out,err = executed.communicate() Logger.info("Hdfs-zkfc: "+action) Logger.info(out) Logger.info(err) rc = executed.returncode check_rc(rc,out,err)
def oozie(action=None,is_server=False): import params if action == 'install': cmd=Popen(['/usr/sbin/usermod','-a','-G','hadoop','oozie'],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() Logger.info('Oozie action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err)) if action == 'start' or action == 'stop' or action == 'status': cmd=Popen(['service','oozie',action],stdout=PIPE,stderr=PIPE) out,err=cmd.communicate() Logger.info('Oozie action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err)) if action == 'start' or action == 'status': check_rc(cmd.returncode,stdout=out,stderr=err) if action == 'config' : File(params.oozie_config_dir + '/oozie-site.xml', content=Template('oozie-site.j2'), owner=params.oozie_user, group=params.oozie_group, mode=0644) File(params.oozie_config_dir+'/oozie-env.sh', content=Template('oozie-env.j2'), owner=params.oozie_user, group=params.oozie_group, mode=0644) File(params.oozie_config_dir+'/adminusers.txt', content=Template('adminusers.txt.j2'), owner=params.oozie_user, group=params.oozie_group, mode=0644) if is_server : #File('/usr/lib/oozie/libext/ext-2.2.1.zip', # content=StaticFile('ext-2.2.1.zip')) ## oozie expect ext-2.2 directory and looks to be hardcoded #extract_cmd=[ 'unzip', '/usr/lib/oozie/libext/ext-2.2.1.zip','-d','/usr/lib/oozie/libext/ext-2.2' ] #Popen(extract_cmd) extract_cmd=[ 'ln', '-s','/usr/share/java/mysql-connector-java.jar','/usr/lib/oozie/libext/mysql-connector-java.jar' ] cmd=Popen(extract_cmd) out,err=cmd.communicate() Logger.info("Creating mysql-connector-java.jar symbolic link in /usr/lib/oozie/libext/") Logger.info(out) Logger.info(err) Logger.info(params.oozie_jdbc_driver) if params.oozie_jdbc_driver == "com.mysql.jdbc.Driver": create_db_cmd = format('su --shell=/bin/bash -l oozie -c "source /etc/profile.d/java.sh && /usr/lib/oozie/bin/ooziedb.sh create -sqlfile oozie.sql -run"') if params.oozie_jdbc_driver == "org.postgresql.Driver": pass if params.oozie_jdbc_driver == "oracle.jdbc.driver.OracleDriver": pass cmd=Popen(create_db_cmd, shell=True) out,err=cmd.communicate() Logger.info("Installing the Oozie Schema in the DB") Logger.info(out) Logger.info(err)
def hue(service=None, action=None): if action == "initdb": import params cmd = Popen(['/usr/lib/hue/build/env/bin/hue', 'syncdb', '--noinput'], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() Logger.info("Hue installation: sync db") Logger.info(out) Logger.info(err) cmd = Popen(['/usr/lib/hue/build/env/bin/hue', 'migrate'], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() Logger.info("Hue installation: migrate tables") Logger.info(out) Logger.info(err) Logger.info("Hue installation: Installation of kafka-hue") os.chmod('/usr/lib/hue/tools/app_reg/app_reg.py', 0755) os.chdir('/usr/lib/hue/apps') cmd = Popen([ '../tools/app_reg/app_reg.py', '--install', 'kafka', '--relative-paths' ], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() Logger.info(out) Logger.info(err) Logger.info("Hue installation: Installation of storm-hue") cmd = Popen([ '../tools/app_reg/app_reg.py', '--install', 'storm', '--relative-paths' ], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() Logger.info(out) Logger.info(err) if action == "config": import params #configurations = params.config['configurations']['spark'] File(format(params.hue_conf_dir + "hue.ini"), content=Template("hue.ini.j2"), owner="root", group="root") if action == "start" or action == "stop": cmd = Popen(['service', 'hue', action], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() rc = cmd.returncode Logger.info("Hue service %s: %s" % (action, cmd.returncode == 0)) if action == "status": from utils import check_rc cmd = Popen(['service', 'hue', action], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() rc = cmd.returncode Logger.info(rc) Logger.info("Hue service %s: %s" % (action, cmd.returncode == 0)) check_rc(rc, stdout=out, stderr=err)
def oozie(action=None, is_server=False): import params if action == 'install': cmd = Popen(['/usr/sbin/usermod', '-a', '-G', 'hadoop', 'oozie'], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() Logger.info('Oozie action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action, out, err)) if action == 'start' or action == 'stop' or action == 'status': cmd = Popen(['service', 'oozie', action], stdout=PIPE, stderr=PIPE) out, err = cmd.communicate() Logger.info('Oozie action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action, out, err)) if action == 'start' or action == 'status': check_rc(cmd.returncode, stdout=out, stderr=err) if action == 'config': File(params.oozie_config_dir + '/oozie-site.xml', content=Template('oozie-site.j2'), owner=params.oozie_user, group=params.oozie_group, mode=0644) File(params.oozie_config_dir + '/oozie-env.sh', content=Template('oozie-env.j2'), owner=params.oozie_user, group=params.oozie_group, mode=0644) File(params.oozie_config_dir + '/adminusers.txt', content=Template('adminusers.txt.j2'), owner=params.oozie_user, group=params.oozie_group, mode=0644) if is_server: #File('/usr/lib/oozie/libext/ext-2.2.1.zip', # content=StaticFile('ext-2.2.1.zip')) ## oozie expect ext-2.2 directory and looks to be hardcoded #extract_cmd=[ 'unzip', '/usr/lib/oozie/libext/ext-2.2.1.zip','-d','/usr/lib/oozie/libext/ext-2.2' ] #Popen(extract_cmd) extract_cmd = [ 'ln', '-s', '/usr/share/java/mysql-connector-java.jar', '/usr/lib/oozie/libext/mysql-connector-java.jar' ] cmd = Popen(extract_cmd) out, err = cmd.communicate() Logger.info( "Creating mysql-connector-java.jar symbolic link in /usr/lib/oozie/libext/" ) Logger.info(out) Logger.info(err) Logger.info(params.oozie_jdbc_driver) if params.oozie_jdbc_driver == "com.mysql.jdbc.Driver": create_db_cmd = format( 'su --shell=/bin/bash -l oozie -c "source /etc/profile.d/java.sh && /usr/lib/oozie/bin/ooziedb.sh create -sqlfile oozie.sql -run"' ) if params.oozie_jdbc_driver == "org.postgresql.Driver": pass if params.oozie_jdbc_driver == "oracle.jdbc.driver.OracleDriver": pass cmd = Popen(create_db_cmd, shell=True) out, err = cmd.communicate() Logger.info("Installing the Oozie Schema in the DB") Logger.info(out) Logger.info(err)
def flume(action = None): import params import flume_extra if action == 'config': # remove previously defined meta's for n in find_expected_agent_names(): os.unlink(os.path.join(params.flume_conf_dir, n, 'ambari-meta.json')) Directory(params.flume_conf_dir, recursive=True) Directory(params.flume_log_dir, owner=params.flume_user) File(os.path.join(params.flume_conf_dir, 'log4j.properties'), content=StaticFile("log4j.properties")) flume_agents = {} if params.flume_conf_content is not None: flume_agents = build_flume_topology(params.flume_conf_content) if params.flume_env_content is not None: flume_agents_env = build_flume_env(params.flume_env_content) for agent in flume_agents.keys(): flume_meta_file = os.path.join(params.flume_conf_dir, 'ambari-meta_%s.json' % agent ) File(flume_meta_file, content = json.dumps(ambari_meta(agent, flume_agents[agent])), mode = 0644) flume_env = flume_agents_env[agent] if flume_agents_env.has_key(agent) else flume_agents_env['default'] File(os.path.join(params.flume_agent_conf_dir,'%s-env.sh' % agent), content = InlineTemplate(flume_env), mode = 0644) flume_agent_conf_file = os.path.join(params.flume_agent_conf_dir, '%s.conf' % agent) PropertiesFile(flume_agent_conf_file, properties=flume_agents[agent], mode = 0644) if params.flume_extra is not None: #import jsonschema #File('/tmp/flume_extra.schema', # content=StaticFile("flume_extra.schema")) #flume_extra_schema = json.loads(open('/tmp/flume_extra.schema').read()) for extra_file_raw in params.flume_extra.values(): extra_file=json.loads(extra_file_raw) if not extra_file.has_key('path'): raise Exception("path field is required") File(extra_file['path'], mode = int(extra_file['mode'],8), owner = extra_file['owner'], group = extra_file['group'], content=extra_file['content'] ) if action == "start" or action == "stop" or action == "status": executed = Popen(["service","flume-agent",action],stdout=PIPE,stderr=PIPE) out,err = executed.communicate() if action == "status": from utils import check_rc rc = executed.returncode check_rc(rc,out,err) if action == "stop": filelist = glob.glob(params.flume_agent_conf_dir+"/*.conf") for f in filelist: os.remove(f) # Removes flume-env files filelist = glob.glob(params.flume_agent_conf_dir+"/*.sh") for f in filelist: os.remove(f)