Exemplo n.º 1
0
def kibana(action=None):
  
  if action == 'start' or action == 'stop' or action == 'status':
    cmd=Popen(['service','httpd',action],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    Logger.info('Kibana action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err))
    if action == 'start' or action == 'status':
      check_rc(cmd.returncode,stdout=out,stderr=err)

  if action == 'config' :
    import params
    File('/etc/httpd/conf.d/kibana.conf',
      content=Static('kibana.conf'),
      owner='apache',
      group='apache'
    )

    File('/etc/kibana/conf/config.js',
      content=Template('config.js.j2'),
      owner="apache",
      group="apache",
      mode=0644)

  if action == 'install':
    os.symlink('/usr/lib/kibana','/var/www/kibana')
    os.chdir('/var/www/kibana')
    cmd=Popen(['gem','install','bundle'])
    cmd=Popen(['bundle','install']) 
Exemplo n.º 2
0
def kibana(action=None):

    if action == 'start' or action == 'stop' or action == 'status':
        cmd = Popen(['service', 'httpd', action], stdout=PIPE, stderr=PIPE)
        out, err = cmd.communicate()
        Logger.info('Kibana action: %s.\nSTDOUT=%s\nSTDERR=%s' %
                    (action, out, err))
        if action == 'start' or action == 'status':
            check_rc(cmd.returncode, stdout=out, stderr=err)

    if action == 'config':
        import params
        File('/etc/httpd/conf.d/kibana.conf',
             content=StaticFile('kibana.conf'),
             owner='apache',
             group='apache')

        File('/etc/kibana/conf/config.js',
             content=Template('config.js.j2'),
             owner="apache",
             group="apache",
             mode=0644)

    if action == 'install':
        if not os.path.exists('/var/www/html/kibana'):
            os.symlink('/usr/lib/kibana', '/var/www/html/kibana')
        #os.chdir('/var/www/html/kibana')
        cmd = Popen(['gem', 'install', '--no-ri', '--no-rdoc', 'bundle'])
Exemplo n.º 3
0
def snamenode(action=None, format=False):

  if action == "configure":
    import params
    Directory(params.fs_checkpoint_dir,
              recursive=True,
              mode=0755,
              owner=params.hdfs_user,
              group=params.user_group)
    File(params.exclude_file_path,
         content=Template("exclude_hosts_list.j2"),
         owner=params.hdfs_user,
         group=params.user_group)
  elif action == "start" or action == "stop" or action == "status":
    """
    In this point, HDP code uses a much more complex execution,
    I assume it is for standarization porpuses and avoid using
    /etc/init.d
    """
    executed = Popen(["service","hadoop-hdfs-secondarynamenode",action],stdout=PIPE,stderr=PIPE)
    out,err = executed.communicate()
    Logger.info("Secondary namenode:"+action)
    Logger.info(out)
    Logger.info(err)
    rc = executed.returncode
    check_rc(rc,out,err)
Exemplo n.º 4
0
def elasticsearch(action=None):
  
  if action == 'start' or action == 'stop' or action == 'status':
    cmd=Popen(['service','elasticsearch',action],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    Logger.info('Elasticsearch action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err))
    if action == 'start' or action == 'status':
      check_rc(cmd.returncode,stdout=out,stderr=err)

  if action == 'config' :
    import params
    File('/etc/security/limits.d/elasticsearch.conf',
      content=Template('limits.j2'),
      owner='root',
      group='root'
    )
    Directory(params.path_data,
      recursive=True,
      owner='elasticsearch',
      group='elasticsearch'
    )


    File('/etc/elasticsearch/elasticsearch.yml',
      content=Template('elasticsearch.j2'),
      owner="elasticsearch",
      group="elasticsearch",
      mode=0644)

    File('/etc//sysconfig/elasticsearch',
      content=Template('elasticsearch-env.j2'),
      owner="elasticsearch",
      group="elasticsearch",
      mode=0644)
Exemplo n.º 5
0
def snamenode(action=None, format=False):

    if action == "configure":
        import params
        Directory(params.fs_checkpoint_dir,
                  recursive=True,
                  mode=0755,
                  owner=params.hdfs_user,
                  group=params.user_group)
        File(params.exclude_file_path,
             content=Template("exclude_hosts_list.j2"),
             owner=params.hdfs_user,
             group=params.user_group)
    elif action == "start" or action == "stop" or action == "status":
        """
    In this point, HDP code uses a much more complex execution,
    I assume it is for standarization porpuses and avoid using
    /etc/init.d
    """
        executed = Popen(["service", "hadoop-hdfs-secondarynamenode", action],
                         stdout=PIPE,
                         stderr=PIPE)
        out, err = executed.communicate()
        rc = executed.returncode
        check_rc(rc, out, err)
Exemplo n.º 6
0
def resourcemanager(action=None):
  if action == "configure" or action == "start":
    yarn_conf.configure(service="resourcemanager")

  if action == "start" or action == "stop" or action == "status":
    cmd = Popen(["service","hadoop-yarn-resourcemanager",action],stdout=PIPE,stderr=PIPE)
    out,err = cmd.communicate()
    rc=cmd.returncode
    check_rc(rc,stdout=out,stderr=err)
def historyserver(action=None):
  if action == "configure" or action == "start":
    yarn_conf.configure(service="historyserver")

  if action == "start" or action == "stop" or action == "status":
    cmd = Popen(["service","hadoop-mapreduce-historyserver",action],stdout=PIPE,stderr=PIPE)
    out,err = cmd.communicate()
    rc=cmd.returncode
    check_rc(rc,stdout=out,stderr=err)
Exemplo n.º 8
0
def resourcemanager(action=None):
    if action == "configure" or action == "start":
        yarn_conf.configure(service="resourcemanager")

    if action == "start" or action == "stop" or action == "status":
        cmd = Popen(["service", "hadoop-yarn-resourcemanager", action],
                    stdout=PIPE,
                    stderr=PIPE)
        out, err = cmd.communicate()
        rc = cmd.returncode
        check_rc(rc, stdout=out, stderr=err)
Exemplo n.º 9
0
def historyserver(action=None):
    if action == "configure" or action == "start":
        yarn_conf.configure(service="historyserver")

    if action == "start" or action == "stop" or action == "status":
        cmd = Popen(["service", "hadoop-mapreduce-historyserver", action],
                    stdout=PIPE,
                    stderr=PIPE)
        out, err = cmd.communicate()
        rc = cmd.returncode
        check_rc(rc, stdout=out, stderr=err)
Exemplo n.º 10
0
def hue(service=None,action=None):

  if action == "initdb":
    import params 
    cmd=Popen(['/usr/lib/hue/build/env/bin/hue','syncdb','--noinput'],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    Logger.info("Hue installation: sync db")
    Logger.info(out)
    Logger.info(err)
    cmd=Popen(['/usr/lib/hue/build/env/bin/hue','migrate'],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    Logger.info("Hue installation: migrate tables")
    Logger.info(out) 
    Logger.info(err) 
    
    Logger.info("Hue installation: Installation of kafka-hue")
    os.chmod('/usr/lib/hue/tools/app_reg/app_reg.py',0755)
    os.chdir('/usr/lib/hue/apps')
    cmd=Popen(['../tools/app_reg/app_reg.py','--install','kafka','--relative-paths'],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    Logger.info(out)
    Logger.info(err)
    Logger.info("Hue installation: Installation of storm-hue")
    cmd=Popen(['../tools/app_reg/app_reg.py','--install','storm','--relative-paths'],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    Logger.info(out)
    Logger.info(err)

  if action == "config":
    import params
    #configurations = params.config['configurations']['spark']
    File(format(params.hue_conf_dir+"hue.ini"),
       content=Template("hue.ini.j2"),
       owner="root",
       group="root"
    )

  if action == "start" or action == "stop":
    cmd=Popen(['service','hue',action],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    rc=cmd.returncode
    Logger.info("Hue service %s: %s" % (action, cmd.returncode == 0)) 

  if action == "status":
      from utils import check_rc
      cmd=Popen(['service','hue',action],stdout=PIPE,stderr=PIPE)
      out,err=cmd.communicate()
      rc=cmd.returncode
      Logger.info(rc)
      Logger.info("Hue service %s: %s" % (action, cmd.returncode == 0))

      check_rc(rc,stdout=out,stderr=err)
Exemplo n.º 11
0
def kafka(action):
  import utils
  # FIX: When status is checked params  shouldn't be called
  if action != 'status':
    import params
    if action == 'config':
      utils.os_mkdir(params.log_dirs,owner='kafka')
      kafka_server_properties = "broker.id=" + str(params.kafka_id) + "\n"
      for param in params.kafka_conf.keys():
        value=params.kafka_conf[param]
        if type(value)==bool:
          value=str(params.kafka_conf[param]).lower()+"\n"
        else:
          value=str(params.kafka_conf[param])+"\n"
        kafka_server_properties+=param+"="+value
      if params.has_ganglia_server:
        File('/etc/default/kafka.sh',
          content=StaticFile('kafka.profiles.d')
        )
      File('/etc/kafka/conf/server.properties',
        content=kafka_server_properties,
        owner='kafka',
        group='kafka')
      if params.has_ganglia_server:
        File('/etc/jmxtrans/config/jmxtrans.config',
          content=Template('jmxtrans.j2')
        )
        File('/etc/jmxtrans/config/KafkaMetrics.json',
          content=StaticFile('KafkaMetrics.json')
        )
    elif action == 'start' or action == 'stop':
      executed = Popen(["service","kafka",action],stdout=PIPE,stderr=PIPE)
      out,err = executed.communicate()
      Logger.info("Kafka service:")
      Logger.info(action) 
      Logger.info(out)
      Logger.info(err)
      if params.has_ganglia_server:
        executed2=Popen(["service","jmxtrans",action])
        out,err = executed2.communicate()
        Logger.info("JMXtrans service")
        Logger.info(action)
        Logger.info(out)
        Logger.info(err)
  elif action == "status":
    executed = Popen(["service","kafka",action],stdout=PIPE,stderr=PIPE)
    out,err = executed.communicate()
    rc = executed.returncode
    utils.check_rc(rc,out,err)
Exemplo n.º 12
0
def elasticsearch(action=None):
  
  if action == 'start' or action == 'stop' or action == 'status':
    cmd=Popen(['service','elasticsearch',action],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    Logger.info('Elasticsearch action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err))
    if action == 'start' or action == 'status':
      check_rc(cmd.returncode,stdout=out,stderr=err)

  if action == 'config' :
    File('/etc/elasticsearch/elasticsearch.yml',
      content=Template('elasticsearch.j2'),
      owner="elasticsearch",
      group="elasticsearch",
      mode=0644)
Exemplo n.º 13
0
def journalnode(action=None, format=False):
  if action == "configure":
    import params
    Directory(params.jn_edits_dir,
              recursive=True,
              owner=params.hdfs_user,
              group=params.user_group
    )

  if action == "start" or action == "stop" or action == "status":
    executed = Popen(["service","hadoop-hdfs-journalnode",action],stdout=PIPE,stderr=PIPE)
    out,err = executed.communicate()
    if action == "status":
      rc = executed.returncode
      check_rc(rc,out,err)
Exemplo n.º 14
0
def kibana(action=None):

    if action == 'start' or action == 'stop' or action == 'status':
        cmd = Popen(['service', 'kibana4', action], stdout=PIPE, stderr=PIPE)
        out, err = cmd.communicate()
        Logger.info('Kibana4 action: %s.\nSTDOUT=%s\nSTDERR=%s' %
                    (action, out, err))
        if action == 'start' or action == 'status':
            check_rc(cmd.returncode, stdout=out, stderr=err)

    if action == 'config':
        import params
        Directory('/var/log/kibana', owner='kibana', group='kibana', mode=0550)
        File('/usr/lib/kibana4/config/kibana.yml',
             content=Template('kibana.j2'),
             owner='kibana',
             group='kibana')
Exemplo n.º 15
0
def elasticsearch(action=None):

    if action == 'start' or action == 'stop' or action == 'status':
        cmd = Popen(['service', 'elasticsearch', action],
                    stdout=PIPE,
                    stderr=PIPE)
        out, err = cmd.communicate()
        Logger.info('Elasticsearch action: %s.\nSTDOUT=%s\nSTDERR=%s' %
                    (action, out, err))
        if action == 'start' or action == 'status':
            check_rc(cmd.returncode, stdout=out, stderr=err)

    if action == 'config':
        File('/etc/elasticsearch/elasticsearch.yml',
             content=Template('elasticsearch.j2'),
             owner="elasticsearch",
             group="elasticsearch",
             mode=0644)
Exemplo n.º 16
0
def cassandra(action=None):
  
  if action == 'start' or action == 'stop' or action == 'status':
    cmd=Popen(['service','cassandra',action],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    Logger.info('Cassandra action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err))
    if action == 'start' or action == 'status':
      check_rc(cmd.returncode,stdout=out,stderr=err)

  if action == 'config':
    import params
    File('/etc/cassandra/conf/cassandra.yaml',
      content=Template('cassandra.j2'),
      owner='cassandra',
      group='cassandra'
    )
    File('/etc/cassandra/conf/cassandra-rackdc.properties',
      content=Template('cassandra-rackdc.properties.j2'),
      owner='cassandra',
      group='cassandra'
    )
Exemplo n.º 17
0
def kibana(action=None):
  
  if action == 'start' or action == 'stop' or action == 'status':
    cmd=Popen(['service','kibana4',action],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    Logger.info('Kibana4 action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err))
    if action == 'start' or action == 'status':
      check_rc(cmd.returncode,stdout=out,stderr=err)

  if action == 'config' :
    import params
    Directory('/var/log/kibana',
      owner='kibana',
      group='kibana',
      mode=0550
    )
    File('/usr/lib/kibana4/config/kibana.yml',
      content=Template('kibana.j2'),
      owner='kibana',
      group='kibana'
    )
Exemplo n.º 18
0
def zkfc(action=None, format=False):
  if action == "configure":
    File("/etc/monit.d/hadoop-hdfs-zkfc",
      content=StaticFile("monit.d_hadoop-hdfs-zkfc"))
    File("/etc/monit.conf",
      content=StaticFile("monit.conf"))
 
  #First check monit is running and start if not
  executed = Popen(["service","monit","status"])
  out,err=executed.communicate()
  Logger.info("Checking monit status")
  Logger.info(out)
  Logger.info(err)
  rc = executed.returncode
  if rc > 0 or action=="configure":
    executed = Popen(["service","monit","restart"])
    out,err=executed.communicate()
    Logger.info("Restarting monit")
    Logger.info(out)
    Logger.info(err)


  if action == "start":
    # AC: this avoids a race condition at restart
    time.sleep(30)
    executed = Popen(["monit",action,"hadoop-hdfs-zkfc","-v"],stdout=PIPE,stderr=PIPE)
    out,err = executed.communicate()
    Logger.info("Monit Hdfs-zkfc: "+action)
    Logger.info(out)
    Logger.info(err)
    rc = executed.returncode
    check_rc(rc,out,err)

  if action == "stop":
    executed = Popen(["monit",action,"hadoop-hdfs-zkfc","-v"],stdout=PIPE,stderr=PIPE)
    out,err = executed.communicate()
    Logger.info("Monit Hdfs-zkfc: "+action)
    Logger.info(out)
    Logger.info(err)
    rc = executed.returncode
    check_rc(rc,out,err)

  if action == "status":
    executed = Popen(["service","hadoop-hdfs-zkfc",action],stdout=PIPE,stderr=PIPE)
    out,err = executed.communicate()
    Logger.info("Hdfs-zkfc: "+action)
    Logger.info(out)
    Logger.info(err)
    rc = executed.returncode
    check_rc(rc,out,err)
Exemplo n.º 19
0
def oozie(action=None,is_server=False):
  import params
  
  if action == 'install':
    cmd=Popen(['/usr/sbin/usermod','-a','-G','hadoop','oozie'],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    Logger.info('Oozie action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err))


  if action == 'start' or action == 'stop' or action == 'status':
    cmd=Popen(['service','oozie',action],stdout=PIPE,stderr=PIPE)
    out,err=cmd.communicate()
    Logger.info('Oozie action: %s.\nSTDOUT=%s\nSTDERR=%s' % (action,out,err))
    if action == 'start' or action == 'status':
      check_rc(cmd.returncode,stdout=out,stderr=err)

  if action == 'config' :
    File(params.oozie_config_dir + '/oozie-site.xml',
      content=Template('oozie-site.j2'),
      owner=params.oozie_user,
      group=params.oozie_group,
      mode=0644)

    File(params.oozie_config_dir+'/oozie-env.sh',
      content=Template('oozie-env.j2'),
      owner=params.oozie_user,
      group=params.oozie_group,
      mode=0644)

    File(params.oozie_config_dir+'/adminusers.txt',
      content=Template('adminusers.txt.j2'),
      owner=params.oozie_user,
      group=params.oozie_group,
      mode=0644)

    if is_server :
      #File('/usr/lib/oozie/libext/ext-2.2.1.zip',
      # content=StaticFile('ext-2.2.1.zip'))
      ## oozie expect ext-2.2 directory and looks to be hardcoded
      #extract_cmd=[ 'unzip', '/usr/lib/oozie/libext/ext-2.2.1.zip','-d','/usr/lib/oozie/libext/ext-2.2' ]
      #Popen(extract_cmd)
      extract_cmd=[ 'ln', '-s','/usr/share/java/mysql-connector-java.jar','/usr/lib/oozie/libext/mysql-connector-java.jar' ] 
      cmd=Popen(extract_cmd)
      out,err=cmd.communicate() 
      Logger.info("Creating mysql-connector-java.jar symbolic link in /usr/lib/oozie/libext/")
      Logger.info(out)
      Logger.info(err)
          
      Logger.info(params.oozie_jdbc_driver)
      if params.oozie_jdbc_driver == "com.mysql.jdbc.Driver":
        create_db_cmd = format('su --shell=/bin/bash -l oozie -c "source /etc/profile.d/java.sh && /usr/lib/oozie/bin/ooziedb.sh create -sqlfile oozie.sql -run"') 
      
      if params.oozie_jdbc_driver == "org.postgresql.Driver":
        pass
      
      if params.oozie_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
        pass

      cmd=Popen(create_db_cmd, shell=True)
      out,err=cmd.communicate()
      Logger.info("Installing the Oozie Schema in the DB")
      Logger.info(out)
      Logger.info(err)
Exemplo n.º 20
0
def hue(service=None, action=None):

    if action == "initdb":
        import params
        cmd = Popen(['/usr/lib/hue/build/env/bin/hue', 'syncdb', '--noinput'],
                    stdout=PIPE,
                    stderr=PIPE)
        out, err = cmd.communicate()
        Logger.info("Hue installation: sync db")
        Logger.info(out)
        Logger.info(err)
        cmd = Popen(['/usr/lib/hue/build/env/bin/hue', 'migrate'],
                    stdout=PIPE,
                    stderr=PIPE)
        out, err = cmd.communicate()
        Logger.info("Hue installation: migrate tables")
        Logger.info(out)
        Logger.info(err)

        Logger.info("Hue installation: Installation of kafka-hue")
        os.chmod('/usr/lib/hue/tools/app_reg/app_reg.py', 0755)
        os.chdir('/usr/lib/hue/apps')
        cmd = Popen([
            '../tools/app_reg/app_reg.py', '--install', 'kafka',
            '--relative-paths'
        ],
                    stdout=PIPE,
                    stderr=PIPE)
        out, err = cmd.communicate()
        Logger.info(out)
        Logger.info(err)
        Logger.info("Hue installation: Installation of storm-hue")
        cmd = Popen([
            '../tools/app_reg/app_reg.py', '--install', 'storm',
            '--relative-paths'
        ],
                    stdout=PIPE,
                    stderr=PIPE)
        out, err = cmd.communicate()
        Logger.info(out)
        Logger.info(err)

    if action == "config":
        import params
        #configurations = params.config['configurations']['spark']
        File(format(params.hue_conf_dir + "hue.ini"),
             content=Template("hue.ini.j2"),
             owner="root",
             group="root")

    if action == "start" or action == "stop":
        cmd = Popen(['service', 'hue', action], stdout=PIPE, stderr=PIPE)
        out, err = cmd.communicate()
        rc = cmd.returncode
        Logger.info("Hue service %s: %s" % (action, cmd.returncode == 0))

    if action == "status":
        from utils import check_rc
        cmd = Popen(['service', 'hue', action], stdout=PIPE, stderr=PIPE)
        out, err = cmd.communicate()
        rc = cmd.returncode
        Logger.info(rc)
        Logger.info("Hue service %s: %s" % (action, cmd.returncode == 0))

        check_rc(rc, stdout=out, stderr=err)
Exemplo n.º 21
0
def oozie(action=None, is_server=False):
    import params

    if action == 'install':
        cmd = Popen(['/usr/sbin/usermod', '-a', '-G', 'hadoop', 'oozie'],
                    stdout=PIPE,
                    stderr=PIPE)
        out, err = cmd.communicate()
        Logger.info('Oozie action: %s.\nSTDOUT=%s\nSTDERR=%s' %
                    (action, out, err))

    if action == 'start' or action == 'stop' or action == 'status':
        cmd = Popen(['service', 'oozie', action], stdout=PIPE, stderr=PIPE)
        out, err = cmd.communicate()
        Logger.info('Oozie action: %s.\nSTDOUT=%s\nSTDERR=%s' %
                    (action, out, err))
        if action == 'start' or action == 'status':
            check_rc(cmd.returncode, stdout=out, stderr=err)

    if action == 'config':
        File(params.oozie_config_dir + '/oozie-site.xml',
             content=Template('oozie-site.j2'),
             owner=params.oozie_user,
             group=params.oozie_group,
             mode=0644)

        File(params.oozie_config_dir + '/oozie-env.sh',
             content=Template('oozie-env.j2'),
             owner=params.oozie_user,
             group=params.oozie_group,
             mode=0644)

        File(params.oozie_config_dir + '/adminusers.txt',
             content=Template('adminusers.txt.j2'),
             owner=params.oozie_user,
             group=params.oozie_group,
             mode=0644)

        if is_server:
            #File('/usr/lib/oozie/libext/ext-2.2.1.zip',
            # content=StaticFile('ext-2.2.1.zip'))
            ## oozie expect ext-2.2 directory and looks to be hardcoded
            #extract_cmd=[ 'unzip', '/usr/lib/oozie/libext/ext-2.2.1.zip','-d','/usr/lib/oozie/libext/ext-2.2' ]
            #Popen(extract_cmd)
            extract_cmd = [
                'ln', '-s', '/usr/share/java/mysql-connector-java.jar',
                '/usr/lib/oozie/libext/mysql-connector-java.jar'
            ]
            cmd = Popen(extract_cmd)
            out, err = cmd.communicate()
            Logger.info(
                "Creating mysql-connector-java.jar symbolic link in /usr/lib/oozie/libext/"
            )
            Logger.info(out)
            Logger.info(err)

            Logger.info(params.oozie_jdbc_driver)
            if params.oozie_jdbc_driver == "com.mysql.jdbc.Driver":
                create_db_cmd = format(
                    'su --shell=/bin/bash -l oozie -c "source /etc/profile.d/java.sh && /usr/lib/oozie/bin/ooziedb.sh create -sqlfile oozie.sql -run"'
                )

            if params.oozie_jdbc_driver == "org.postgresql.Driver":
                pass

            if params.oozie_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
                pass

            cmd = Popen(create_db_cmd, shell=True)
            out, err = cmd.communicate()
            Logger.info("Installing the Oozie Schema in the DB")
            Logger.info(out)
            Logger.info(err)
Exemplo n.º 22
0
def flume(action = None):
  import params
  import flume_extra 

  
  if action == 'config':
    # remove previously defined meta's
    for n in find_expected_agent_names():
      os.unlink(os.path.join(params.flume_conf_dir, n, 'ambari-meta.json'))

    Directory(params.flume_conf_dir, recursive=True)
    Directory(params.flume_log_dir, owner=params.flume_user)

    File(os.path.join(params.flume_conf_dir, 'log4j.properties'),
      content=StaticFile("log4j.properties"))

    flume_agents = {}
    if params.flume_conf_content is not None:
      flume_agents = build_flume_topology(params.flume_conf_content)

    if params.flume_env_content is not None:
      flume_agents_env = build_flume_env(params.flume_env_content)

    for agent in flume_agents.keys():
      flume_meta_file = os.path.join(params.flume_conf_dir, 'ambari-meta_%s.json' % agent )
      File(flume_meta_file,
        content = json.dumps(ambari_meta(agent, flume_agents[agent])),
        mode = 0644)
      flume_env = flume_agents_env[agent] if flume_agents_env.has_key(agent) else flume_agents_env['default']
      File(os.path.join(params.flume_agent_conf_dir,'%s-env.sh' % agent),
        content = InlineTemplate(flume_env),
        mode = 0644)
      flume_agent_conf_file = os.path.join(params.flume_agent_conf_dir, '%s.conf' % agent)

      PropertiesFile(flume_agent_conf_file,
        properties=flume_agents[agent],
        mode = 0644)

    if params.flume_extra is not None:
      #import jsonschema
      #File('/tmp/flume_extra.schema',
      #  content=StaticFile("flume_extra.schema"))
      #flume_extra_schema = json.loads(open('/tmp/flume_extra.schema').read())

      for extra_file_raw in params.flume_extra.values():
        extra_file=json.loads(extra_file_raw)
        if not extra_file.has_key('path'):
          raise Exception("path field is required")
        File(extra_file['path'],
          mode = int(extra_file['mode'],8),
          owner = extra_file['owner'],
          group = extra_file['group'],
          content=extra_file['content']
        )

  if action == "start" or action == "stop" or action == "status":
    executed = Popen(["service","flume-agent",action],stdout=PIPE,stderr=PIPE)
    out,err = executed.communicate()
    if action == "status":
      from utils import check_rc
      rc = executed.returncode
      check_rc(rc,out,err)
    if action == "stop":
      filelist = glob.glob(params.flume_agent_conf_dir+"/*.conf")
      for f in filelist:
        os.remove(f)
      # Removes flume-env files  
      filelist = glob.glob(params.flume_agent_conf_dir+"/*.sh")
      for f in filelist:
        os.remove(f)