Ejemplo n.º 1
0
def process_command_build(args):
    if not args.offline or not args.package_dir:
        Log.print_critical("ERROR: Building Minos offline needs to specify " \
          "the arguments '--offline' and the offline packages directory " \
          "'--package_dir' explicitly")

    build_utils.pip_install_offline(args.package_dir)
Ejemplo n.º 2
0
def execute_command(cmd, log_message="", error_message=""):
  if log_message:
    Log.print_info(log_message)
  try:
    subprocess.check_call(cmd)
  except BaseException, e:
    Log.print_critical('ERROR: %s' % error_message if error_message else str(e))
Ejemplo n.º 3
0
def start_owl_monitor():
  owl_monitor_http_port = build_utils.get_build_info_option('owl', 'owl_port')
  if not owl_monitor_http_port:
    Log.print_critical("Owl port is null")

  build_utils.start_daemon_process('Owl monitor', OWL_MONITOR_PID_FILE,
    OWL_ROOT, './start_owl_monitor.sh', owl_monitor_http_port)
Ejemplo n.º 4
0
def _create_deployment_directory(deploy_path):
    # Create deployment directory for supervisor
    for dir in SUPERVISOR_DEPLOYMENT_DIRS:
        deploy_dir = os.path.join(deploy_path, dir)
        if not os.path.exists(deploy_dir):
            Log.print_info('Creating the %s root %s' % (dir, deploy_dir))
            os.makedirs(deploy_dir)
Ejemplo n.º 5
0
def execute_command(cmd, log_message="", error_message=""):
    if log_message:
        Log.print_info(log_message)
    try:
        subprocess.check_call(cmd)
    except BaseException, e:
        Log.print_critical("ERROR: %s" % error_message if error_message else str(e))
Ejemplo n.º 6
0
def process_command_build(args):
  if not args.offline or not args.package_dir:
    Log.print_critical("ERROR: Building Minos offline needs to specify " \
      "the arguments '--offline' and the offline packages directory " \
      "'--package_dir' explicitly")

  build_utils.pip_install_offline(args.package_dir)
Ejemplo n.º 7
0
def check_command_output(cmd, error_message="", skip_error=False):
  try:
    out = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
  except BaseException, e:
    if skip_error:
      return 0
    else:
      Log.print_critical('ERROR: %s' % error_message if error_message else str(e))
Ejemplo n.º 8
0
def check_command_output(cmd, error_message="", skip_error=False):
    try:
        out = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
    except BaseException, e:
        if skip_error:
            return 0
        else:
            Log.print_critical("ERROR: %s" % error_message if error_message else str(e))
Ejemplo n.º 9
0
def configure_owl_config(args):
    Log.print_info("Configure owl config file: %s" % OWL_CONFIG_FILE)
    owl_config_dict = {
        'owl_ip': args.owl_ip,
        'opentsdb_port': OPENTSDB_PORT,
    }
    build_utils.generate_config_file(OWL_CONFIG_TEMPLATE, OWL_CONFIG_FILE,
                                     owl_config_dict)
Ejemplo n.º 10
0
def configure_owl_config(args):
  Log.print_info("Configure owl config file: %s" % OWL_CONFIG_FILE)
  owl_config_dict = {
    'owl_ip': args.owl_ip,
    'opentsdb_port': OPENTSDB_PORT,
  }
  build_utils.generate_config_file(OWL_CONFIG_TEMPLATE,
    OWL_CONFIG_FILE, owl_config_dict)
Ejemplo n.º 11
0
def configure_mysql_for_owl(database_name, host='localhost', port='3306'):
  Log.print_info("Configuring mysql for owl in %s" % OWL_SETTING_FILE)
  owl_setting_dict = {
    'DATABASE': database_name,
    'HOST': host,
    'PORT': port,
  }
  build_utils.generate_config_file(OWL_SETTING_TEMPLATE,
    OWL_SETTING_FILE, owl_setting_dict)
Ejemplo n.º 12
0
def configure_mysql_for_owl(database_name, host='localhost', port='3306'):
    Log.print_info("Configuring mysql for owl in %s" % OWL_SETTING_FILE)
    owl_setting_dict = {
        'DATABASE': database_name,
        'HOST': host,
        'PORT': port,
    }
    build_utils.generate_config_file(OWL_SETTING_TEMPLATE, OWL_SETTING_FILE,
                                     owl_setting_dict)
Ejemplo n.º 13
0
def start_owl_monitor():
    owl_monitor_http_port = build_utils.get_build_info_option(
        'owl', 'owl_port')
    if not owl_monitor_http_port:
        Log.print_critical("Owl port is null")

    build_utils.start_daemon_process('Owl monitor', OWL_MONITOR_PID_FILE,
                                     OWL_ROOT, './start_owl_monitor.sh',
                                     owl_monitor_http_port)
Ejemplo n.º 14
0
def configure_opentsdb_collector(owl_port):
  # Configure opentsdb collector config file
  Log.print_info("Configuring opentsdb collector in %s" %
    OPENTSDB_COLLECTOR_CONFIG_FILE)
  opentsdb_collector_dict = {
    'owl_monitor_http_port': owl_port,
    'tsdb': OPENTSDB_BIN_PATH,
  }
  build_utils.generate_config_file(OPENTSDB_COLLECTOR_CONFIG_TEMPLATE,
    OPENTSDB_COLLECTOR_CONFIG_FILE, opentsdb_collector_dict)
Ejemplo n.º 15
0
def _deploy_supervisor(args, deploy_path):
    Log.print_info("Deploying supervisor in %s" % SUPERVISOR_ROOT)

    # Generate supervisord.conf according to the deploying information
    deploy_info_dict = {
        'DEPLOY_PATH': deploy_path,
        'PACKAGE_SERVER': "%s:%d" % (args.tank_ip, args.tank_port),
    }
    build_utils.generate_config_file(SUPERVISOR_CONFIG_TEMPLATE,
                                     SUPERVISOR_CONFIG_FILE, deploy_info_dict)
Ejemplo n.º 16
0
def configure_opentsdb_collector(owl_port):
    # Configure opentsdb collector config file
    Log.print_info("Configuring opentsdb collector in %s" %
                   OPENTSDB_COLLECTOR_CONFIG_FILE)
    opentsdb_collector_dict = {
        'owl_monitor_http_port': owl_port,
        'tsdb': OPENTSDB_BIN_PATH,
    }
    build_utils.generate_config_file(OPENTSDB_COLLECTOR_CONFIG_TEMPLATE,
                                     OPENTSDB_COLLECTOR_CONFIG_FILE,
                                     opentsdb_collector_dict)
Ejemplo n.º 17
0
def generate_hbase_configuration():
    Log.print_info("Modify hbase-site.xml in %s" % HBASE_CONFIG_ROOT)
    cmd = "hbase_rootdir=${TMPDIR-'/tmp'}/tsdhbase;" \
      "iface=lo`uname | sed -n s/Darwin/0/p`; echo $hbase_rootdir,$iface"
    hbase_rootdir, iface = build_utils.get_command_variable(cmd).split(',')

    configuration_dict = {
        'hbase_rootdir': hbase_rootdir,
        'iface': iface,
    }
    build_utils.generate_config_file(HBASE_CONFIG_TEMPLATE, HBASE_CONFIG_FILE,
                                     configuration_dict)
Ejemplo n.º 18
0
def create_owl_database(args, database_name, host="", port=""):
  root_pwd = getpass.getpass("Please enter password of the Mysql root user: "******"ERROR: %s" % str(e))
Ejemplo n.º 19
0
def generate_hbase_configuration():
  Log.print_info("Modify hbase-site.xml in %s" % HBASE_CONFIG_ROOT)
  cmd = "hbase_rootdir=${TMPDIR-'/tmp'}/tsdhbase;" \
    "iface=lo`uname | sed -n s/Darwin/0/p`; echo $hbase_rootdir,$iface"
  hbase_rootdir, iface = build_utils.get_command_variable(cmd).split(',')

  configuration_dict = {
    'hbase_rootdir': hbase_rootdir,
    'iface': iface,
  }
  build_utils.generate_config_file(HBASE_CONFIG_TEMPLATE,
    HBASE_CONFIG_FILE, configuration_dict)
Ejemplo n.º 20
0
def _build(args):
    if args.owl_ip == '127.0.0.1' or args.owl_port == 0:
        Log.print_critical("ERROR: Building owl needs to specify the localhost ip " \
          "with '--owl_ip' and the owl monitor http port with '--owl_port'")

    Log.print_info("Building owl")
    # Check and install prerequisite python libraries
    Log.print_info("Check and install prerequisite python libraries")
    build_utils.check_and_install_modules(OWL_PREREQUISITE_PYTHON_LIBS)

    check_third_party_tool_exists("gnuplot")
    check_third_party_tool_exists("mysql")
    create_and_configure_mysql_for_owl(args)
    create_django_database()

    # Deploy hbase
    if not args.skip_setup_hbase:
        build_hbase()
        start_hbase()

    # Deploy opentsdb
    deploy_opentsdb()
    if not args.skip_setup_hbase:
        start_opentsdb()

    # Configure opentsdb collector
    configure_opentsdb_collector(str(args.owl_port))
    # Configure owl config
    configure_owl_config(args)

    # Output build information
    build_utils.output_build_info(args.component, 'owl_port', args.owl_port)
    build_utils.output_build_info(args.component, 'build_status', 'success')
    Log.print_info("The component %s is built successfully" % args.component)
Ejemplo n.º 21
0
def _build(args):
  if args.owl_ip == '127.0.0.1' or args.owl_port == 0:
    Log.print_critical("ERROR: Building owl needs to specify the localhost ip " \
      "with '--owl_ip' and the owl monitor http port with '--owl_port'")

  Log.print_info("Building owl")
  # Check and install prerequisite python libraries
  Log.print_info("Check and install prerequisite python libraries")
  build_utils.check_and_install_modules(OWL_PREREQUISITE_PYTHON_LIBS)

  check_third_party_tool_exists("gnuplot")
  check_third_party_tool_exists("mysql")
  create_and_configure_mysql_for_owl(args)
  create_django_database()

  # Deploy hbase
  if not args.skip_setup_hbase:
    build_hbase()
    start_hbase()

  # Deploy opentsdb
  deploy_opentsdb()
  if not args.skip_setup_hbase:
    start_opentsdb()

  # Configure opentsdb collector
  configure_opentsdb_collector(str(args.owl_port))
  # Configure owl config
  configure_owl_config(args)

  # Output build information
  build_utils.output_build_info(args.component, 'owl_port', args.owl_port)
  build_utils.output_build_info(args.component, 'build_status', 'success')
  Log.print_info("The component %s is built successfully" % args.component)
Ejemplo n.º 22
0
def _build(args):
    if args.tank_ip == TANK_DEFAULT_IP:
        Log.print_critical("ERROR: Building supervisor needs to specify the package server " \
          "with '--tank_ip' and '--tank_port'")

    Log.print_info("Building supervisor")
    # Check and install prerequisite python libraries
    Log.print_info("Check and install prerequisite python libraries")
    build_utils.check_and_install_modules(SUPERVISOR_PREREQUISITE_PYTHON_LIBS)

    # Create deployment directory
    deploy_path = raw_input("Please input the root directory to deploy services " \
      "(default: /home/%s): " % getpass.getuser())

    if deploy_path:
        deploy_path = os.path.abspath(os.path.realpath(deploy_path))
    else:
        deploy_path = "/home/%s" % getpass.getuser()
    _create_deployment_directory(deploy_path)

    # Deploy supervisor
    _deploy_supervisor(args, deploy_path)

    # Output build information
    build_utils.output_build_info(args.component, 'build_status', 'success')
    Log.print_info("The component %s is built successfully" % args.component)
Ejemplo n.º 23
0
def create_owl_database(args, database_name, host="", port=""):
    root_pwd = getpass.getpass(
        "Please enter password of the Mysql root user: "******"ERROR: %s" % str(e))
Ejemplo n.º 24
0
def create_and_configure_mysql_for_owl(args):
    if build_utils.get_build_info_option('owl', 'mysql') == 'created':
        return
    # Support both local and remote database
    choice = raw_input("Please choose Mysql server you want to use " \
      "(1 for Local, 2 for Remote): ")
    owl_prefix = raw_input("Please enter the prefix of your owl database name " \
      "(default: %s): " % getpass.getuser())
    if not owl_prefix:
        owl_prefix = getpass.getuser()
    database_name = "%s_owl" % owl_prefix

    # Using local mysql
    if int(choice) == 1:
        # Check mysql server is running
        cmd = 'ps -ef | grep mysqld | grep -v grep'
        error_message = "Please start mysql server firstly"
        build_utils.check_command_output(cmd, error_message=error_message)
        # Create owl database
        create_owl_database(args, database_name)
        # Configure mysql for owl
        configure_mysql_for_owl(database_name)

    # Using remote mysql
    elif int(choice) == 2:
        remote_address = raw_input("Please input the remote mysql " \
          "server's address (ip:port): ")
        remote_host, remote_port = remote_address.split(":")
        # Create owl database
        create_owl_database(args,
                            database_name,
                            host=remote_host,
                            port=remote_port)
        # Configure mysql for owl
        configure_mysql_for_owl(database_name, remote_host, remote_port)
    else:
        Log.print_critical("ERROR: invalid choice")

    # Mark mysql database created
    build_utils.output_build_info('owl', 'mysql', 'created')
Ejemplo n.º 25
0
def start_daemon_process(process_name, pid_file, dest_path, script, *extra_para):
  if check_process_is_running(pid_file):
    Log.print_warning("%s is running, please check" % process_name)
    return

  Log.print_info("Starting %s" % process_name)
  exec_daemon_script(dest_path, script, *extra_para)
  Log.print_success("Start %s success" % process_name)
Ejemplo n.º 26
0
def stop_daemon_process(process_name, pid_file, dest_path, script):
    if not check_process_is_running(pid_file):
        Log.print_warning("%s is not running" % process_name)
        return

    Log.print_info("Stopping %s" % process_name)
    exec_daemon_script(dest_path, script, str(get_process_running_pid(pid_file)))
    Log.print_success("Stop %s success" % process_name)
Ejemplo n.º 27
0
def start_daemon_process(process_name, pid_file, dest_path, script, *extra_para):
    if check_process_is_running(pid_file):
        Log.print_warning("%s is running, please check" % process_name)
        return

    Log.print_info("Starting %s" % process_name)
    exec_daemon_script(dest_path, script, *extra_para)
    Log.print_success("Start %s success" % process_name)
Ejemplo n.º 28
0
def stop_daemon_process(process_name, pid_file, dest_path, script):
  if not check_process_is_running(pid_file):
    Log.print_warning("%s is not running" % process_name)
    return

  Log.print_info("Stopping %s" % process_name)
  exec_daemon_script(dest_path, script, str(get_process_running_pid(pid_file)))
  Log.print_success("Stop %s success" % process_name)
Ejemplo n.º 29
0
def create_and_configure_mysql_for_owl(args):
  if build_utils.get_build_info_option('owl', 'mysql') == 'created':
    return
  # Support both local and remote database
  choice = raw_input("Please choose Mysql server you want to use " \
    "(1 for Local, 2 for Remote): ")
  owl_prefix = raw_input("Please enter the prefix of your owl database name " \
    "(default: %s): " % getpass.getuser())
  if not owl_prefix:
    owl_prefix = getpass.getuser()
  database_name = "%s_owl" % owl_prefix

  # Using local mysql
  if int(choice) == 1:
    # Check mysql server is running
    cmd = 'ps -ef | grep mysqld | grep -v grep'
    error_message = "Please start mysql server firstly"
    build_utils.check_command_output(cmd, error_message=error_message)
    # Create owl database
    create_owl_database(args, database_name)
    # Configure mysql for owl
    configure_mysql_for_owl(database_name)

  # Using remote mysql
  elif int(choice) == 2:
    remote_address = raw_input("Please input the remote mysql " \
      "server's address (ip:port): ")
    remote_host, remote_port = remote_address.split(":")
    # Create owl database
    create_owl_database(args, database_name, host=remote_host, port=remote_port)
    # Configure mysql for owl
    configure_mysql_for_owl(database_name, remote_host, remote_port)
  else:
    Log.print_critical("ERROR: invalid choice")

  # Mark mysql database created
  build_utils.output_build_info('owl', 'mysql', 'created')
Ejemplo n.º 30
0
def _build(args):
  Log.print_info("Building tank server")

  # Check and install prerequisite python libraries
  Log.print_info("Check and install prerequisite python libraries")
  build_utils.check_and_install_modules(TANK_PREREQUISITE_PYTHON_LIBS)

  # Output build information
  if args.tank_ip != TANK_DEFAULT_IP or args.tank_port != TANK_DEFAULT_PORT:
    build_utils.output_build_info(args.component, 'tank_ip', args.tank_ip)
    build_utils.output_build_info(args.component, 'tank_port', args.tank_port)

  build_utils.output_build_info(args.component, 'build_status', 'success')
  Log.print_info("The component %s is built successfully" % args.component)
Ejemplo n.º 31
0
def _build(args):
    Log.print_info("Building tank server")

    # Check and install prerequisite python libraries
    Log.print_info("Check and install prerequisite python libraries")
    build_utils.check_and_install_modules(TANK_PREREQUISITE_PYTHON_LIBS)

    # Output build information
    if args.tank_ip != TANK_DEFAULT_IP or args.tank_port != TANK_DEFAULT_PORT:
        build_utils.output_build_info(args.component, 'tank_ip', args.tank_ip)
        build_utils.output_build_info(args.component, 'tank_port',
                                      args.tank_port)

    build_utils.output_build_info(args.component, 'build_status', 'success')
    Log.print_info("The component %s is built successfully" % args.component)
Ejemplo n.º 32
0
def process_command_start(args):
    build_tool = COMPONENT_BUILD_TOOL_MAP.get(args.component)
    if build_tool:
        return build_tool.start(args)
    Log.print_critical("Unknown component name: %s.", args.component)
Ejemplo n.º 33
0
def build_client():
  # Check and install prerequisite python libraries
  Log.print_info("Check and install prerequisite python libraries")
  build_utils.check_and_install_modules(CLIENT_PREREQUISITE_PYTHON_LIBS)
  Log.print_success("Build Minos client success")
Ejemplo n.º 34
0
def process_command_stop(args):
  build_tool = COMPONENT_BUILD_TOOL_MAP.get(args.component)
  if build_tool:
    return build_tool.stop(args)
  Log.print_critical("Unknown component name: %s.", args.component)
Ejemplo n.º 35
0
def stop(args):
    input = raw_input("Do you really want to do this ? (y/n)")
    if check_input(input):
        _do_stop()
    else:
        Log.print_info("Skip stopping owl component")
Ejemplo n.º 36
0
def get_process_running_pid(pid_file):
    try:
        with open(pid_file) as fp:
            return int(fp.readline())
    except ValueError, e:
        Log.print_critical("Error: Pid file %s is null" % pid_file)
Ejemplo n.º 37
0
def build_client():
    # Check and install prerequisite python libraries
    Log.print_info("Check and install prerequisite python libraries")
    build_utils.check_and_install_modules(CLIENT_PREREQUISITE_PYTHON_LIBS)
    Log.print_success("Build Minos client success")
Ejemplo n.º 38
0
def stop(args):
  input = raw_input("Do you really want to do this ? (y/n)")
  if check_input(input):
    _do_stop()
  else:
    Log.print_info("Skip stopping owl component")
Ejemplo n.º 39
0
def get_process_running_pid(pid_file):
    try:
        with open(pid_file) as fp:
            return int(fp.readline())
    except ValueError, e:
        Log.print_critical("Error: Pid file %s is null" % pid_file)