Пример #1
0
def heron_class(class_name, lib_jars, extra_jars=[], args=[], javaDefines=[]):
  # Format all java -D options that need to be passed while running
  # the class locally.
  javaOpts = map(lambda opt: '-D' + opt, javaDefines)

  # Construct the command line for the sub process to run
  # Because of the way Python execute works,
  # the java opts must be passed as part of the list
  all_args = [
      utils.get_java_path(), "-client", "-Xmx1g", opts.get_heron_config()] + \
      javaOpts + \
      ["-cp", utils.get_classpath(lib_jars + extra_jars),
  ]

  all_args += [class_name] + list(args)

  # print the verbose message
  if opts.verbose():
    print('$> %s' % ' '.join(all_args))

  # invoke the command with subprocess and print error message, if any
  if not opts.trace_execution():
    status = subprocess.call(all_args)
    if status != 0:
      err_str = "User main failed with status %d. Bailing out..." % status
      raise RuntimeError(err_str)
Пример #2
0
def heron_class(class_name, lib_jars, extra_jars=[], args=[], javaDefines=[]):
    # Format all java -D options that need to be passed while running
    # the class locally.
    javaOpts = map(lambda opt: '-D' + opt, javaDefines)

    # Construct the command line for the sub process to run
    # Because of the way Python execute works,
    # the java opts must be passed as part of the list
    all_args = [
        utils.get_java_path(), "-client", "-Xmx1g", opts.get_heron_config()] + \
        javaOpts + \
        ["-cp", utils.get_classpath(lib_jars + extra_jars),
    ]

    all_args += [class_name] + list(args)

    # print the verbose message
    if opts.verbose():
        print('$> %s' % ' '.join(all_args))

    # invoke the command with subprocess and print error message, if any
    if not opts.trace_execution():
        status = subprocess.call(all_args)
        if status != 0:
            err_str = "User main failed with status %d. Bailing out..." % status
            raise RuntimeError(err_str)
Пример #3
0
def main():
  '''
  Run the command
  :return:
  '''
  # verify if the environment variables are correctly set
  check_environment()

  # create the argument parser
  parser = create_parser()

  # if no argument is provided, print help and exit
  if len(sys.argv[1:]) == 0:
    parser.print_help()
    return 0

  # insert the boolean values for some of the options
  sys.argv = parse.insert_bool_values(sys.argv)

  # parse the args
  args, unknown_args = parser.parse_known_args()
  command_line_args = vars(args)

  try:
    if command_line_args['verbose']:
      opts.set_verbose()
    if command_line_args['trace_execution']:
      opts.set_trace_execution()
  except:
    pass

  # command to be execute
  command = command_line_args['subcommand']

  # file resources to be cleaned when exit
  files = []

  if command != 'help' and command != 'version':
    command_line_args = extract_common_args(command, parser, command_line_args)
    # bail out if args are empty
    if not command_line_args:
      return 1
    # register dirs cleanup function during exit
    files.append(command_line_args['override_config_file'])

  atexit.register(cleanup, files)

  # print the input parameters, if verbose is enabled
  if opts.verbose():
    print command_line_args

  start = time.time()
  retcode = run(command, parser, command_line_args, unknown_args)
  end = time.time()

  if command != 'help':
    sys.stdout.flush()
    Log.info('Elapsed time: %.3fs.' % (end - start))

  return 0 if retcode else 1
Пример #4
0
def main():

    # verify if the environment variables are correctly set
    check_environment()

    # create the argument parser
    parser = create_parser()

    # if no argument is provided, print help and exit
    if len(sys.argv[1:]) == 0:
        parser.print_help()
        return 0

    # insert the boolean values for some of the options
    sys.argv = parse.insert_bool_values(sys.argv)

    # parse the args
    args, unknown_args = parser.parse_known_args()
    command_line_args = vars(args)

    try:
        if command_line_args['verbose']:
            opts.set_verbose()
        if command_line_args['trace_execution']:
            opts.set_trace_execution()
    except:
        pass

    # command to be execute
    command = command_line_args['subcommand']

    # file resources to be cleaned when exit
    files = []

    if command != 'help' and command != 'version':
        command_line_args = extract_common_args(command, parser,
                                                command_line_args)
        # bail out if args are empty
        if not command_line_args:
            return 1
        # register dirs cleanup function during exit
        files.append(command_line_args['override_config_file'])

    atexit.register(cleanup, files)

    # print the input parameters, if verbose is enabled
    if opts.verbose():
        print command_line_args

    start = time.time()
    retcode = run(command, parser, command_line_args, unknown_args)
    end = time.time()

    if command != 'help':
        sys.stdout.flush()
        Log.info('Elapsed time: %.3fs.' % (end - start))

    return 0 if retcode == True else 1
Пример #5
0
def run(command, parser, cl_args, unknown_args, action):
    '''
  helper function to take action on topologies
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :param action:        description of action taken
  :return:
  '''
    try:
        topology_name = cl_args['topology-name']

        new_args = [
            "--cluster",
            cl_args['cluster'],
            "--role",
            cl_args['role'],
            "--environment",
            cl_args['environ'],
            "--heron_home",
            utils.get_heron_dir(),
            "--config_path",
            cl_args['config_path'],
            "--override_config_file",
            cl_args['override_config_file'],
            "--release_file",
            utils.get_heron_release_file(),
            "--topology_name",
            topology_name,
            "--command",
            command,
        ]

        if opts.verbose():
            new_args.append("--verbose")

        lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
                                        jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
                            lib_jars,
                            extra_jars=[],
                            args=new_args)

    except Exception:
        Log.error('Failed to %s \'%s\'' % (action, topology_name))
        return False

    Log.info('Successfully %s \'%s\'' % (action, topology_name))
    return True
Пример #6
0
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
  '''
  Launch a topology given topology jar, its definition file and configurations
  :param cl_args:
  :param tmp_dir:
  :param topology_file:
  :param topology_defn_file:
  :return:
  '''
  # get the normalized path for topology.tar.gz
  topology_pkg_path = utils.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))

  # get the release yaml file
  release_yaml_file = utils.get_heron_release_file()

  # create a tar package with the cluster configuration and generated config files
  config_path = cl_args['config_path']
  tar_pkg_files = [topology_file, topology_defn_file]
  generated_config_files = [release_yaml_file, cl_args['override_config_file']]

  utils.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)

  # pass the args to submitter main
  args = [
      "--cluster", cl_args['cluster'],
      "--role", cl_args['role'],
      "--environment", cl_args['environ'],
      "--heron_home", utils.get_heron_dir(),
      "--config_path", config_path,
      "--override_config_file", cl_args['override_config_file'],
      "--release_file", release_yaml_file,
      "--topology_package", topology_pkg_path,
      "--topology_defn", topology_defn_file,
      "--topology_jar", topology_file
  ]

  if opts.verbose():
    args.append("--verbose")

  lib_jars = utils.get_heron_libs(
      jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
  )

  # invoke the submitter to submit and launch the topology
  execute.heron_class(
      'com.twitter.heron.scheduler.SubmitterMain',
      lib_jars,
      extra_jars=[],
      args=args,
      java_defines=[]
  )
Пример #7
0
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
    '''
  Launch a topology given topology jar, its definition file and configurations
  :param cl_args:
  :param tmp_dir:
  :param topology_file:
  :param topology_defn_file:
  :return:
  '''
    # get the normalized path for topology.tar.gz
    topology_pkg_path = utils.normalized_class_path(
        os.path.join(tmp_dir, 'topology.tar.gz'))

    # get the release yaml file
    release_yaml_file = utils.get_heron_release_file()

    # create a tar package with the cluster configuration and generated config files
    config_path = cl_args['config_path']
    tar_pkg_files = [topology_file, topology_defn_file]
    generated_config_files = [
        release_yaml_file, cl_args['override_config_file']
    ]

    utils.create_tar(topology_pkg_path, tar_pkg_files, config_path,
                     generated_config_files)

    # pass the args to submitter main
    args = [
        "--cluster", cl_args['cluster'], "--role", cl_args['role'],
        "--environment", cl_args['environ'], "--heron_home",
        utils.get_heron_dir(), "--config_path", config_path,
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", release_yaml_file, "--topology_package",
        topology_pkg_path, "--topology_defn", topology_defn_file,
        "--topology_jar", topology_file
    ]

    if opts.verbose():
        args.append("--verbose")

    lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
                                    jars.uploader_jars() +
                                    jars.statemgr_jars() + jars.packing_jars())

    # invoke the submitter to submit and launch the topology
    execute.heron_class('com.twitter.heron.scheduler.SubmitterMain',
                        lib_jars,
                        extra_jars=[],
                        args=args,
                        java_defines=[])
Пример #8
0
def heron_class(class_name,
                lib_jars,
                extra_jars=None,
                args=None,
                java_defines=None):
    '''
  Execute a heron class given the args and the jars needed for class path
  :param class_name:
  :param lib_jars:
  :param extra_jars:
  :param args:
  :param javaDefines:
  :return:
  '''
    # default optional params to empty list if not provided
    if extra_jars is None:
        extra_jars = []
    if args is None:
        args = []
    if java_defines is None:
        java_defines = []

    # Format all java -D options that need to be passed while running
    # the class locally.
    java_opts = ['-D' + opt for opt in java_defines]

    # Construct the command line for the sub process to run
    # Because of the way Python execute works,
    # the java opts must be passed as part of the list
    all_args = [utils.get_java_path(), "-client", "-Xmx1g", opts.get_heron_config()] + \
               java_opts + \
               ["-cp", utils.get_classpath(lib_jars + extra_jars)]

    all_args += [class_name] + list(args)

    # print the verbose message
    if opts.verbose():
        print '$> %s' % ' '.join(all_args)

    # invoke the command with subprocess and print error message, if any
    if not opts.trace_execution():
        status = subprocess.call(all_args)
        if status != 0:
            err_str = "User main failed with status %d. Bailing out..." % status
            raise RuntimeError(err_str)
Пример #9
0
def run(command, parser, cl_args, unknown_args):

    try:
        topology_name = cl_args['topology-name']

        new_args = [
            "--cluster",
            cl_args['cluster'],
            "--role",
            cl_args['role'],
            "--environment",
            cl_args['environ'],
            "--heron_home",
            utils.get_heron_dir(),
            "--config_path",
            cl_args['config_path'],
            "--override_config_file",
            cl_args['override_config_file'],
            "--release_file",
            utils.get_heron_release_file(),
            "--topology_name",
            topology_name,
            "--command",
            command,
        ]

        if opts.verbose():
            new_args.append("--verbose")

        lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
                                        jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
                            lib_jars,
                            extra_jars=[],
                            args=new_args)

    except Exception as ex:
        print 'Error: %s' % str(ex)
        Log.error('Failed to activate topology \'%s\'' % topology_name)
        return False

    Log.info('Successfully activated topology \'%s\'' % topology_name)
    return True
Пример #10
0
def run(command, parser, cl_args, unknown_args, action):
  '''
  helper function to take action on topologies
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :param action:        description of action taken
  :return:
  '''
  try:
    topology_name = cl_args['topology-name']

    new_args = [
        "--cluster", cl_args['cluster'],
        "--role", cl_args['role'],
        "--environment", cl_args['environ'],
        "--heron_home", utils.get_heron_dir(),
        "--config_path", cl_args['config_path'],
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", utils.get_heron_release_file(),
        "--topology_name", topology_name,
        "--command", command,
    ]

    if opts.verbose():
      new_args.append("--verbose")

    lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

    # invoke the runtime manager to kill the topology
    execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args=new_args
    )

  except Exception:
    Log.error('Failed to %s \'%s\'' % (action, topology_name))
    return False

  Log.info('Successfully %s \'%s\'' % (action, topology_name))
  return True
Пример #11
0
def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=None):
  '''
  Execute a heron class given the args and the jars needed for class path
  :param class_name:
  :param lib_jars:
  :param extra_jars:
  :param args:
  :param javaDefines:
  :return:
  '''
  # default optional params to empty list if not provided
  if extra_jars is None:
    extra_jars = []
  if args is None:
    args = []
  if java_defines is None:
    java_defines = []

  # Format all java -D options that need to be passed while running
  # the class locally.
  java_opts = ['-D' + opt for opt in java_defines]

  # Construct the command line for the sub process to run
  # Because of the way Python execute works,
  # the java opts must be passed as part of the list
  all_args = [utils.get_java_path(), "-client", "-Xmx1g", opts.get_heron_config()] + \
             java_opts + \
             ["-cp", utils.get_classpath(lib_jars + extra_jars)]

  all_args += [class_name] + list(args)

  # print the verbose message
  if opts.verbose():
    print '$> %s' % ' '.join(all_args)

  # invoke the command with subprocess and print error message, if any
  if not opts.trace_execution():
    status = subprocess.call(all_args)
    if status != 0:
      err_str = "User main failed with status %d. Bailing out..." % status
      raise RuntimeError(err_str)
Пример #12
0
def run(command, parser, cl_args, unknown_args):
    try:
        topology_name = cl_args["topology-name"]

        new_args = [
            "--cluster",
            cl_args["cluster"],
            "--role",
            cl_args["role"],
            "--environment",
            cl_args["environ"],
            "--heron_home",
            utils.get_heron_dir(),
            "--config_path",
            cl_args["config_path"],
            "--override_config_file",
            cl_args["override_config_file"],
            "--release_file",
            utils.get_heron_release_file(),
            "--topology_name",
            topology_name,
            "--command",
            command,
        ]

        if opts.verbose():
            new_args.append("--verbose")

        lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class("com.twitter.heron.scheduler.RuntimeManagerMain", lib_jars, extra_jars=[], args=new_args)

    except Exception as ex:
        Log.error("Failed to kill topology '%s'" % topology_name)
        return False

    Log.info("Successfully killed topology '%s'" % topology_name)
    return True
Пример #13
0
def run(command, parser, cl_args, unknown_args):

  try:
    topology_name = cl_args['topology-name']

    new_args = [
        "--cluster", cl_args['cluster'],
        "--role", cl_args['role'],
        "--environment", cl_args['environ'],
        "--heron_home", utils.get_heron_dir(),
        "--config_path", cl_args['config_path'],
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", utils.get_heron_release_file(),
        "--topology_name", topology_name,
        "--command", command,
    ]

    if opts.verbose():
      new_args.append("--verbose")

    lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

    # invoke the runtime manager to kill the topology
    execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args= new_args
    )

  except Exception as ex:
    print 'Error: %s' % str(ex)
    Log.error('Failed to deactivate topology \'%s\'' % topology_name)
    return False

  Log.info('Successfully deactivated topology \'%s\'' % topology_name)
  return True