Пример #1
0
def submit_fatjar(cl_args, unknown_args, tmp_dir):

  # execute main of the topology to create the topology definition
  topology_file = cl_args['topology-file-name']
  try:
    execute.heron_class(
      cl_args['topology-class-name'],
      utils.get_heron_libs(jars.topology_jars()),
      extra_jars = [topology_file],
      args = tuple(unknown_args),
      javaDefines = cl_args['javaDefines'])

  except Exception as ex:
    Log.error("Unable to execute topology main class")
    return False

  try:
    launch_topologies(cl_args, topology_file, tmp_dir)

  except Exception as ex:
    return False

  finally:
    shutil.rmtree(tmp_dir)

  return True
Пример #2
0
def run(command, parser, cl_args, unknown_args, action):
    '''
  helper function to take action on topologies
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :param action:        description of action taken
  :return:
  '''
    try:
        topology_name = cl_args['topology-name']

        new_args = [
            "--cluster",
            cl_args['cluster'],
            "--role",
            cl_args['role'],
            "--environment",
            cl_args['environ'],
            "--heron_home",
            config.get_heron_dir(),
            "--config_path",
            cl_args['config_path'],
            "--override_config_file",
            cl_args['override_config_file'],
            "--release_file",
            config.get_heron_release_file(),
            "--topology_name",
            topology_name,
            "--command",
            command,
        ]

        if Log.getEffectiveLevel() == logging.DEBUG:
            new_args.append("--verbose")

        lib_jars = config.get_heron_libs(jars.scheduler_jars() +
                                         jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
                            lib_jars,
                            extra_jars=[],
                            args=new_args)

    except Exception:
        Log.error('Failed to %s \'%s\'' % (action, topology_name))
        return False

    Log.info('Successfully %s \'%s\'' % (action, topology_name))
    return True
Пример #3
0
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
  '''
  Launch a topology given topology jar, its definition file and configurations
  :param cl_args:
  :param tmp_dir:
  :param topology_file:
  :param topology_defn_file:
  :return:
  '''
  # get the normalized path for topology.tar.gz
  topology_pkg_path = utils.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))

  # get the release yaml file
  release_yaml_file = utils.get_heron_release_file()

  # create a tar package with the cluster configuration and generated config files
  config_path = cl_args['config_path']
  tar_pkg_files = [topology_file, topology_defn_file]
  generated_config_files = [release_yaml_file, cl_args['override_config_file']]

  utils.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)

  # pass the args to submitter main
  args = [
      "--cluster", cl_args['cluster'],
      "--role", cl_args['role'],
      "--environment", cl_args['environ'],
      "--heron_home", utils.get_heron_dir(),
      "--config_path", config_path,
      "--override_config_file", cl_args['override_config_file'],
      "--release_file", release_yaml_file,
      "--topology_package", topology_pkg_path,
      "--topology_defn", topology_defn_file,
      "--topology_jar", topology_file
  ]

  if opts.verbose():
    args.append("--verbose")

  lib_jars = utils.get_heron_libs(
      jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
  )

  # invoke the submitter to submit and launch the topology
  execute.heron_class(
      'com.twitter.heron.scheduler.SubmitterMain',
      lib_jars,
      extra_jars=[],
      args=args,
      java_defines=[]
  )
Пример #4
0
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file):
    '''
  Launch a topology given topology jar, its definition file and configurations
  :param cl_args:
  :param tmp_dir:
  :param topology_file:
  :param topology_defn_file:
  :return:
  '''
    # get the normalized path for topology.tar.gz
    topology_pkg_path = utils.normalized_class_path(
        os.path.join(tmp_dir, 'topology.tar.gz'))

    # get the release yaml file
    release_yaml_file = utils.get_heron_release_file()

    # create a tar package with the cluster configuration and generated config files
    config_path = cl_args['config_path']
    tar_pkg_files = [topology_file, topology_defn_file]
    generated_config_files = [
        release_yaml_file, cl_args['override_config_file']
    ]

    utils.create_tar(topology_pkg_path, tar_pkg_files, config_path,
                     generated_config_files)

    # pass the args to submitter main
    args = [
        "--cluster", cl_args['cluster'], "--role", cl_args['role'],
        "--environment", cl_args['environ'], "--heron_home",
        utils.get_heron_dir(), "--config_path", config_path,
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", release_yaml_file, "--topology_package",
        topology_pkg_path, "--topology_defn", topology_defn_file,
        "--topology_jar", topology_file
    ]

    if opts.verbose():
        args.append("--verbose")

    lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
                                    jars.uploader_jars() +
                                    jars.statemgr_jars() + jars.packing_jars())

    # invoke the submitter to submit and launch the topology
    execute.heron_class('com.twitter.heron.scheduler.SubmitterMain',
                        lib_jars,
                        extra_jars=[],
                        args=args,
                        java_defines=[])
Пример #5
0
def run(command, parser, cl_args, unknown_args):

    try:
        topology_name = cl_args['topology-name']

        new_args = [
            "--cluster",
            cl_args['cluster'],
            "--role",
            cl_args['role'],
            "--environment",
            cl_args['environ'],
            "--heron_home",
            utils.get_heron_dir(),
            "--config_path",
            cl_args['config_path'],
            "--override_config_file",
            cl_args['override_config_file'],
            "--release_file",
            utils.get_heron_release_file(),
            "--topology_name",
            topology_name,
            "--command",
            command,
        ]

        if opts.verbose():
            new_args.append("--verbose")

        lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
                                        jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
                            lib_jars,
                            extra_jars=[],
                            args=new_args)

    except Exception as ex:
        print 'Error: %s' % str(ex)
        Log.error('Failed to activate topology \'%s\'' % topology_name)
        return False

    Log.info('Successfully activated topology \'%s\'' % topology_name)
    return True
Пример #6
0
def run(command, parser, cl_args, unknown_args, action):
  '''
  helper function to take action on topologies
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :param action:        description of action taken
  :return:
  '''
  try:
    topology_name = cl_args['topology-name']

    new_args = [
        "--cluster", cl_args['cluster'],
        "--role", cl_args['role'],
        "--environment", cl_args['environ'],
        "--heron_home", config.get_heron_dir(),
        "--config_path", cl_args['config_path'],
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", config.get_heron_release_file(),
        "--topology_name", topology_name,
        "--command", command,
    ]

    if Log.getEffectiveLevel() == logging.DEBUG:
      new_args.append("--verbose")

    lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

    # invoke the runtime manager to kill the topology
    execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args=new_args
    )

  except Exception:
    Log.error('Failed to %s \'%s\'' % (action, topology_name))
    return False

  Log.info('Successfully %s \'%s\'' % (action, topology_name))
  return True
Пример #7
0
def run(command, parser, cl_args, unknown_args):
  '''
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :return:
  '''
  try:
    topology_name = cl_args['topology-name']
    container_id = cl_args['container-id']

    new_args = [
        "--cluster", cl_args['cluster'],
        "--role", cl_args['role'],
        "--environment", cl_args['environ'],
        "--heron_home", utils.get_heron_dir(),
        "--config_path", cl_args['config_path'],
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", utils.get_heron_release_file(),
        "--topology_name", topology_name,
        "--command", command,
        "--container_id", str(container_id)
    ]

    lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

    # invoke the runtime manager to kill the topology
    execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args=new_args
    )

  except Exception as ex:
    print 'Error: %s' % str(ex)
    Log.error('Failed to restart topology \'%s\'' % topology_name)
    return False

  Log.info('Successfully restarted topology \'%s\'' % topology_name)
  return True
Пример #8
0
def submit_fatjar(cl_args, unknown_args, tmp_dir):
  '''
   We use the packer to make a package for the jar and dump it
  to a well-known location. We then run the main method of class
  with the specified arguments. We pass arguments as heron.options.

  This will run the jar file with the topology_class_name. The submitter
  inside will write out the topology defn file to a location that
  we specify. Then we write the topology defn file to a well known
  location. We then write to appropriate places in zookeeper
  and launch the scheduler jobs
  :param cl_args:
  :param unknown_args:
  :param tmp_dir:
  :return:
  '''
  # execute main of the topology to create the topology definition
  topology_file = cl_args['topology-file-name']
  try:
    execute.heron_class(
        cl_args['topology-class-name'],
        utils.get_heron_libs(jars.topology_jars()),
        extra_jars=[topology_file],
        args=tuple(unknown_args),
        java_defines=cl_args['topology_main_jvm_property'])

  except Exception:
    Log.error("Unable to execute topology main class")
    return False

  try:
    launch_topologies(cl_args, topology_file, tmp_dir)

  except Exception:
    return False

  finally:
    shutil.rmtree(tmp_dir)

  return True
Пример #9
0
def run(command, parser, cl_args, unknown_args):
    '''
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :return:
  '''
    try:
        topology_name = cl_args['topology-name']
        container_id = cl_args['container-id']

        new_args = [
            "--cluster", cl_args['cluster'], "--role", cl_args['role'],
            "--environment", cl_args['environ'], "--heron_home",
            utils.get_heron_dir(), "--config_path", cl_args['config_path'],
            "--override_config_file", cl_args['override_config_file'],
            "--release_file",
            utils.get_heron_release_file(), "--topology_name", topology_name,
            "--command", command, "--container_id",
            str(container_id)
        ]

        lib_jars = utils.get_heron_libs(jars.scheduler_jars() +
                                        jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class('com.twitter.heron.scheduler.RuntimeManagerMain',
                            lib_jars,
                            extra_jars=[],
                            args=new_args)

    except Exception as ex:
        print 'Error: %s' % str(ex)
        Log.error('Failed to restart topology \'%s\'' % topology_name)
        return False

    Log.info('Successfully restarted topology \'%s\'' % topology_name)
    return True
Пример #10
0
def run(command, parser, cl_args, unknown_args):
    try:
        topology_name = cl_args["topology-name"]

        new_args = [
            "--cluster",
            cl_args["cluster"],
            "--role",
            cl_args["role"],
            "--environment",
            cl_args["environ"],
            "--heron_home",
            utils.get_heron_dir(),
            "--config_path",
            cl_args["config_path"],
            "--override_config_file",
            cl_args["override_config_file"],
            "--release_file",
            utils.get_heron_release_file(),
            "--topology_name",
            topology_name,
            "--command",
            command,
        ]

        if opts.verbose():
            new_args.append("--verbose")

        lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

        # invoke the runtime manager to kill the topology
        execute.heron_class("com.twitter.heron.scheduler.RuntimeManagerMain", lib_jars, extra_jars=[], args=new_args)

    except Exception as ex:
        Log.error("Failed to kill topology '%s'" % topology_name)
        return False

    Log.info("Successfully killed topology '%s'" % topology_name)
    return True
Пример #11
0
def submit_fatjar(cl_args, unknown_args, tmp_dir):
    '''
   We use the packer to make a package for the jar and dump it
  to a well-known location. We then run the main method of class
  with the specified arguments. We pass arguments as heron.options.

  This will run the jar file with the topology_class_name. The submitter
  inside will write out the topology defn file to a location that
  we specify. Then we write the topology defn file to a well known
  location. We then write to appropriate places in zookeeper
  and launch the scheduler jobs
  :param cl_args:
  :param unknown_args:
  :param tmp_dir:
  :return:
  '''
    # execute main of the topology to create the topology definition
    topology_file = cl_args['topology-file-name']
    try:
        execute.heron_class(cl_args['topology-class-name'],
                            utils.get_heron_libs(jars.topology_jars()),
                            extra_jars=[topology_file],
                            args=tuple(unknown_args),
                            java_defines=cl_args['topology_main_jvm_property'])

    except Exception:
        Log.error("Unable to execute topology main class")
        return False

    try:
        launch_topologies(cl_args, topology_file, tmp_dir)

    except Exception:
        return False

    finally:
        shutil.rmtree(tmp_dir)

    return True
Пример #12
0
def run(command, parser, cl_args, unknown_args):

  try:
    topology_name = cl_args['topology-name']

    new_args = [
        "--cluster", cl_args['cluster'],
        "--role", cl_args['role'],
        "--environment", cl_args['environ'],
        "--heron_home", utils.get_heron_dir(),
        "--config_path", cl_args['config_path'],
        "--override_config_file", cl_args['override_config_file'],
        "--release_file", utils.get_heron_release_file(),
        "--topology_name", topology_name,
        "--command", command,
    ]

    if opts.verbose():
      new_args.append("--verbose")

    lib_jars = utils.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())

    # invoke the runtime manager to kill the topology
    execute.heron_class(
        'com.twitter.heron.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args= new_args
    )

  except Exception as ex:
    print 'Error: %s' % str(ex)
    Log.error('Failed to deactivate topology \'%s\'' % topology_name)
    return False

  Log.info('Successfully deactivated topology \'%s\'' % topology_name)
  return True