Beispiel #1
0
def twister2_tar(class_name, topology_tar, arguments, tmpdir_root, java_defines):
    '''
    :param class_name:
    :param topology_tar:
    :param arguments:
    :param tmpdir_root:
    :param java_defines:
    :return:
    '''
    # Extract tar to a tmp folder.
    tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')

    with contextlib.closing(tarfile.open(topology_tar)) as tar:
        tar.extractall(path=tmpdir)

    # A tar generated by pants has all dependency jars under libs/
    # in addition to the topology jar at top level. Pants keeps
    # filename for jar and tar the same except for extension.

    topology_jar = os.path.basename(topology_tar).replace(".tar.gz", "").replace(".tar", "") + ".jar"

    extra_jars = [
        os.path.join(tmpdir, "twister2-instance.jar"),
        os.path.join(tmpdir, topology_jar),
        os.path.join(tmpdir, "*"),
        os.path.join(tmpdir, "libs/*")
    ]

    lib_jars = config.get_twister2_libs(jars.job_jars())

    # Now execute the class
    return twister2_class(class_name, lib_jars, extra_jars, arguments, java_defines)
Beispiel #2
0
def submit_python_zip(cl_args, unknown_args):
    # set up the system properties
    java_system_props = setup_java_system_properties(cl_args)

    props = read_client_properties(cl_args)

    # execute main of the job to create the job definition
    job_file = cl_args['job-file-name']
    main_class = cl_args['job-class-name']

    java_system_props += ["python_file=" + job_file, "main_file=" + main_class]

    res = execute.twister2_class(
        class_name="edu.iu.dsc.tws.python.PythonWorker",
        lib_jars=config.get_twister2_libs(jars.job_jars()),
        extra_jars=[],
        args=tuple(unknown_args),
        java_defines=java_system_props,
        client_props=props)

    result.render(res)

    if not res.is_successful():
        err_context = ("Failed to create job definition " \
                       "Please check logs for more information")
        res.add_context(err_context)
        return res

    return res
Beispiel #3
0
def twister2_tar(class_name,
                 topology_tar,
                 arguments,
                 tmpdir_root,
                 java_defines,
                 client_props=None):
    '''
    :param class_name:
    :param topology_tar:
    :param arguments:
    :param tmpdir_root:
    :param java_defines:
    :return:
    '''
    # Extract tar to a tmp folder.
    tmpdir = tempfile.mkdtemp(dir=tmpdir_root, prefix='tmp')
    topology_jar = os.path.basename(topology_tar).replace(".zip", "")

    with ZipFile(topology_tar, 'r') as zipObj:
        print(topology_tar, tmpdir, topology_jar)
        # Extract all the contents of zip file in different directory
        zipObj.extractall(tmpdir)

    if java_defines is None:
        java_defines = []

    # Format all java -D options that need to be passed while running
    # the class locally.
    java_opts = ['-D' + opt for opt in java_defines]

    extra_jars = [
        os.path.join(tmpdir, topology_jar, "*.jar"),
        os.path.join(tmpdir, topology_jar, "libs/*.jar")
    ]

    for root, dirs, files in os.walk(os.path.join(tmpdir, topology_jar)):
        for file in files:
            if file.endswith('.jar'):
                extra_jars.append(os.path.abspath(os.path.join(root, file)))

    print(extra_jars)

    lib_jars = config.get_twister2_libs(jars.job_jars())

    print(lib_jars)

    if client_props:
        if 'twister2.client.debug' in client_props:
            java_opts.append(client_props['twister2.client.debug'])

    # Now execute the class
    return twister2_class(class_name, lib_jars, extra_jars, arguments,
                          java_defines, client_props)
Beispiel #4
0
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
    '''
    helper function to take action on topologies
    :param command:
    :param cl_args:
    :param action:        description of action taken
    :return:
    '''
    job_id = cl_args['job-id']

    new_args = [
        "--cluster",
        cl_args['cluster'],
        "--twister2_home",
        config.get_twister2_dir(),
        "--config_path",
        config.get_twister2_conf_dir(),
        "--job_id",
        job_id,
        "--command",
        command,
    ]
    new_args += extra_args

    lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars() +
                                        jars.statemgr_jars())
    lib_jars += extra_lib_jars

    if Log.getEffectiveLevel() == logging.DEBUG:
        new_args.append("--verbose")

    java_defines = []
    conf_dir_common = config.get_twister2_cluster_conf_dir(
        "common", config.get_twister2_conf_dir())
    java_defines.append("java.util.logging.config.file=" + conf_dir_common +
                        "/logger.properties")

    # invoke the runtime manager to kill the job
    result = execute.twister2_class(
        'edu.iu.dsc.tws.rsched.core.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args=new_args,
        java_defines=java_defines)

    err_msg = "Failed to %s %s" % (action, job_id)
    succ_msg = "Successfully %s %s" % (action, job_id)
    result.add_context(err_msg, succ_msg)
    return result
Beispiel #5
0
def run(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
    '''
    helper function to take action on topologies
    :param command:
    :param cl_args:
    :param action:        description of action taken
    :return:
    '''
    topology_name = cl_args['job-name']

    new_args = [
        "--cluster",
        cl_args['cluster'],
        "--twister2_home",
        config.get_twister2_dir(),
        "--config_path",
        cl_args['config_path'],
        "--override_config_file",
        cl_args['override_config_file'],
        "--job_name",
        job_name,
        "--command",
        command,
    ]
    new_args += extra_args

    lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars() +
                                        jars.statemgr_jars())
    lib_jars += extra_lib_jars

    if Log.getEffectiveLevel() == logging.DEBUG:
        new_args.append("--verbose")

    # invoke the runtime manager to kill the job
    result = execute.twister2_class(
        'com.twitter.twister2.scheduler.RuntimeManagerMain',
        lib_jars,
        extra_jars=[],
        args=new_args)

    err_msg = "Failed to %s %s" % (action, topology_name)
    succ_msg = "Successfully %s %s" % (action, topology_name)
    result.add_context(err_msg, succ_msg)
    return result
Beispiel #6
0
def submit_fatjar(cl_args, unknown_args):
    '''
    We use the packer to make a package for the jar and dump it
    to a well-known location. We then run the main method of class
    with the specified arguments. We pass arguments as an environment variable TWISTER2_OPTIONS.

    This will run the jar file with the job_class_name. The submitter
    inside will write out the job defn file to a location that
    we specify. Then we write the job defn file to a well known
    location. We then write to appropriate places in zookeeper
    and launch the scheduler jobs
    :param cl_args:
    :param unknown_args:
    :param tmp_dir:
    :return:
    '''
    # set up the system properties
    java_system_props = setup_java_system_properties(cl_args)

    props = read_client_properties(cl_args)

    # execute main of the job to create the job definition
    job_file = cl_args['job-file-name']

    main_class = cl_args['job-class-name']

    res = execute.twister2_class(class_name=main_class,
                                 lib_jars=config.get_twister2_libs(
                                     jars.job_jars()),
                                 extra_jars=[job_file],
                                 args=tuple(unknown_args),
                                 java_defines=java_system_props,
                                 client_props=props)

    result.render(res)

    if not res.is_successful():
        err_context = ("Failed to create job definition " \
                       "Please check logs for more information")
        res.add_context(err_context)
        return res

    return res
Beispiel #7
0
def run(command, parser, args, unknown_args):
    '''
    :param command:
    :param parser:
    :param args:
    :param unknown_args:
    :return:
    '''

    job_id = args['job-id']
    conf_path = os.path.join(config.get_twister2_dir(), "conf/common")

    new_args = [
        "--twister2_home", config.get_twister2_dir(),
        "--config_path",conf_path,
        "--job_id", job_id,
        "--command", command,
    ]

    new_args += unknown_args
    lib_jars = config.get_twister2_libs(jars.resource_scheduler_jars())

    if Log.getEffectiveLevel() == logging.DEBUG:
        new_args.append("--verbose")

    java_defines = []
    conf_dir_common = config.get_twister2_cluster_conf_dir("common", config.get_twister2_conf_dir())
    java_defines.append("java.util.logging.config.file=" + conf_dir_common + "/logger.properties")

    # invoke the runtime manager to kill the job
    result = execute.twister2_class(
        'edu.iu.dsc.tws.rsched.job.ZKJobLister',
        lib_jars,
        extra_jars=[],
        args=new_args,
        java_defines=java_defines
    )

    err_msg = "Failed to %s %s" % (command, job_id)
    succ_msg = "Successfully %s %s" % (command, job_id)
    result.add_context(err_msg, succ_msg)

    return result