Пример #1
0
def on_add_classpath_clash_check(unit, *args):
    jdeps_val = (unit.get('CHECK_JAVA_DEPS_VALUE') or '').lower()
    if jdeps_val and jdeps_val not in ('yes', 'no', 'strict'):
        ymake.report_configure_error(
            'CHECK_JAVA_DEPS: "yes", "no" or "strict" required')
    if jdeps_val and jdeps_val != 'no':
        unit.onjava_test_deps(jdeps_val)
Пример #2
0
def ontest_srcs(unit, *args):
    used = set(args) & {"NAMESPACE", "TOP_LEVEL", "__main__.py"}
    if used:
        param = list(used)[0]
        ymake.report_configure_error('in TEST_SRCS: you cannot use {} here - it would broke testing machinery'.format(param))
    if unit.get('PYTEST_BIN') != 'no':
        unit.onpy_srcs(["NAMESPACE", "__tests__"] + list(args))
Пример #3
0
def ontest_srcs(unit, *args):
    used = set(args) & {"NAMESPACE", "TOP_LEVEL", "__main__.py"}
    if used:
        param = list(used)[0]
        ymake.report_configure_error('in TEST_SRCS: you cannot use {} here - it would broke testing machinery'.format(param))
    if unit.get('PYTEST_BIN') != 'no':
        unit.onpy_srcs(["NAMESPACE", "__tests__"] + list(args))
Пример #4
0
def add_python_lint_checks(unit, py_ver, files):
    def get_resolved_files():
        resolved_files = []
        for path in files:
            resolved = unit.resolve_arc_path([path])
            if resolved.startswith('$S'):  # path was resolved as source file.
                resolved_files.append(resolved)
        return resolved_files

    if unit.get('LINT_LEVEL_VALUE') == "none":

        no_lint_allowed_paths = (
            "contrib/",
            "devtools/",
            "junk/",
            # temporary allowed, TODO: remove
            "taxi/uservices/",
            "travel/",
            "market/report/lite/",  # MARKETOUT-38662, deadline: 2021-08-12
        )

        upath = unit.path()[3:]

        if not upath.startswith(no_lint_allowed_paths):
            ymake.report_configure_error("NO_LINT() is allowed only in " + ", ".join(no_lint_allowed_paths))

    if files and unit.get('LINT_LEVEL_VALUE') not in ("none", "none_internal"):
        resolved_files = get_resolved_files()
        flake8_cfg = 'build/config/tests/flake8.conf'
        unit.onadd_check(["flake8.py{}".format(py_ver), flake8_cfg] + resolved_files)
Пример #5
0
def _check_test_srcs(*args):
    used = set(args) & {"NAMESPACE", "TOP_LEVEL", "__main__.py"}
    if used:
        param = list(used)[0]
        ymake.report_configure_error(
            'in TEST_SRCS: you cannot use {} here - it would broke testing machinery'
            .format(param))
Пример #6
0
def on_go_process_srcs(unit):
    """
        _GO_PROCESS_SRCS() macro processes only 'CGO' files. All remaining *.go files
        and other input files are currently processed by a link command of the
        GO module (GO_LIBRARY, GO_PROGRAM)
    """

    go_files = get_appended_values(unit, 'GO_SRCS_VALUE')
    for f in go_files:
        if f.endswith('_test.go'):
            ymake.report_configure_error('file {} must be listed in GO_TEST_SRCS() or GO_XTEST_SRCS() macros'.format(f))
    go_test_files = get_appended_values(unit, 'GO_TEST_SRCS_VALUE')
    go_xtest_files = get_appended_values(unit, 'GO_XTEST_SRCS_VALUE')
    for f in go_test_files + go_xtest_files:
        if not f.endswith('_test.go'):
            ymake.report_configure_error('file {} should not be listed in GO_TEST_SRCS() or GO_XTEST_SRCS() macros'.format(f))

    go_std_root = unit.get('GOSTD') + os.path.sep

    proto_files = filter(lambda x: x.endswith('.proto'), go_files)
    if len(proto_files) > 0:
        for f in proto_files:
            unit.ongo_proto_cmd(f)

    in_files = filter(lambda x: x.endswith('.in'), go_files)
    if len(in_files) > 0:
        for f in in_files:
            unit.onsrc(f)

    if compare_versions('1.12', unit.get('GOSTD_VERSION')) >= 0:
        asm_files = filter(lambda x: x.endswith('.s'), go_files)
        if len(asm_files) > 0:
            unit.ongo_compile_symabis(asm_files)

    s_files = filter(lambda x: x.endswith('.S'), go_files)
    c_files = filter(lambda x: x.endswith('.c'), go_files)
    if len(c_files) + len(s_files) > 0:
        cgo_flags = get_appended_values(unit, 'CGO_CFLAGS_VALUE')
        for f in c_files + s_files:
            unit.onsrc([f] + cgo_flags)

    cgo_files = get_appended_values(unit, 'CGO_SRCS_VALUE')
    if len(cgo_files) > 0:
        import_path = rootrel_arc_src(unit.path(), unit)
        if import_path.startswith(go_std_root):
            import_path = import_path[len(go_std_root):]
        if import_path != runtime_cgo_path:
            unit.onpeerdir(os.path.join(go_std_root, runtime_cgo_path))
        import_runtime_cgo = 'false' if import_path in [runtime_cgo_path, runtime_msan_path, runtime_race_path] else 'true'
        import_syscall = 'false' if import_path == runtime_cgo_path else 'true'
        args = [import_path] + cgo_files + ['FLAGS', '-import_runtime_cgo=' + import_runtime_cgo, '-import_syscall=' + import_syscall]
        unit.ongo_compile_cgo1(args)
        args = [unit.get('GO_PACKAGE_VALUE') or unit.get('REALPRJNAME')] + cgo_files
        if len(c_files) > 0:
            args += ['C_FILES'] + c_files
        if len(s_files) > 0:
            args += ['S_FILES'] + s_files
        unit.ongo_compile_cgo2(args)
Пример #7
0
def onadd_pytest_bin(unit, *args):
    flat, kws = _common.sort_by_keywords({'RUNNER_BIN': 1}, args)
    if flat:
        ymake.report_configure_error(
            'Unknown arguments found while processing add_pytest_bin macro: {!r}'
            .format(flat))

    runner_bin = kws.get('RUNNER_BIN', [None])[0]
    add_test_to_dart(unit, "pytest.bin", runner_bin=runner_bin)
Пример #8
0
def generate_dart(unit, as_lib=False):
    module_dir = os.path.normpath(unit.path()[3:])
    docs_dir = (unit.get('DOCSDIR') or '').rstrip('/')
    if docs_dir:
        docs_dir = os.path.normpath(docs_dir)
        unit.set(['SRCDIR', docs_dir])
    else:
        docs_dir = module_dir

    build_tool = unit.get('DOCSBUILDER') or 'mkdocs'

    if build_tool not in ['mkdocs', 'yfm']:
        unit.message(['error', 'Unsupported build tool {}'.format(build_tool)])

    docs_config = unit.get('DOCSCONFIG')
    if not docs_config:
        docs_config = 'mkdocs.yml' if build_tool == 'mkdocs' else '.yfm'

    docs_config = os.path.normpath(docs_config)
    if os.path.sep not in docs_config:
        docs_config = os.path.join(module_dir if build_tool == 'mkdocs' else docs_dir, docs_config)

    if not docs_config.startswith(docs_dir + os.path.sep) and not docs_config.startswith(module_dir + os.path.sep) :
        unit.message(['error', 'DOCS_CONFIG value "{}" is outside the project directory and DOCS_DIR'.format(docs_config)])
        return

    if not os.path.exists(unit.resolve('$S/' + docs_config)):
        unit.message(['error', 'DOCS_CONFIG value "{}" does not exist'.format(docs_config)])
        return

    includes = extract_macro_calls(unit, 'DOCSINCLUDESOURCES')

    data = {
        'DOCS_NAME': unit.name(),
        'PATH': module_dir,
        'MODULE_TAG': unit.get('MODULE_TAG'),
        'DOCSDIR': docs_dir,
        'DOCSCONFIG': docs_config,
        'DOCSVARS': macro_calls_to_dict(unit, extract_macro_calls(unit, 'DOCSVARS')),
        'DOCSINCLUDESOURCES': includes,
        'DOCSLIB': as_lib,
        'PEERDIRS': '${PEERDIR}',
        'DOCSBUILDER': build_tool,
    }

    dart = 'DOCS_DART: ' + base64.b64encode(json.dumps(data)) + '\n' + DELIM + '\n'

    unit.set_property(['DOCS_DART_DATA', dart])

    for i in includes:
        include_path = unit.resolve('$S/' + i)

        if not os.path.exists(include_path):
            ymake.report_configure_error('DOCS_INCLUDE_SOURCES value "{}" does not exist'.format(i))

        elif not os.path.isfile(include_path):
            ymake.report_configure_error('DOCS_INCLUDE_SOURCES value "{}" must be a file'.format(i))
Пример #9
0
def on_check_java_srcdir(unit, *args):
    args = list(args)
    for arg in args:
        srcdir = unit.resolve_arc_path(arg)
        if not srcdir.startswith('$S'):
            continue
        abs_srcdir = unit.resolve(srcdir)
        if not os.path.exists(abs_srcdir) or not os.path.isdir(abs_srcdir):
            ymake.report_configure_error('SRCDIR {} does not exists or not a directory'.format(srcdir[3:]))
Пример #10
0
def onexternal_jar(unit, *args):
    args = list(args)
    flat, kv = common.sort_by_keywords({'SOURCES': 1}, args)
    if not flat:
        ymake.report_configure_error('EXTERNAL_JAR requires exactly one resource URL of compiled jar library')
    res = flat[0]
    resid = res[4:] if res.startswith('sbr:') else res
    unit.set(['JAR_LIB_RESOURCE', resid])
    unit.set(['JAR_LIB_RESOURCE_URL', res])
Пример #11
0
def onsetup_exectest(unit, *args):
    command = unit.get(["EXECTEST_COMMAND_VALUE"])
    if command is None:
        ymake.report_configure_error("EXECTEST must have at least one RUN macro")
        return
    command = command.replace("$EXECTEST_COMMAND_VALUE", "")
    if "PYTHON_BIN" in command:
        unit.ondepends('contrib/tools/python')
    unit.set(["TEST_BLOB_DATA", base64.b64encode(command)])
    add_test_to_dart(unit, "exectest", binary_path=os.path.join(unit.path(), unit.filename()).replace(".pkg", ""))
Пример #12
0
def get_tidy_config_map(unit, map_path):
    config_map_path = unit.resolve(os.path.join("$S", map_path))
    config_map = {}
    try:
        with open(config_map_path, 'r') as afile:
            config_map = json.load(afile)
    except ValueError:
        ymake.report_configure_error("{} is invalid json".format(map_path))
    except Exception as e:
        ymake.report_configure_error(str(e))
    return config_map
Пример #13
0
def onrestrict_path(unit, *args):
    if args:
        if 'MSG' in args:
            pos = args.index('MSG')
            paths, msg = args[:pos], args[pos + 1:]
            msg = ' '.join(msg)
        else:
            paths, msg = args, 'forbidden'
        if not _common.strip_roots(unit.path()).startswith(paths):
            error_msg = "Path '[[imp]]{}[[rst]]' is restricted - [[bad]]{}[[rst]]. Valid path prefixes are: [[unimp]]{}[[rst]]".format(unit.path(), msg, ', '.join(paths))
            ymake.report_configure_error(error_msg)
Пример #14
0
def dump_test(kw):
    errors = validate_test(kw)
    if errors:
        for e in errors:
            ymake.report_configure_error(e)
        return None
    string_handler = StringIO.StringIO()
    for k, v in kw.iteritems():
        print >>string_handler, k + ': ' + v
    print >>string_handler, BLOCK_SEPARATOR
    data = string_handler.getvalue()
    string_handler.close()
    return data
Пример #15
0
def on_check_java_srcdir(unit, *args):
    args = list(args)
    for arg in args:
        if not '$' in arg:
            abs_srcdir = unit.resolve(os.path.join("$S/", unit.get('MODDIR'), arg))
            if not os.path.exists(abs_srcdir) or not os.path.isdir(abs_srcdir):
                ymake.report_configure_error('SRCDIR {} does not exists or not a directory'.format(abs_srcdir))
        srcdir = unit.resolve_arc_path(arg)
        if srcdir and not srcdir.startswith('$S'):
            continue
        abs_srcdir = unit.resolve(srcdir) if srcdir else unit.resolve(arg)
        if not os.path.exists(abs_srcdir) or not os.path.isdir(abs_srcdir):
            ymake.report_configure_error('SRCDIR {} does not exists or not a directory'.format(abs_srcdir))
Пример #16
0
def dump_test(kw, is_fuzz_test=False):
    valid_kw, errors = validate_test(kw, is_fuzz_test)
    if errors:
        for e in errors:
            ymake.report_configure_error(e)
    if valid_kw is None:
        return None
    string_handler = StringIO.StringIO()
    for k, v in valid_kw.iteritems():
        print >>string_handler, k + ': ' + v
    print >>string_handler, BLOCK_SEPARATOR
    data = string_handler.getvalue()
    string_handler.close()
    return data
Пример #17
0
def dump_test(kw, is_fuzz_test=False):
    valid_kw, errors = validate_test(kw, is_fuzz_test)
    if errors:
        for e in errors:
            ymake.report_configure_error(e)
    if valid_kw is None:
        return None
    string_handler = StringIO.StringIO()
    for k, v in valid_kw.iteritems():
        print >> string_handler, k + ': ' + v
    print >> string_handler, BLOCK_SEPARATOR
    data = string_handler.getvalue()
    string_handler.close()
    return data
Пример #18
0
def on_check_java_srcdir(unit, *args):
    args = list(args)
    source_root = unit.resolve('$S')
    mod_root = os.path.join(source_root, unit.get('MODDIR'))
    for arg in args:
        srcdir = unit.resolve(arg)
        if not os.path.isabs(srcdir):
            srcdir = os.path.join(mod_root, srcdir)
        if not srcdir.startswith(source_root):
            continue
        if not os.path.exists(srcdir) or not os.path.isdir(srcdir):
            ymake.report_configure_error(
                'SRCDIR {} does not exists or not a directory'.format(
                    os.path.relpath(srcdir, source_root)))
Пример #19
0
def ongenerate_script(unit, *args):
    """
    heretic@ promised to make tutorial here
    Don't forget
    Feel free to remind
    """
    flat, kv = common.sort_by_keywords(
        {'OUT': 1, 'TEMPLATE': -1, 'CUSTOM_PROPERTY': -1},
        args
    )
    if len(kv.get('TEMPLATE', [])) > 1:
        ymake.report_configure_error('To mane arguments for TEMPLATE parameter')
    prev = unit.get(['GENERATE_SCRIPT_VALUE']) or ''
    new_val = (prev + ' ' + base64.b64encode(json.dumps(list(args), encoding='utf-8'))).strip()
    unit.set(['GENERATE_SCRIPT_VALUE', new_val])
Пример #20
0
def ongenerate_script(unit, *args):
    """
    heretic@ promised to make tutorial here
    Don't forget
    Feel free to remind
    """
    flat, kv = common.sort_by_keywords(
        {'OUT': 1, 'TEMPLATE': -1, 'CUSTOM_PROPERTY': -1},
        args
    )
    if len(kv.get('TEMPLATE', [])) > 1:
        ymake.report_configure_error('To mane arguments for TEMPLATE parameter')
    prev = unit.get(['GENERATE_SCRIPT_VALUE']) or ''
    new_val = (prev + ' ' + base64.b64encode(json.dumps(list(args), encoding='utf-8'))).strip()
    unit.set(['GENERATE_SCRIPT_VALUE', new_val])
Пример #21
0
def dump_test(unit, kw):
    valid_kw, warnings, errors = validate_test(unit, kw)
    for w in warnings:
        unit.message(['warn', w])
    for e in errors:
        ymake.report_configure_error(e)
    if valid_kw is None:
        return None
    string_handler = StringIO.StringIO()
    for k, v in valid_kw.iteritems():
        print >>string_handler, k + ': ' + v
    print >>string_handler, BLOCK_SEPARATOR
    data = string_handler.getvalue()
    string_handler.close()
    return data
Пример #22
0
def onadd_pytest_bin(unit, *args):
    if unit.get("TIDY") == "yes":
        # graph changed for clang_tidy tests
        return
    flat, kws = _common.sort_by_keywords({'RUNNER_BIN': 1}, args)
    if flat:
        ymake.report_configure_error(
            'Unknown arguments found while processing add_pytest_bin macro: {!r}'
            .format(flat)
        )

    runner_bin = kws.get('RUNNER_BIN', [None])[0]
    test_type = 'py3test.bin' if (unit.get("PYTHON3") == 'yes') else "pytest.bin"

    add_test_to_dart(unit, test_type, runner_bin=runner_bin)
Пример #23
0
def onrun_java(unit, *args):
    flat, kv = common.sort_by_keywords(
        {
            'CLASSPATH': -1,
            'IN': -1,
            'OUT': -1,
            'OUT_NOAUTO': -1,
            'OUTPUT_INCLUDES': -1,
            'DEBUG': 0,
            'JAR': 1
        }, args)
    if not (kv.get('CLASSPATH', []) + kv.get('JAR', [])):
        ymake.report_configure_error(
            'Java program for RUN_JAVA is not specified')

    depends = []
    if not unit.get('IDE_MSVS_CALL'):
        for jar in (kv.get('CLASSPATH', []) + kv.get('JAR', [])):
            depends.append(jar)

    classpath = ':'.join(classpath)

    # Generate java cmd
    cmd = []
    if kv.get('JAR'):
        cmd += [
            '-jar',
            ':'.join(['$SCARAB_SLIM'] + kv.get('JAR')),
        ]
    cmd += [
        '-classpath',
        ':'.join(['$SCARAB'] + kv.get('JAR', []) + kv.get('CLASSPATH', [])),
        '-Dfile.encoding=UTF-8',
    ]

    cmd += flat

    if 'DEBUG' not in kv:
        cmd += ['HIDE_OUTPUT']

    for k in 'IN', 'OUT', 'OUT_NOAUTO', 'OUTPUT_INCLUDES':
        if kv.get(k):
            cmd += [k] + kv[k]

    if depends:
        cmd += ['TOOL'] + depends

    unit.on_run_java(cmd)
Пример #24
0
def onios_app_settings(unit, *args):
    tail, kv = common.sort_by_keywords({'OS_VERSION': 1, 'DEVICES': -1}, args)
    if tail:
        ymake.report_configure_error(
            'Bad IOS_COMMON_SETTINGS usage - unknown data: ' + str(tail))
    if kv.get('OS_VERSION', []):
        unit.onios_app_common_flags(
            ['--minimum-deployment-target',
             kv.get('OS_VERSION', [])[0]])
        unit.onios_app_assets_flags(
            ['--filter-for-device-os-version',
             kv.get('OS_VERSION', [])[0]])
    devices_flags = []
    for device in kv.get('DEVICES', []):
        devices_flags += ['--target-device', device]
    if devices_flags:
        unit.onios_app_common_flags(devices_flags)
Пример #25
0
def parse_pyx_includes(filename, path, source_root, seen=None):
    normpath = lambda *x: os.path.normpath(os.path.join(*x))

    abs_path = normpath(source_root, filename)
    seen = seen or set()
    if abs_path in seen:
        return
    seen.add(abs_path)

    if not os.path.exists(abs_path):
        # File might be missing, because it might be generated
        return

    with open(abs_path, 'rb') as f:
        # Don't parse cimports and etc - irrelevant for cython, it's linker work
        includes = ymake.parse_cython_includes(f.read())

    abs_dirname = os.path.dirname(abs_path)
    # All includes are relative to the file which include
    path_dirname = os.path.dirname(path)
    file_dirname = os.path.dirname(filename)

    for incfile in includes:
        abs_path = normpath(abs_dirname, incfile)
        if os.path.exists(abs_path):
            incname, incpath = normpath(file_dirname, incfile), normpath(
                path_dirname, incfile)
            yield (incname, incpath)
            # search for includes in the included files
            for e in parse_pyx_includes(incname, incpath, source_root, seen):
                yield e
        else:
            # There might be arcadia root or cython relative include.
            # Don't treat such file as missing, because there must be PEERDIR on py_library
            # which contains it.
            for path in [
                    source_root,
                    source_root + "/contrib/tools/cython/Cython/Includes",
            ]:
                if os.path.exists(normpath(path, incfile)):
                    break
            else:
                ymake.report_configure_error(
                    "'{}' includes missing file: {} ({})".format(
                        path, incfile, abs_path))
Пример #26
0
def onregister_sandbox_import(unit, *args):
    args = iter(args)
    for path in args:
        path = os.path.normpath(path)
        source = unit.resolve_arc_path(path)
        abs_source = unit.resolve(source)
        if not os.path.exists(abs_source):
            ymake.report_configure_error(
                'REGISTER_SANDBOX_IMPORT: File or directory {} does not exists'
                .format(path))
        splited_path = path.split(os.sep)
        l, r = 0, len(splited_path)
        if splited_path[-1] == "__init__.py":
            r -= 1
        if not splited_path[0]:
            l += 1
        path = ".".join(splited_path[l:r])
        unit.onresource(
            ["-", "{}.{}={}".format("SANDBOX_TASK_REGISTRY", path, path)])
Пример #27
0
def on_check_java_srcdir(unit, *args):
    args = list(args)
    for arg in args:
        if not '$' in arg:
            arc_srcdir = os.path.join(unit.get('MODDIR'), arg)
            abs_srcdir = unit.resolve(os.path.join("$S/", arc_srcdir))
            if not os.path.exists(abs_srcdir) or not os.path.isdir(abs_srcdir):
                ymake.report_configure_error(
                    'Trying to set a [[alt1]]JAVA_SRCS[[rst]] for a missing directory: [[imp]]$S/{}[[rst]]',
                    missing_dir=arc_srcdir)
            return
        srcdir = unit.resolve_arc_path(arg)
        if srcdir and not srcdir.startswith('$S'):
            continue
        abs_srcdir = unit.resolve(srcdir) if srcdir else unit.resolve(arg)
        if not os.path.exists(abs_srcdir) or not os.path.isdir(abs_srcdir):
            ymake.report_configure_error(
                'Trying to set a [[alt1]]JAVA_SRCS[[rst]] for a missing directory: [[imp]]{}[[rst]]',
                missing_dir=srcdir)
Пример #28
0
def on_go_resource(unit, *args):
    args = list(args)
    files = args[::2]
    keys = args[1::2]
    resource_go = os.path.join("resource.res.go")

    unit.onpeerdir(["library/go/core/resource"])

    if len(files) != len(keys):
        ymake.report_configure_error("last file {} is missing resource key".format(files[-1]))

    for i, (key, filename) in enumerate(zip(keys, files)):
        if not key:
            ymake.report_configure_error("file key must be non empty")
            return

        if filename == "-" and "=" not in key:
            ymake.report_configure_error("key \"{}\" must contain = sign".format(key))
            return

        # quote key, to avoid automatic substitution of filename by absolute
        # path in RUN_PROGRAM
        args[2*i+1] = "notafile" + args[2*i+1]

    files = [file for file in files if file != "-"]
    unit.onrun_program([
        "library/go/core/resource/cc",
        "-package", go_package_name(unit),
        "-o", resource_go] + list(args) + [
        "IN"] + files + [
        "OUT", resource_go])
Пример #29
0
def onios_assets(unit, *args):
    _, kv = common.sort_by_keywords({
        'ROOT': 1,
        'CONTENTS': -1,
        'FLAGS': -1
    }, args)
    if not kv.get('ROOT', []) and kv.get('CONTENTS', []):
        ymake.report_configure_error(
            'Please specify ROOT directory for assets')
    origin_root = kv.get('ROOT')[0]
    destination_root = os.path.normpath(
        os.path.join('$BINDIR', os.path.basename(origin_root)))
    rel_list = []
    for cont in kv.get('CONTENTS', []):
        rel = os.path.relpath(cont, origin_root)
        if rel.startswith('..'):
            ymake.report_configure_error('{} is not subpath of {}'.format(
                cont, origin_root))
        rel_list.append(rel)
    if not rel_list:
        return
    results_list = [
        os.path.join('$B',
                     unit.path()[3:], os.path.basename(origin_root), i)
        for i in rel_list
    ]
    if len(kv.get('CONTENTS', [])) != len(results_list):
        ymake.report_configure_error(
            'IOS_ASSETTS content length is not equals results')
    for s, d in zip(kv.get('CONTENTS', []), results_list):
        unit.oncopy_file([s, d])
    if kv.get('FLAGS', []):
        unit.onios_app_assets_flags(kv.get('FLAGS', []))
    unit.on_ios_assets([destination_root] + results_list)
Пример #30
0
def parse_pyx_includes(path, srcdir, unit_path, source_root, seen=None):
    normpath = lambda *x: os.path.normpath(os.path.join(*x))

    abs_path = normpath(source_root, srcdir, path)

    seen = seen or set()
    if abs_path in seen:
        return
    seen.add(abs_path)

    if not os.path.exists(abs_path):
        # File might be missing, because it might be generated
        return

    with open(abs_path, 'rb') as f:
        # Don't parse cimports and etc - irrelevant for cython, it's linker work
        includes, _, _ = PyxParser.parse_includes(f.readlines(),
                                                  perm_includes=False,
                                                  direct_includes_only=True)

    abs_dirname = os.path.dirname(abs_path)
    # All includes are relative to the file which include
    path_shift = os.path.dirname(path)

    for incfile in includes:
        abs_path = normpath(abs_dirname, incfile)
        if os.path.exists(abs_path):
            yield (normpath(unit_path, incfile), normpath(path_shift, incfile))
            # search for includes in the included files
            for e in parse_pyx_includes(normpath(path_shift, incfile), srcdir,
                                        unit_path, source_root, seen):
                yield e
        else:
            # There might be arcadia root relative include.
            # Don't treat such file as missing, because there must be PEERDIR on py_library
            # which contains it.
            if not os.path.exists(normpath(source_root, incfile)):
                ymake.report_configure_error(
                    "'{}' includes missing file: {} ({})".format(
                        path, incfile, abs_path))
Пример #31
0
def onrun_java(unit, *args):
    flat, kv = common.sort_by_keywords(
        {'CLASSPATH': -1, 'IN': -1, 'OUT': -1, 'OUT_NOAUTO': -1, 'OUTPUT_INCLUDES': -1, 'DEBUG': 0, 'JAR': 1},
        args
    )
    if not (kv.get('CLASSPATH', []) + kv.get('JAR', [])):
        ymake.report_configure_error('Java program for RUN_JAVA is not specified')

    if not unit.get('IDE_MSVS_CALL'):
        for jar in (kv.get('CLASSPATH', []) + kv.get('JAR', [])):
            unit.oninternal_recurse(jar)

    classpath = ':'.join(classpath)

    # Generate java cmd
    cmd = []
    if kv.get('JAR'):
        cmd += [
            '-jar',
            ':'.join(['$SCARAB_SLIM'] + kv.get('JAR')),
        ]
    cmd += [
        '-classpath',
        ':'.join(['$SCARAB'] + kv.get('JAR', []) + kv.get('CLASSPATH', [])),
        '-Dfile.encoding=UTF-8',
    ]

    cmd += flat

    if 'DEBUG' not in kv:
        cmd += ['HIDE_OUTPUT']

    for k in 'IN', 'OUT', 'OUT_NOAUTO', 'OUTPUT_INCLUDES':
        if kv.get(k):
            cmd += [k] + kv[k]

    unit.on_run_java(cmd)
Пример #32
0
def parse_words(words):
    kv = extract_words(words, {'OUT', 'TEMPLATE'})
    ws = []
    for item in ('OUT', 'TEMPLATE'):
        for i, word in list(enumerate(kv[item])):
            if word == 'CUSTOM_PROPERTY':
                ws += kv[item][i:]
                kv[item] = kv[item][:i]
    tepmlates = kv['TEMPLATE']
    outputs = kv['OUT']
    if len(outputs) < len(tepmlates):
        ymake.report_configure_error(
            'To many arguments for TEMPLATE parameter')
        return
    if ws and ws[0] != 'CUSTOM_PROPERTY':
        ymake.report_configure_error('''Can't parse {}'''.format(ws))
    custom_props = []
    for item in ws:
        if item == 'CUSTOM_PROPERTY':
            custom_props.append([])
        else:
            custom_props[-1].append(item)
    props = []
    for p in custom_props:
        if not p:
            ymake.report_configure_error('Empty CUSTOM_PROPERTY')
            continue
        props.append('-B')
        if len(p) > 1:
            props.append(
                base64.b64encode("{}={}".format(p[0], ' '.join(p[1:]))))
        else:
            ymake.report_configure_error(
                'CUSTOM_PROPERTY "{}" value is not specified'.format(p[0]))
    for i, o in enumerate(outputs):
        yield o, tepmlates[min(i, len(tepmlates) - 1)], props
Пример #33
0
def on_go_process_srcs(unit):
    """
        _GO_PROCESS_SRCS() macro processes only 'CGO' files. All remaining *.go files
        and other input files are currently processed by a link command of the
        GO module (GO_LIBRARY, GO_PROGRAM)
    """

    srcs_files = get_appended_values(unit, 'GO_SRCS_VALUE')
    for f in srcs_files:
        if f.endswith('_test.go'):
            ymake.report_configure_error('file {} must be listed in GO_TEST_SRCS() or GO_XTEST_SRCS() macros'.format(f))
    go_test_files = get_appended_values(unit, 'GO_TEST_SRCS_VALUE')
    go_xtest_files = get_appended_values(unit, 'GO_XTEST_SRCS_VALUE')
    for f in go_test_files + go_xtest_files:
        if not f.endswith('_test.go'):
            ymake.report_configure_error('file {} should not be listed in GO_TEST_SRCS() or GO_XTEST_SRCS() macros'.format(f))

    if unit.get('GO_TEST_MODULE') and unit.get('GO_TEST_COVER'):
        temp_srcs_files = []
        cover_info = []
        for f in srcs_files:
            if f.endswith('.go') and not f.endswith('_test.go'):
                cover_var = 'GoCover_' + base64.b32encode(f).rstrip('=')
                cover_file = unit.resolve_arc_path(f)
                unit.on_go_gen_cover_go([cover_file, cover_var])
                if cover_file.startswith('$S/'):
                    cover_file = arc_project_prefix + cover_file[3:]
                cover_info.append('{}:{}'.format(cover_var, cover_file))
            else:
                temp_srcs_files.append(f)
        srcs_files = temp_srcs_files
        unit.set(['GO_SRCS_VALUE', ' '.join(srcs_files)])
        unit.set(['GO_COVER_INFO_VALUE', ' '.join(cover_info)])

    resolved_go_files = []
    for path in srcs_files + go_test_files + go_xtest_files:
        if path.endswith(".go"):
            resolved = unit.resolve_arc_path([path])
            if resolved != path and not resolved.startswith("$S/vendor/") and not resolved.startswith("$S/contrib/"):
                resolved_go_files.append(resolved)
    if resolved_go_files:
        basedirs = {}
        for f in resolved_go_files:
            basedir = os.path.dirname(f)
            if basedir not in basedirs:
                basedirs[basedir] = []
            basedirs[basedir].append(f)
        for basedir in basedirs:
            unit.onadd_check(["gofmt"] + basedirs[basedir])

    go_std_root = unit.get('GOSTD') + os.path.sep

    proto_files = filter(lambda x: x.endswith('.proto'), srcs_files)
    if len(proto_files) > 0:
        for f in proto_files:
            unit.on_go_proto_cmd(f)

    in_files = filter(lambda x: x.endswith('.in'), srcs_files)
    if len(in_files) > 0:
        for f in in_files:
            unit.onsrc(f)

    if compare_versions('1.12', unit.get('GOSTD_VERSION')) >= 0:
        asm_files = filter(lambda x: x.endswith('.s'), srcs_files)
        if len(asm_files) > 0:
            unit.on_go_compile_symabis(asm_files)

    s_files = filter(lambda x: x.endswith('.S'), srcs_files)
    c_files = filter(lambda x: x.endswith('.c'), srcs_files)
    syso_files = filter(lambda x: x.endswith('.syso'), srcs_files)
    cgo_files = get_appended_values(unit, 'CGO_SRCS_VALUE')

    cgo_cflags = []
    if len(c_files) + len(s_files) + len(cgo_files) > 0:
        cgo_cflags = get_appended_values(unit, 'CGO_CFLAGS_VALUE')

    for f in c_files + s_files:
        unit.onsrc([f] + cgo_cflags)

    if len(cgo_files) > 0:
        if not unit.enabled('CGO_ENABLED'):
            ymake.report_configure_error('trying to build with CGO (CGO_SRCS is non-empty) when CGO is disabled')
        import_path = rootrel_arc_src(unit.path(), unit)
        if import_path.startswith(go_std_root):
            import_path = import_path[len(go_std_root):]
        if import_path != runtime_cgo_path:
            unit.onpeerdir(os.path.join(go_std_root, runtime_cgo_path))
        race_mode = 'race' if unit.enabled('RACE') else 'norace'
        import_runtime_cgo = 'false' if import_path in import_runtime_cgo_false[race_mode] else 'true'
        import_syscall = 'false' if import_path in import_syscall_false[race_mode] else 'true'
        args = [import_path] + cgo_files + ['FLAGS', '-import_runtime_cgo=' + import_runtime_cgo, '-import_syscall=' + import_syscall]
        unit.on_go_compile_cgo1(args)
        cgo2_cflags = get_appended_values(unit, 'CGO2_CFLAGS_VALUE')
        for f in cgo_files:
            if f.endswith('.go'):
                unit.onsrc([f[:-2] + 'cgo2.c'] + cgo_cflags + cgo2_cflags)
            else:
                ymake.report_configure_error('file {} should not be listed in CGO_SRCS() macros'.format(f))
        args = [go_package_name(unit)] + cgo_files
        if len(c_files) > 0:
            args += ['C_FILES'] + c_files
        if len(s_files) > 0:
            args += ['S_FILES'] + s_files
        if len(syso_files) > 0:
            args += ['OBJ_FILES'] + syso_files
        unit.on_go_compile_cgo2(args)
Пример #34
0
def onpy_srcs(unit, *args):
    """
        PY_SRCS() - is rule to build extended versions of Python interpreters and containing all application code in its executable file. It can be used to collect only the executables but not shared libraries, and, in particular, not to collect the modules that are imported using import directive.
        The main disadvantage is the lack of IDE support; There is also no readline yet.
        The application can be collect from any of the sources from which the C library, and with the help of PY_SRCS .py , .pyx,.proto and .swg files.
        At the same time extensions for Python on C language generating from .pyx and .swg, will be registered in Python's as built-in modules, and sources on .py are stored as static data: when the interpreter starts, the initialization code will add a custom loader of these modules to sys.meta_path.
        By default .pyx files are collected as C++-extensions. To collect them as C (similar to BUILDWITH_CYTHON_C, but with the ability to specify namespace), you must specify the Directive CYTHON_C.
        Building with pyx automatically registers modules, you do not need to call PY_REGISTER for them
        __init__.py never required, but if present (and specified in PY_SRCS), it will be imported when you import package modules with __init__.py Oh.

        Example of library declaration with PY_SRCS():
        PY_LIBRARY(mymodule)
        PY_SRCS({| CYTHON_C} { | TOP_LEVEL | NAMESPACE ns} a.py sub/dir/b.py e.proto sub/dir/f.proto c.pyx sub/dir/d.pyx g.swg sub/dir/h.swg)
        END()

        Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/
    """
    # Each file arg must either be a path, or "${...}/buildpath=modname", where
    # "${...}/buildpath" part will be used as a file source in a future macro,
    # and "modname" will be used as a module name.
    unit.onuse_python([])

    if '/library/python/runtime' not in unit.path():
        unit.onpeerdir(['library/python/runtime'])

    if unit.get('MODULE_TYPE') == 'PROGRAM':
        py_program(unit)

    ns = (unit.get('PY_NAMESPACE_VALUE')
          or unit.path()[3:].replace('/', '.')) + '.'
    cython_directives = []

    pyxs_c = []
    pyxs_cpp = []
    pyxs = pyxs_cpp
    pys = []
    protos = []
    evs = []
    swigs = []

    args = iter(args)
    for arg in args:
        # Namespace directives.
        if arg == 'TOP_LEVEL':
            ns = ''
        elif arg == 'NAMESPACE':
            ns = next(args) + '.'
        # Cython directives.
        elif arg == 'CYTHON_C':
            pyxs = pyxs_c
        elif arg == 'CYTHON_CPP':
            pyxs = pyxs_cpp
        elif arg == 'CYTHON_DIRECTIVE':
            cython_directives += ['-X', next(args)]
        # Unsupported but legal PROTO_LIBRARY arguments.
        elif arg == 'GLOBAL' or arg.endswith('.gztproto'):
            pass
        # Sources.
        else:
            if '=' in arg:
                path, mod = arg.split('=', 1)
            else:
                path = arg
                if arg == '__main__.py' or arg.endswith('/__main__.py'):
                    mod = '__main__'
                else:
                    mod = ns + stripext(arg).replace('/', '.')

            pathmod = (path, mod)

            if path.endswith('.py'):
                pys.append(pathmod)
            elif path.endswith('.pyx'):
                pyxs.append(pathmod)
            elif path.endswith('.proto'):
                protos.append(pathmod)
            elif path.endswith('.ev'):
                evs.append(pathmod)
            elif path.endswith('.swg'):
                swigs.append(path)  # ignore mod, use last (and only) ns
            else:
                ymake.report_configure_error(
                    'in PY_SRCS: unrecognized arg {!r}'.format(path))

    if pyxs:
        for pyxs, cython in [
            (pyxs_c, unit.onbuildwith_cython_c),
            (pyxs_cpp, unit.onbuildwith_cython_cpp),
        ]:
            for path, mod in pyxs:
                cython([
                    path,
                    '--module-name',
                    mod,
                    '--init-name',
                    'init' + mangle(mod),
                ] + cython_directives)
                unit.onpy_register([mod])

    if pys:
        res = []

        for path, mod in pys:
            root_rel_path = rootrel_arc_src(path, unit)
            unit.onpy_compile_bytecode([root_rel_path + '-', path])
            key = '/py_modules/' + mod
            res += [
                path,
                key,
                '-',
                'resfs/src/{}={}'.format(key, root_rel_path),
                path + '.yapyc',
                '/py_code/' + mod,
            ]

        unit.onresource(res)
        add_python_lint_checks(unit, [path for path, mod in pys])

    if protos:
        if '/contrib/libs/protobuf/python/google_lib' not in unit.path():
            unit.onpeerdir(['contrib/libs/protobuf/python/google_lib'])

        grpc = unit.get('GRPC_FLAG') == 'yes'

        if grpc:
            unit.onpeerdir(['contrib/libs/grpc/python'])

        unit.ongenerate_py_protos([path for path, mod in protos])
        unit.onpy_srcs([pb2_arg(path, mod, unit) for path, mod in protos])

        if grpc:
            unit.onpy_srcs(
                [pb2_grpc_arg(path, mod, unit) for path, mod in protos])

    if evs:
        if '/contrib/libs/protobuf/python/google_lib' not in unit.path():
            unit.onpeerdir(['contrib/libs/protobuf/python/google_lib'])

        unit.ongenerate_py_evs([path for path, mod in evs])
        unit.onpy_srcs([ev_arg(path, mod, unit) for path, mod in evs])

    if swigs:
        unit.onsrcs(swigs)
        prefix = unit.get('MODULE_PREFIX')
        project = unit.get('REALPRJNAME')
        unit.onpy_register([prefix + project])
        path = '${ARCADIA_BUILD_ROOT}/' + '{}/{}.py'.format(
            unit.path()[3:], project)
        arg = '{}={}'.format(path, ns + project.replace('/', '.'))
        unit.onpy_srcs([arg])
Пример #35
0
def onpy_srcs(unit, *args):
    # Each file arg must either be a path, or "${...}/buildpath=modname", where
    # "${...}/buildpath" part will be used as a file source in a future macro,
    # and "modname" will be used as a module name.
    unit.onuse_python([])

    if '/library/python/runtime' not in unit.path():
        unit.onpeerdir(['library/python/runtime'])

    if unit.get('MODULE_TYPE') == 'PROGRAM':
        py_program(unit)

    ns = (unit.get('PY_NAMESPACE_VALUE') or unit.path()[3:].replace('/', '.')) + '.'
    cython_directives = []

    pyxs_c = []
    pyxs_cpp = []
    pyxs = pyxs_cpp
    pys = []
    protos = []
    evs = []
    swigs = []

    args = iter(args)
    for arg in args:
        # Namespace directives.
        if arg == 'TOP_LEVEL':
            ns = ''
        elif arg == 'NAMESPACE':
            ns = next(args) + '.'
        # Cython directives.
        elif arg == 'CYTHON_C':
            pyxs = pyxs_c
        elif arg == 'CYTHON_CPP':
            pyxs = pyxs_cpp
        elif arg == 'CYTHON_DIRECTIVE':
            cython_directives += ['-X', next(args)]
        # Unsupported but legal PROTO_LIBRARY arguments.
        elif arg == 'GLOBAL' or arg.endswith('.gztproto'):
            pass
        # Sources.
        else:
            if '=' in arg:
                path, mod = arg.split('=', 1)
            else:
                path = arg
                if arg == '__main__.py' or arg.endswith('/__main__.py'):
                    mod = '__main__'
                else:
                    mod = ns + stripext(arg).replace('/', '.')

            pathmod = (path, mod)

            if path.endswith('.py'):
                pys.append(pathmod)
            elif path.endswith('.pyx'):
                pyxs.append(pathmod)
            elif path.endswith('.proto'):
                protos.append(pathmod)
            elif path.endswith('.ev'):
                evs.append(pathmod)
            elif path.endswith('.swg'):
                swigs.append(path)  # ignore mod, use last (and only) ns
            else:
                ymake.report_configure_error('in PY_SRCS: unrecognized arg {!r}'.format(path))

    if pyxs:
        for pyxs, cython in [
            (pyxs_c, unit.onbuildwith_cython_c),
            (pyxs_cpp, unit.onbuildwith_cython_cpp),
        ]:
            for path, mod in pyxs:
                cython([
                    path,
                    '--module-name', mod,
                    '--init-name', 'init' + mangle(mod),
                ] + cython_directives)
                unit.onpy_register([mod])

    if pys:
        res = []

        for path, mod in pys:
            root_rel_path = rootrel_arc_src(path, unit)
            unit.onpy_compile_bytecode([root_rel_path + '-', path])
            key = '/py_modules/' + mod
            res += [
                path, key,
                '-', 'resfs/src/{}={}'.format(key, root_rel_path),
                path + '.yapyc', '/py_code/' + mod,
            ]

        unit.onresource(res)
        add_python_lint_checks(unit, [path for path, mod in pys])

    if protos:
        if '/contrib/libs/protobuf/python/google_lib' not in unit.path():
            unit.onpeerdir(['contrib/libs/protobuf/python/google_lib'])

        grpc = unit.get('GRPC_FLAG') == 'yes'

        if grpc:
            unit.onpeerdir(['contrib/libs/grpc/python'])

        unit.ongenerate_py_protos([path for path, mod in protos])
        unit.onpy_srcs([pb2_arg(path, mod, unit) for path, mod in protos])

        if grpc:
            unit.onpy_srcs([pb2_grpc_arg(path, mod, unit) for path, mod in protos])

    if evs:
        if '/contrib/libs/protobuf/python/google_lib' not in unit.path():
            unit.onpeerdir(['contrib/libs/protobuf/python/google_lib'])

        unit.ongenerate_py_evs([path for path, mod in evs])
        unit.onpy_srcs([ev_arg(path, mod, unit) for path, mod in evs])

    if swigs:
        unit.onsrcs(swigs)
        prefix = unit.get('MODULE_PREFIX')
        project = unit.get('REALPRJNAME')
        unit.onpy_register([prefix + project])
        path = '${ARCADIA_BUILD_ROOT}/' + '{}/{}.py'.format(unit.path()[3:], project)
        arg = '{}={}'.format(path, ns + project.replace('/', '.'))
        unit.onpy_srcs([arg])
Пример #36
0
def onpy3_srcs(unit, *args):
    # Each file arg must either be a path, or "${...}/buildpath=modname", where
    # "${...}/buildpath" part will be used as a file source in a future macro,
    # and "modname" will be used as a module name.
    if '/contrib/tools/python3/src/Lib' not in unit.path():
        unit.onuse_python3([])

        if '/library/python/runtime_py3' not in unit.path():
            unit.onpeerdir(['library/python/runtime_py3'])

    if unit.get('MODULE_TYPE') == 'PROGRAM':
        py3_program(unit)

    py_namespace_value = unit.get('PY_NAMESPACE_VALUE')
    if py_namespace_value == ".":
        ns = ""
    else:
        ns = (unit.get('PY_NAMESPACE_VALUE') or unit.path()[3:].replace('/', '.')) + '.'
    cython_directives = []

    pyxs_c = []
    pyxs_cpp = []
    pyxs = pyxs_cpp
    pys = []

    args = iter(args)
    for arg in args:
        # Namespace directives.
        if arg == 'TOP_LEVEL':
            ns = ''
        elif arg == 'NAMESPACE':
            ns = next(args) + '.'
        # Cython directives.
        elif arg == 'CYTHON_C':
            pyxs = pyxs_c
        elif arg == 'CYTHON_CPP':
            pyxs = pyxs_cpp
        elif arg == 'CYTHON_DIRECTIVE':
            cython_directives += ['-X', next(args)]
        # Unsupported but legal PROTO_LIBRARY arguments.
        elif arg == 'GLOBAL' or arg.endswith('.gztproto'):
            pass
        # Sources.
        else:
            if '=' in arg:
                path, mod = arg.split('=', 1)
            else:
                path = arg
                if arg == '__main__.py' or arg.endswith('/__main__.py'):
                    mod = '__main__'
                else:
                    mod = ns + stripext(arg).replace('/', '.')

            pathmod = (path, mod)

            if path.endswith('.py'):
                pys.append(pathmod)
            elif path.endswith('.pyx'):
                pyxs.append(pathmod)
            else:
                ymake.report_configure_error('in PY3_SRCS: unrecognized arg {!r}'.format(path))

    if pyxs:
        for pyxs, cython in [
            (pyxs_c, unit.onbuildwith_cython_c),
            (pyxs_cpp, unit.onbuildwith_cython_cpp),
        ]:
            for path, mod in pyxs:
                filename = get_pyx_mod_name(unit, path)
                cython([
                    path,
                    '--module-name', mod,
                    '--init-name', 'PyInit_' + mangle(mod),
                    '--source-root', '${ARCADIA_ROOT}',
                    # set arcadia root relative __file__ for generated modules
                    '-X', 'set_initial_path={}'.format(filename),
                ] + cython_directives)
                unit.onpy3_register([mod])

    if pys:
        res = []

        for path, mod in pys:
            root_rel_path = rootrel_arc_src(path, unit)
            unit.onpy3_compile_bytecode([root_rel_path + '-', path])
            dest = 'py/' + mod.replace('.', '/') + '.py'
            res += [
                'DEST', dest, path,
                'DEST', dest + '.yapyc', path + '.yapyc'
            ]

        unit.onresource_files(res)
Пример #37
0
def onpy_srcs(unit, *args):
    """
        PY_SRCS() - is rule to build extended versions of Python interpreters and containing all application code in its executable file. It can be used to collect only the executables but not shared libraries, and, in particular, not to collect the modules that are imported using import directive.
        The main disadvantage is the lack of IDE support; There is also no readline yet.
        The application can be collect from any of the sources from which the C library, and with the help of PY_SRCS .py , .pyx,.proto and .swg files.
        At the same time extensions for Python on C language generating from .pyx and .swg, will be registered in Python's as built-in modules, and sources on .py are stored as static data: when the interpreter starts, the initialization code will add a custom loader of these modules to sys.meta_path.
        By default .pyx files are collected as C++-extensions. To collect them as C (similar to BUILDWITH_CYTHON_C, but with the ability to specify namespace), you must specify the Directive CYTHON_C.
        Building with pyx automatically registers modules, you do not need to call PY_REGISTER for them
        __init__.py never required, but if present (and specified in PY_SRCS), it will be imported when you import package modules with __init__.py Oh.

        Example of library declaration with PY_SRCS():
        PY_LIBRARY(mymodule)
        PY_SRCS({| CYTHON_C} { | TOP_LEVEL | NAMESPACE ns} a.py sub/dir/b.py e.proto sub/dir/f.proto c.pyx sub/dir/d.pyx g.swg sub/dir/h.swg)
        END()

        Documentation: https://wiki.yandex-team.ru/devtools/commandsandvars/py_srcs/
    """
    # Each file arg must either be a path, or "${...}/buildpath=modname", where
    # "${...}/buildpath" part will be used as a file source in a future macro,
    # and "modname" will be used as a module name.
    unit.onuse_python([])

    if '/library/python/runtime' not in unit.path():
        unit.onpeerdir(['library/python/runtime'])

    if unit.get('MODULE_TYPE') == 'PROGRAM':
        py_program(unit)

    py_namespace_value = unit.get('PY_NAMESPACE_VALUE')
    if py_namespace_value == ".":
        ns = ""
    else:
        ns = (unit.get('PY_NAMESPACE_VALUE') or unit.path()[3:].replace('/', '.')) + '.'
    cython_directives = []

    pyxs_c = []
    pyxs_cpp = []
    pyxs = pyxs_cpp
    pys = []
    protos = []
    evs = []
    swigs = []

    args = iter(args)
    for arg in args:
        # Namespace directives.
        if arg == 'TOP_LEVEL':
            ns = ''
        elif arg == 'NAMESPACE':
            ns = next(args) + '.'
        # Cython directives.
        elif arg == 'CYTHON_C':
            pyxs = pyxs_c
        elif arg == 'CYTHON_CPP':
            pyxs = pyxs_cpp
        elif arg == 'CYTHON_DIRECTIVE':
            cython_directives += ['-X', next(args)]
        # Unsupported but legal PROTO_LIBRARY arguments.
        elif arg == 'GLOBAL' or arg.endswith('.gztproto'):
            pass
        # Sources.
        else:
            if '=' in arg:
                path, mod = arg.split('=', 1)
            else:
                path = arg
                if arg == '__main__.py' or arg.endswith('/__main__.py'):
                    mod = '__main__'
                else:
                    mod = ns + stripext(arg).replace('/', '.')

            pathmod = (path, mod)

            if path.endswith('.py'):
                pys.append(pathmod)
            elif path.endswith('.pyx'):
                pyxs.append(pathmod)
            elif path.endswith('.proto'):
                protos.append(pathmod)
            elif path.endswith('.ev'):
                evs.append(pathmod)
            elif path.endswith('.swg'):
                swigs.append(path)  # ignore mod, use last (and only) ns
            else:
                ymake.report_configure_error('in PY_SRCS: unrecognized arg {!r}'.format(path))

    if pyxs:
        for pyxs, cython in [
            (pyxs_c, unit.onbuildwith_cython_c),
            (pyxs_cpp, unit.onbuildwith_cython_cpp),
        ]:
            for path, mod in pyxs:
                filename = get_pyx_mod_name(unit, path)
                cython([
                    path,
                    '--module-name', mod,
                    '--init-name', 'init' + mangle(mod),
                    '--source-root', '${ARCADIA_ROOT}',
                    # set arcadia root relative __file__ for generated modules
                    '-X', 'set_initial_path={}'.format(filename),
                ] + cython_directives)
                unit.onpy_register([mod])

    if pys:
        res = []

        for path, mod in pys:
            root_rel_path = rootrel_arc_src(path, unit)
            unit.onpy_compile_bytecode([root_rel_path + '-', path])
            key = '/py_modules/' + mod
            res += [
                path, key,
                '-', 'resfs/src/{}={}'.format(key, root_rel_path),
                path + '.yapyc', '/py_code/' + mod,
            ]

        unit.onresource(res)
        add_python_lint_checks(unit, [path for path, mod in pys])

    if protos:
        if '/contrib/libs/protobuf/python/google_lib' not in unit.path():
            unit.onpeerdir(['contrib/libs/protobuf/python/google_lib'])

        grpc = unit.get('GRPC_FLAG') == 'yes'

        if grpc:
            unit.onpeerdir(['contrib/libs/grpc/python'])

        unit.ongenerate_py_protos([path for path, mod in protos])
        unit.onpy_srcs([pb2_arg(path, mod, unit) for path, mod in protos])

        if grpc:
            unit.onpy_srcs([pb2_grpc_arg(path, mod, unit) for path, mod in protos])

    if evs:
        if '/contrib/libs/protobuf/python/google_lib' not in unit.path():
            unit.onpeerdir(['contrib/libs/protobuf/python/google_lib'])

        unit.ongenerate_py_evs([path for path, mod in evs])
        unit.onpy_srcs([ev_arg(path, mod, unit) for path, mod in evs])

    if swigs:
        unit.onsrcs(swigs)
        prefix = unit.get('MODULE_PREFIX')
        project = unit.get('REALPRJNAME')
        unit.onpy_register([prefix + project])
        path = '${ARCADIA_BUILD_ROOT}/' + '{}/{}.py'.format(unit.path()[3:], project)
        arg = '{}={}'.format(path, ns + project.replace('/', '.'))
        unit.onpy_srcs([arg])
Пример #38
0
def onjava_module(unit, *args):
    unit.oninternal_recurse('contrib/java/org/sonarsource/scanner/cli/sonar-scanner-cli/2.8')  # TODO if <needs_sonar>

    if unit.get('COVERAGE'):
        unit.oninternal_recurse('devtools/jacoco-agent')

    args_delim = unit.get('ARGS_DELIM')

    data = {
        'PATH': unit.path(),
        'MODULE_TYPE': unit.get('MODULE_TYPE'),
        'MODULE_ARGS': unit.get('MODULE_ARGS'),
        'PEERDIR': unit.get_module_dirs('PEERDIRS'),
        'EXCLUDE': extract_macro_calls(unit, 'EXCLUDE_VALUE', args_delim),
        'JAVA_SRCS': extract_macro_calls(unit, 'JAVA_SRCS_VALUE', args_delim),
        'JAVAC_FLAGS': extract_macro_calls(unit, 'JAVAC_FLAGS_VALUE', args_delim),
        'ANNOTATION_PROCESSOR': extract_macro_calls(unit, 'ANNOTATION_PROCESSOR_VALUE', args_delim),
        'EXTERNAL_JAR': extract_macro_calls(unit, 'EXTERNAL_JAR_VALUE', args_delim),
        'RUN_JAVA_PROGRAM': extract_macro_calls2(unit, 'RUN_JAVA_PROGRAM_VALUE'),
        'ADD_WAR': extract_macro_calls(unit, 'ADD_WAR_VALUE', args_delim),
        'DEPENDENCY_MANAGEMENT': extract_macro_calls(unit, 'DEPENDENCY_MANAGEMENT_VALUE', args_delim),

        # TODO remove when java test dart is in prod
        'UNITTEST_DIR': unit.get('UNITTEST_DIR'),
        'SYSTEM_PROPERTIES': extract_macro_calls(unit, 'SYSTEM_PROPERTIES_VALUE', args_delim),
        'JVM_ARGS': extract_macro_calls(unit, 'JVM_ARGS_VALUE', args_delim),
        'TEST_CWD': extract_macro_calls(unit, 'TEST_CWD_VALUE', args_delim),
        'TEST_DATA': extract_macro_calls(unit, '__test_data', args_delim),
        'TEST_FORK_MODE': extract_macro_calls(unit, 'TEST_FORK_MODE', args_delim),
        'SPLIT_FACTOR': extract_macro_calls(unit, 'TEST_SPLIT_FACTOR', args_delim),
        'TIMEOUT': extract_macro_calls(unit, 'TEST_TIMEOUT', args_delim),
        'TAG': extract_macro_calls(unit, 'TEST_TAGS_VALUE', args_delim),
        'SIZE': extract_macro_calls(unit, 'TEST_SIZE_NAME', args_delim),
        'DEPENDS': extract_macro_calls(unit, 'TEST_DEPENDS_VALUE', args_delim),
        'IDEA_EXCLUDE': extract_macro_calls(unit, 'IDEA_EXCLUDE_DIRS_VALUE', args_delim),
    }
    if unit.get('JAVA_ADD_DLLS_VALUE') == 'yes':
        data['ADD_DLLS_FROM_DEPENDS'] = extract_macro_calls(unit, 'JAVA_ADD_DLLS_VALUE', args_delim)

    if unit.get('ERROR_PRONE_VALUE') == 'yes':
        data['ERROR_PRONE'] = extract_macro_calls(unit, 'ERROR_PRONE_VALUE', args_delim)

    if unit.get('MAKE_UBERJAR_VALUE') == 'yes':
        if unit.get('MODULE_TYPE') != 'JAVA_PROGRAM':
            ymake.report_configure_error('{}: UBERJAR supported only for JAVA_PROGRAM module type'.format(unit.path()))
        data['UBERJAR'] = extract_macro_calls(unit, 'MAKE_UBERJAR_VALUE', args_delim)
        data['UBERJAR_PREFIX'] = extract_macro_calls(unit, 'UBERJAR_PREFIX_VALUE', args_delim)
        data['UBERJAR_HIDE_EXCLUDE'] = extract_macro_calls(unit, 'UBERJAR_HIDE_EXCLUDE_VALUE', args_delim)
        data['UBERJAR_PATH_EXCLUDE'] = extract_macro_calls(unit, 'UBERJAR_PATH_EXCLUDE_VALUE', args_delim)

    for dm_paths in data['DEPENDENCY_MANAGEMENT']:
        for p in dm_paths:
            unit.oninternal_recurse(p)

    for k, v in data.items():
        if not v:
            data.pop(k)

    dart = 'JAVA_DART: ' + base64.b64encode(json.dumps(data)) + '\n' + DELIM + '\n'

    unit.set_property(['JAVA_DART_DATA', dart])
    if unit.get('MODULE_TYPE') in ('JAVA_PROGRAM', 'JAVA_LIBRARY', 'JTEST', 'TESTNG') and not unit.path().startswith('$S/contrib/java'):
        if (unit.get('CHECK_JAVA_DEPS_VALUE') or '').lower() == 'yes':
            unit.onjava_test_deps()
        if unit.get('LINT_LEVEL_VALUE') != "none":
            unit.onadd_check(['JAVA_STYLE', unit.get('LINT_LEVEL_VALUE')])