def onsplit_codegen(unit, *args): keywords = {"OUT_NUM": 1} flat_args, spec_args = sort_by_keywords(keywords, args) num_outputs = _DEFAULT_CPP_PARTS + _ADDITIONAL_STREAM_COUNT if "OUT_NUM" in spec_args: num_outputs = int(spec_args["OUT_NUM"][0]) tool = flat_args[0] prefix = flat_args[1] cmd = [tool, prefix, 'OUT'] for num in range(num_outputs): cmd.append('{}.{}.cpp'.format(prefix, num)) cpp_parts = int(num_outputs) - _ADDITIONAL_STREAM_COUNT cpp_parts_args = ['--cpp-parts', str(cpp_parts)] if len(flat_args) > 2: if flat_args[2] != 'OUTPUT_INCLUDES': cmd.append('OPTS') cmd += cpp_parts_args + flat_args[2:] else: cmd += ['OPTS'] + cpp_parts_args unit.onsplit_codegen_base(cmd)
def onregister_yql_python_udf(unit, *args): flat, kv = sort_by_keywords({'NAME': 1, 'RESOURCE_NAME': 1}, args) assert len(flat) == 0 name = get_or_default(kv, 'NAME', 'CustomPython') resource_name = get_or_default(kv, 'RESOURCE_NAME', name) use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') == 'yes' unit.onyql_abi_version(['2', '0', '0']) unit.onpeerdir(['yql/udfs/common/python/python_udf']) unit.onpeerdir(['yql/library/udf']) if use_arcadia_python: flavor = 'Arcadia' unit.onpeerdir([ 'library/python/runtime', 'yql/udfs/common/python/main', ]) else: flavor = 'System' output_includes = [ 'yql/udfs/common/python/python_udf/python_udf.h', 'yql/library/udf/udf_registrator.h', ] path = name + '.yql_python_udf.cpp' unit.onbuiltin_python([ 'build/scripts/gen_yql_python_udf.py', flavor, name, resource_name, path, 'OUT', path, 'OUTPUT_INCLUDES', ] + output_includes )
def oncopy(unit, *args): keywords = {'RESULT': 1, 'KEEP_DIR_STRUCT': 0, 'DESTINATION': 1, 'FROM': 1} flat_args, spec_args = sort_by_keywords(keywords, args) dest_dir = spec_args['DESTINATION'][0] if 'DESTINATION' in spec_args else '' from_dir = spec_args['FROM'][0] if 'FROM' in spec_args else '' keep_struct = 'KEEP_DIR_STRUCT' in spec_args save_in_var = 'RESULT' in spec_args targets = [] for source in flat_args: rel_path = '' path_list = source.split(os.sep) filename = path_list[-1] if keep_struct: if path_list[:-1]: rel_path = os.path.join(*path_list[:-1]) source_path = os.path.join(from_dir, rel_path, filename) target_path = os.path.join(dest_dir, rel_path, filename) if save_in_var: targets.append(target_path) unit.oncopy_file([source_path, target_path]) if save_in_var: unit.set([spec_args["RESULT"][0], " ".join(targets)])
def on_test(unit, *args): flat_args, spec_args = _common.sort_by_keywords({"DEPENDS": -1, "TIMEOUT": 1, "DATA": -1}, args) custom_deps = ' '.join(spec_args["DEPENDS"]) if "DEPENDS" in spec_args else '' test_data = '\"' + ';'.join(spec_args["DATA"]) + '\"' if "DATA" in spec_args else '' timeout = spec_args.get("TIMEOUT", ['0'])[0] test_type = flat_args[0] script_rel_path = None if test_type == "PY_TEST": script_rel_path = "py.test" elif test_type == "FLEUR": script_rel_path = "ytest.py" elif test_type == "PEP8": script_rel_path = "py.test.pep8" elif test_type == "PY_FLAKES": script_rel_path = "py.test.flakes" test_dir = unit.resolve(os.path.join(args[0])) test_file = flat_args[2] unit.set(['TEST-NAME', os.path.splitext(test_file)[0]]) unit.set(['TEST-TIMEOUT', timeout]) unit.set(['SCRIPT-REL-PATH', script_rel_path]) unit.set(['TESTED-PROJECT-NAME', test_file]) unit.set(['SOURCE-FOLDER-PATH', test_dir]) unit.set(['CUSTOM-DEPENDENCIES', custom_deps]) unit.set(['TEST-DATA', test_data])
def onacceleo(unit, *args): flat, kv = common.sort_by_keywords( {'XSD': -1, 'MTL': -1, 'MTL_ROOT': 1, 'MTL_EXTENSION': -1, 'LANG': -1, 'OUT': -1, 'OUT_NOAUTO': -1, 'OUTPUT_INCLUDES': -1, 'DEBUG': 0}, args ) try: mtlroot = kv['MTL_ROOT'][0] except Exception: mtlroot = unit.path().replace('$S/', '') classpath = ['$SCARAB', ] # XXX special word for ya make to replace following paths with real classpath classpath.append('tools/acceleo') classpath.extend(kv.get('MTL_EXTENSION', [])) if not unit.get('IDE_MSVS_CALL'): for jar in classpath[1:]: unit.oninternal_recurse(jar) classpath = ':'.join(classpath) # Generate java cmd cmd = [ '-classpath', classpath, '-Dfile.encoding=UTF-8', 'ru.yandex.se.logsng.tool.Cli', ] for xsd in kv.get('XSD', []): cmd += ['--xsd', xsd] for mtl in kv.get('MTL', []): cmd += ['--mtl', mtl] for lang in kv.get('LANG', []): cmd += ['--lang', lang] cmd += ['--output-dir', unit.path().replace('$S/', '${ARCADIA_BUILD_ROOT}/')] cmd += ['--build-root', '${ARCADIA_BUILD_ROOT}'] cmd += ['--source-root', '${ARCADIA_ROOT}'] cmd += ['--mtl-root', mtlroot] # Generate RUN_JAVA args run_java = cmd if 'DEBUG' not in kv: run_java += ['HIDE_OUTPUT'] inputs = kv.get('XSD', []) + kv.get('MTL', []) + kv.get('LANG', []) if inputs: run_java += ['IN'] + inputs for k in 'OUT', 'OUT_NOAUTO', 'OUTPUT_INCLUDES': if kv.get(k): run_java += [k] + kv[k] unit.onrun_java(run_java)
def onadd_ytest(unit, *args): keywords = {"DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0} flat_args, spec_args = _common.sort_by_keywords(keywords, args) if flat_args[1] == "fuzz.test": unit.ondata("arcadia/fuzzing/{}/corpus.json".format(strip_roots(unit.path()))) elif flat_args[1] == "coverage.extractor" and not match_coverage_extractor_requirements(unit): # XXX # Current ymake implementation doesn't allow to call macro inside the 'when' body # that's why we add ADD_YTEST(coverage.extractor) to every PROGRAM entry and check requirements later return fork_mode = [] if 'FORK_SUBTESTS' in spec_args: fork_mode.append('subtests') if 'FORK_TESTS' in spec_args: fork_mode.append('tests') fork_mode = fork_mode or spec_args.get('FORK_MODE', []) or unit.get('TEST_FORK_MODE').split() fork_mode = ' '.join(fork_mode) if fork_mode else '' test_record = { 'TEST-NAME': flat_args[0], 'SCRIPT-REL-PATH': flat_args[1], 'TESTED-PROJECT-NAME': unit.name(), 'TESTED-PROJECT-FILENAME': unit.filename(), 'SOURCE-FOLDER-PATH': unit.resolve(unit.path()), 'BUILD-FOLDER-PATH': strip_roots(unit.path()), 'BINARY-PATH': strip_roots(os.path.join(unit.path(), unit.filename())), 'CUSTOM-DEPENDENCIES': ' '.join(spec_args.get('DEPENDS', []) + get_values_list(unit, 'TEST_DEPENDS_VALUE')), 'TEST-RECIPES': prepare_recipes(unit.get("TEST_RECIPES_VALUE")), 'TEST-DATA': serialize_list(_common.filter_out_by_keyword(spec_args.get('DATA', []) + (unit.get(['__test_data']) or '').split(' ') + get_values_list(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED')), 'TEST-TIMEOUT': ''.join(spec_args.get('TIMEOUT', [])) or unit.get('TEST_TIMEOUT') or '', 'FORK-MODE': fork_mode, 'SPLIT-FACTOR': ''.join(spec_args.get('SPLIT_FACTOR', [])) or unit.get('TEST_SPLIT_FACTOR') or '', 'SIZE': ''.join(spec_args.get('SIZE', [])) or unit.get('TEST_SIZE_NAME') or '', 'TAG': serialize_list(spec_args.get('TAG', []) + (unit.get(['__test_tags']) or '').split(' ')), 'REQUIREMENTS': serialize_list(spec_args.get('REQUIREMENTS', []) + (unit.get(['__test_requirements']) or '').split(' ')), 'TEST-CWD': unit.get('TEST_CWD_VALUE') or '', 'FUZZ-DICTS': serialize_list(spec_args.get('FUZZ_DICTS', []) + get_unit_list_variable(unit, 'FUZZ_DICTS_VALUE')), 'FUZZ-OPTS': serialize_list(spec_args.get('FUZZ_OPTS', []) + get_unit_list_variable(unit, 'FUZZ_OPTS_VALUE')), 'BLOB': unit.get('TEST_BLOB_DATA') or '', 'SKIP_TEST': unit.get('SKIP_TEST_VALUE') or '', } is_fuzz_test = flat_args[1] == 'fuzz.test' and unit.get('FUZZING') == 'yes' if is_fuzz_test: # use all cores if fuzzing requested test_record['REQUIREMENTS'] = serialize_list(filter(None, deserialize_list(test_record['REQUIREMENTS']) + ["cpu:all", "ram:all"])) data = dump_test(test_record, is_fuzz_test=is_fuzz_test) if data: unit.set_property(["DART_DATA", data]) save_in_file(unit.get('TEST_DART_OUT_FILE'), data)
def onadd_pytest_bin(unit, *args): flat, kws = _common.sort_by_keywords({'RUNNER_BIN': 1}, args) if flat: ymake.report_configure_error( 'Unknown arguments found while processing add_pytest_bin macro: {!r}' .format(flat)) runner_bin = kws.get('RUNNER_BIN', [None])[0] test_type = 'py3test.bin' if (unit.get("PYTHON3") == 'yes') else "pytest.bin" add_test_to_dart(unit, test_type, runner_bin=runner_bin)
def onrun_java_program(unit, *args): flat, kv = common.sort_by_keywords( {'IN': -1, 'IN_DIR': -1, 'OUT': -1, 'OUT_DIR': -1, 'CWD': 1, 'CLASSPATH': -1, 'ADD_SRCS_TO_CLASSPATH': 0}, args ) for cp in kv.get('CLASSPATH', []): unit.oninternal_recurse(cp) prev = unit.get(['RUN_JAVA_PROGRAM_VALUE']) or '' new_val = (prev + ' ' + base64.b64encode(json.dumps(list(args), encoding='utf-8'))).strip() unit.set(['RUN_JAVA_PROGRAM_VALUE', new_val])
def onregister_yql_python_udf(unit, *args): flat, kv = sort_by_keywords( { 'NAME': 1, 'RESOURCE_NAME': 1, 'ADD_LIBRA_MODULES': 1 }, args) assert len(flat) == 0 name = get_or_default(kv, 'NAME', 'CustomPython') resource_name = get_or_default(kv, 'RESOURCE_NAME', name) add_libra_modules = get_or_default(kv, 'ADD_LIBRA_MODULES', 'no') == 'yes' use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') == 'yes' py3 = unit.get('PYTHON3') == 'yes' unit.onyql_abi_version(['2', '9', '0']) unit.onpeerdir(['yql/udfs/common/python/python_udf']) unit.onpeerdir(['ydb/library/yql/public/udf']) if add_libra_modules: unit.onpeerdir(['quality/user_sessions/libra_arc/noyql']) unit.onpeerdir(['yql/udfs/quality/libra/module']) if use_arcadia_python: flavor = 'Arcadia' unit.onpeerdir( ['library/python/runtime', 'yql/udfs/common/python/main'] if not py3 else ['library/python/runtime_py3', 'yql/udfs/common/python/main_py3']) else: flavor = 'System' output_includes = [ 'yql/udfs/common/python/python_udf/python_udf.h', 'ydb/library/yql/public/udf/udf_registrator.h', ] if add_libra_modules: output_includes.append('yql/udfs/quality/libra/module/module.h') path = name + '.yql_python_udf.cpp' libra_flag = '1' if add_libra_modules else '0' unit.onpython([ 'build/scripts/gen_yql_python_udf.py', flavor, name, resource_name, path, libra_flag, 'OUT', path, 'OUTPUT_INCLUDES', ] + output_includes)
def onadd_ytest(unit, *args): keywords = {"DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0} flat_args, spec_args = _common.sort_by_keywords(keywords, args) if flat_args[1] == "fuzz.test": unit.ondata("arcadia/fuzzing/{}/corpus.json".format(strip_roots(unit.path()))) elif flat_args[1] == "coverage.extractor" and not match_coverage_extractor_requirements(unit): # XXX # Current ymake implementation doesn't allow to call macro inside the 'when' body # that's why we add ADD_YTEST(coverage.extractor) to every PROGRAM entry and check requirements later return fork_mode = [] if 'FORK_SUBTESTS' in spec_args: fork_mode.append('subtests') if 'FORK_TESTS' in spec_args: fork_mode.append('tests') fork_mode = fork_mode or spec_args.get('FORK_MODE', []) or unit.get('TEST_FORK_MODE').split() fork_mode = ' '.join(fork_mode) if fork_mode else '' test_record = { 'TEST-NAME': flat_args[0], 'SCRIPT-REL-PATH': flat_args[1], 'TESTED-PROJECT-NAME': unit.name(), 'TESTED-PROJECT-FILENAME': unit.filename(), 'SOURCE-FOLDER-PATH': unit.resolve(unit.path()), 'BUILD-FOLDER-PATH': strip_roots(unit.path()), 'BINARY-PATH': strip_roots(os.path.join(unit.path(), unit.filename())), 'CUSTOM-DEPENDENCIES': ' '.join(spec_args.get('DEPENDS', []) + get_values_list(unit, 'TEST_DEPENDS_VALUE')), 'TEST-DATA': serialize_list(_common.filter_out_by_keyword(spec_args.get('DATA', []) + (unit.get(['__test_data']) or '').split(' ') + get_values_list(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED')), 'TEST-TIMEOUT': ''.join(spec_args.get('TIMEOUT', [])) or unit.get('TEST_TIMEOUT') or '', 'FORK-MODE': fork_mode, 'SPLIT-FACTOR': ''.join(spec_args.get('SPLIT_FACTOR', [])) or unit.get('TEST_SPLIT_FACTOR') or '', 'SIZE': ''.join(spec_args.get('SIZE', [])) or unit.get('TEST_SIZE_NAME') or '', 'TAG': serialize_list(spec_args.get('TAG', []) + (unit.get(['__test_tags']) or '').split(' ')), 'REQUIREMENTS': serialize_list(spec_args.get('REQUIREMENTS', []) + (unit.get(['__test_requirements']) or '').split(' ')), 'TEST-CWD': unit.get('TEST_CWD_VALUE') or '', 'FUZZ-DICTS': serialize_list(spec_args.get('FUZZ_DICTS', []) + get_unit_list_variable(unit, 'FUZZ_DICTS_VALUE')), 'FUZZ-OPTS': serialize_list(spec_args.get('FUZZ_OPTS', []) + get_unit_list_variable(unit, 'FUZZ_OPTS_VALUE')), 'BLOB': unit.get('TEST_BLOB_DATA') or '', 'SKIP_TEST': unit.get('SKIP_TEST_VALUE') or '', } # use all cores if fuzzing requested if flat_args[1] == 'fuzz.test' and unit.get('FUZZING') == 'yes': test_record['REQUIREMENTS'] = serialize_list(filter(None, deserialize_list(test_record['REQUIREMENTS']) + ["cpu:all"])) data = dump_test(test_record) if data: unit.set_property(["DART_DATA", data]) save_in_file(unit.get('TEST_DART_OUT_FILE'), data)
def onadd_check(unit, *args): flat_args, spec_args = _common.sort_by_keywords({"DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1}, args) check_type = flat_args[0] test_dir = unit.resolve(os.path.join(unit.path())) test_timeout = '' if check_type in ["PEP8", "PYFLAKES", "PY_FLAKES"]: script_rel_path = "py.lint.pylint" elif check_type == "JAVA_STYLE": if len(flat_args) < 2: raise Exception("Not enough arguments for JAVA_STYLE check") check_level = flat_args[1] allowed_levels = { 'base': '/yandex_checks.xml', 'strict': '/yandex_checks_strict.xml', 'extended': '/yandex_checks_extended.xml', } if check_level not in allowed_levels: raise Exception('{} is not allowed in LINT(), use one of {}'.format(check_level, allowed_levels.keys())) flat_args[1] = allowed_levels[check_level] script_rel_path = "java.style" test_timeout = '120' else: script_rel_path = check_type use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') test_record = { 'TEST-NAME': check_type.lower(), 'TEST-TIMEOUT': test_timeout, 'SCRIPT-REL-PATH': script_rel_path, 'TESTED-PROJECT-NAME': os.path.basename(test_dir), 'SOURCE-FOLDER-PATH': test_dir, 'CUSTOM-DEPENDENCIES': " ".join(spec_args.get('DEPENDS', [])), 'TEST-DATA': '', 'SPLIT-FACTOR': '', 'FORK-MODE': '', 'FORK-TEST-FILES': '', 'SIZE': 'SMALL', 'TAG': '', 'REQUIREMENTS': '', 'USE_ARCADIA_PYTHON': use_arcadia_python or '', 'OLD_PYTEST': 'no', 'PYTHON-PATHS': '', 'FILES': serialize_list(flat_args[1:]) } data = dump_test(test_record) if data: unit.set_property(["DART_DATA", data]) save_in_file(unit.get('TEST_DART_OUT_FILE'), data)
def onadd_check(unit, *args): flat_args, spec_args = _common.sort_by_keywords({"DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1}, args) check_type = flat_args[0] test_dir = unit.resolve(os.path.join(unit.path())) test_timeout = '' if check_type in ["PEP8", "PYFLAKES", "PY_FLAKES"]: script_rel_path = "py.lint.pylint" elif check_type == "JAVA_STYLE": if len(flat_args) < 2: raise Exception("Not enough arguments for JAVA_STYLE check") check_level = flat_args[1] allowed_levels = { 'base': '/yandex_checks.xml', 'strict': '/yandex_checks_strict.xml', } if check_level not in allowed_levels: raise Exception('{} is not allowed in LINT(), use one of {}'.format(check_level, allowed_levels.keys())) flat_args[1] = allowed_levels[check_level] script_rel_path = "java.style" test_timeout = '120' else: script_rel_path = check_type use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') test_record = { 'TEST-NAME': check_type.lower(), 'TEST-TIMEOUT': test_timeout, 'SCRIPT-REL-PATH': script_rel_path, 'TESTED-PROJECT-NAME': os.path.basename(test_dir), 'SOURCE-FOLDER-PATH': test_dir, 'CUSTOM-DEPENDENCIES': " ".join(spec_args.get('DEPENDS', [])), 'TEST-DATA': '', 'SPLIT-FACTOR': '', 'FORK-MODE': '', 'FORK-TEST-FILES': '', 'SIZE': 'SMALL', 'TAG': '', 'REQUIREMENTS': '', 'USE_ARCADIA_PYTHON': use_arcadia_python or '', 'OLD_PYTEST': 'no', 'PYTHON-PATHS': '', 'FILES': serialize_list(flat_args[1:]) } data = dump_test(test_record) if data: unit.set_property(["DART_DATA", data]) save_in_file(unit.get('TEST_DART_OUT_FILE'), data)
def ongenerate_script(unit, *args): """ heretic@ promised to make tutorial here Don't forget Feel free to remind """ flat, kv = common.sort_by_keywords( {'OUT': 1, 'TEMPLATE': -1, 'CUSTOM_PROPERTY': -1}, args ) if len(kv.get('TEMPLATE', [])) > 1: ymake.report_configure_error('To mane arguments for TEMPLATE parameter') prev = unit.get(['GENERATE_SCRIPT_VALUE']) or '' new_val = (prev + ' ' + base64.b64encode(json.dumps(list(args), encoding='utf-8'))).strip() unit.set(['GENERATE_SCRIPT_VALUE', new_val])
def onrun_java_program(unit, *args): """ Custom code generation @link: https://wiki.yandex-team.ru/yatool/java/#kodogeneracijarunjavaprogram """ flat, kv = common.sort_by_keywords({'IN': -1, 'IN_DIR': -1, 'OUT': -1, 'OUT_DIR': -1, 'CWD': 1, 'CLASSPATH': -1, 'ADD_SRCS_TO_CLASSPATH': 0}, args) depends = kv.get('CLASSPATH', []) + kv.get('JAR', []) if depends: # XXX: hack to force ymake to build dependencies unit.on_run_java(['TOOL'] + depends + ["OUT", "fake.out.{}".format(hash(tuple(depends)))]) prev = unit.get(['RUN_JAVA_PROGRAM_VALUE']) or '' new_val = (prev + ' ' + base64.b64encode(json.dumps(list(args), encoding='utf-8'))).strip() unit.set(['RUN_JAVA_PROGRAM_VALUE', new_val])
def onrun_java(unit, *args): flat, kv = common.sort_by_keywords( { 'CLASSPATH': -1, 'IN': -1, 'OUT': -1, 'OUT_NOAUTO': -1, 'OUTPUT_INCLUDES': -1, 'DEBUG': 0, 'JAR': 1 }, args) if not (kv.get('CLASSPATH', []) + kv.get('JAR', [])): ymake.report_configure_error( 'Java program for RUN_JAVA is not specified') depends = [] if not unit.get('IDE_MSVS_CALL'): for jar in (kv.get('CLASSPATH', []) + kv.get('JAR', [])): depends.append(jar) classpath = ':'.join(classpath) # Generate java cmd cmd = [] if kv.get('JAR'): cmd += [ '-jar', ':'.join(['$SCARAB_SLIM'] + kv.get('JAR')), ] cmd += [ '-classpath', ':'.join(['$SCARAB'] + kv.get('JAR', []) + kv.get('CLASSPATH', [])), '-Dfile.encoding=UTF-8', ] cmd += flat if 'DEBUG' not in kv: cmd += ['HIDE_OUTPUT'] for k in 'IN', 'OUT', 'OUT_NOAUTO', 'OUTPUT_INCLUDES': if kv.get(k): cmd += [k] + kv[k] if depends: cmd += ['TOOL'] + depends unit.on_run_java(cmd)
def onrun_java_program(unit, *args): """ Custom code generation @link: https://wiki.yandex-team.ru/yatool/java/#kodogeneracijarunjavaprogram """ flat, kv = common.sort_by_keywords( {'IN': -1, 'IN_DIR': -1, 'OUT': -1, 'OUT_DIR': -1, 'CWD': 1, 'CLASSPATH': -1, 'ADD_SRCS_TO_CLASSPATH': 0}, args ) for cp in kv.get('CLASSPATH', []): unit.oninternal_recurse(cp) prev = unit.get(['RUN_JAVA_PROGRAM_VALUE']) or '' new_val = (prev + ' ' + base64.b64encode(json.dumps(list(args), encoding='utf-8'))).strip() unit.set(['RUN_JAVA_PROGRAM_VALUE', new_val])
def onios_app_settings(unit, *args): tail, kv = common.sort_by_keywords({'OS_VERSION': 1, 'DEVICES': -1}, args) if tail: ymake.report_configure_error( 'Bad IOS_COMMON_SETTINGS usage - unknown data: ' + str(tail)) if kv.get('OS_VERSION', []): unit.onios_app_common_flags( ['--minimum-deployment-target', kv.get('OS_VERSION', [])[0]]) unit.onios_app_assets_flags( ['--filter-for-device-os-version', kv.get('OS_VERSION', [])[0]]) devices_flags = [] for device in kv.get('DEVICES', []): devices_flags += ['--target-device', device] if devices_flags: unit.onios_app_common_flags(devices_flags)
def onyabs_server_prepare_qxl_from_sandbox(unit, *args): (resource_id, qxl_name), kv = sort_by_keywords({'TIMESTAMP': 1}, args) json_basename = stripext(qxl_name) prefix = unit.path().replace('$S', '${ARCADIA_BUILD_ROOT}') + '/' pickle_name = prefix + json_basename + '.pickle' enabled_stats_name = prefix + json_basename + '.sts' timestamp = int(kv.get('TIMESTAMP', [DEFAULT_TIMESTAMP])[0]) unit.onfrom_sandbox(['FILE', resource_id, 'OUT', prefix + qxl_name]) unit.onrun_program([ MAKE_SQLITE_DB, '--credefs', CREDEFS_JSON, '--qxl', prefix + qxl_name, '--pickle', pickle_name, '--nowtime', str(timestamp - 1), '--stenabled', enabled_stats_name, 'IN', CREDEFS_JSON, 'IN', prefix + qxl_name, 'OUT', pickle_name, 'OUT', enabled_stats_name ]) request_dumps = [[ 'yabs_test.json', ], [ 'simulator.json', ], [ 'http_laas.json', 'http_metrika.json', 'http_metasearch.json', 'http_metapartner.json', 'http_metarank.json', ]] for group in request_dumps: args = [ MAKE_REQUESTS, prefix + qxl_name, ] args += group args += ['IN', prefix + qxl_name] for fname in group: args += ['OUT', fname] unit.onrun_program(args)
def onadd_test(unit, *args): flat_args, spec_args = _common.sort_by_keywords({"DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1}, args) test_type = flat_args[0] test_files = flat_args[1:] if test_type in ["PEP8", "PY_FLAKES"]: return # unit_path = unit.path() # paths = [] # for test_file in test_files: # if test_file == ".": # path_to_check = unit_path # else: # path_to_check = os.path.join(unit_path, test_file) # paths.append(path_to_check) # return onadd_check(unit, *tuple([test_type] + sorted(paths))) custom_deps = spec_args.get('DEPENDS', []) timeout = spec_args.get("TIMEOUT", []) if timeout: timeout = timeout[0] else: timeout = '0' fork_mode = [] if 'FORK_SUBTESTS' in spec_args: fork_mode.append('subtests') if 'FORK_TESTS' in spec_args: fork_mode.append('tests') fork_mode = fork_mode or spec_args.get('FORK_MODE', []) split_factor = ''.join(spec_args.get('SPLIT_FACTOR', [])) or '' test_size = ''.join(spec_args.get('SIZE', [])) or 'SMALL' test_dir = unit.resolve(os.path.join(unit.path())) tags = spec_args.get('TAG', []) + (unit.get(['__test_tags']) or '').split(' ') requirements = spec_args.get('REQUIREMENTS', []) + (unit.get(['__test_requirements']) or '').split(' ') test_data = spec_args.get("DATA", []) + (unit.get(['__test_data']) or '').split(' ') python_paths = get_values_list(unit, 'TEST_PYTHON_PATH_VALUE') if test_type == "PY_TEST": old_pytest = True else: old_pytest = False _dump_test(unit, test_type, test_files, timeout, test_dir, custom_deps, test_data, python_paths, split_factor, fork_mode, test_size, tags, requirements, None, old_pytest)
def onadd_test(unit, *args): flat_args, spec_args = _common.sort_by_keywords({"DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1}, args) test_type = flat_args[0] test_files = flat_args[1:] if test_type in ["PEP8", "PY_FLAKES"]: return # unit_path = unit.path() # paths = [] # for test_file in test_files: # if test_file == ".": # path_to_check = unit_path # else: # path_to_check = os.path.join(unit_path, test_file) # paths.append(path_to_check) # return onadd_check(unit, *tuple([test_type] + sorted(paths))) custom_deps = spec_args.get('DEPENDS', []) timeout = spec_args.get("TIMEOUT", []) if timeout: timeout = timeout[0] else: timeout = '0' fork_mode = [] if 'FORK_SUBTESTS' in spec_args: fork_mode.append('subtests') if 'FORK_TESTS' in spec_args: fork_mode.append('tests') fork_mode = fork_mode or spec_args.get('FORK_MODE', []) split_factor = ''.join(spec_args.get('SPLIT_FACTOR', [])) or '' test_size = ''.join(spec_args.get('SIZE', [])) or 'SMALL' test_dir = unit.path() tags = spec_args.get('TAG', []) + get_values_list(unit, 'TEST_TAGS_VALUE') requirements = spec_args.get('REQUIREMENTS', []) + get_values_list(unit, 'TEST_REQUIREMENTS_VALUE') test_data = spec_args.get("DATA", []) + get_norm_paths(unit, 'TEST_DATA_VALUE') python_paths = get_values_list(unit, 'TEST_PYTHON_PATH_VALUE') if test_type == "PY_TEST": old_pytest = True else: old_pytest = False _dump_test(unit, test_type, test_files, timeout, test_dir, custom_deps, test_data, python_paths, split_factor, fork_mode, test_size, tags, requirements, None, old_pytest)
def onllvm_bc(unit, *args): free_args, kwds = sort_by_keywords({'SYMBOLS': -1, 'NAME': 1}, args) name = kwds['NAME'][0] symbols = kwds.get('SYMBOLS') merged_bc = name + '_merged.bc' out_bc = name + '_optimized.bc' bcs = [] for x in free_args: rel_path = rootrel_arc_src(x, unit) bc_path = '${ARCADIA_BUILD_ROOT}/' + skip_build_root(rel_path) + '.bc' llvm_compile = unit.onllvm_compile_c if x.endswith('.c') else unit.onllvm_compile_cxx llvm_compile([rel_path, bc_path]) bcs.append(bc_path) unit.onllvm_link([merged_bc] + bcs) opt_opts = ['-O2', '-globalopt', '-globaldce'] if symbols: # XXX: '#' used instead of ',' to overcome ymake tendency to split everything by comma opt_opts += ['-internalize', '-internalize-public-api-list=' + '#'.join(symbols)] unit.onllvm_opt([merged_bc, out_bc] + opt_opts) unit.onresource([out_bc, '/llvm_bc/' + name])
def onrun_java_program(unit, *args): flat, kv = common.sort_by_keywords( { 'IN': -1, 'IN_DIR': -1, 'OUT': -1, 'OUT_DIR': -1, 'CWD': 1, 'CLASSPATH': -1, 'ADD_SRCS_TO_CLASSPATH': 0 }, args) for cp in kv.get('CLASSPATH', []): unit.oninternal_recurse(cp) prev = unit.get(['RUN_JAVA_PROGRAM_VALUE']) or '' new_val = ( prev + ' ' + base64.b64encode(json.dumps(list(args), encoding='utf-8'))).strip() unit.set(['RUN_JAVA_PROGRAM_VALUE', new_val])
def oncreate_init_py(unit, *args): keywords = {"DESTINATION": 1, "INCLUDING_DEST_DIR": 0, "RESULT": 1} flat_args, spec_args = sort_by_keywords(keywords, args) generated = [] dest_dir = spec_args["DESTINATION"][0] if "DESTINATION" in spec_args else "$ARCADIA_BUILD_ROOT" if "INCLUDING_DEST_DIR" in spec_args: generated.append(os.path.join(dest_dir, "__init__.py")) for proto_file in flat_args: path_list = proto_file.split(os.sep)[:-1] for idx, val in enumerate(path_list): generated.append(os.path.join(dest_dir, os.path.join(*path_list[0:len(path_list) - idx]), "__init__.py")) generated = list(set(generated)) unit.ontouch(generated) if "RESULT" in spec_args: unit.set([spec_args["RESULT"][0], " ".join(generated)])
def oncopy_files_to_build_prefix(unit, *args): keywords = {'PREFIX': 1, 'GLOBAL': 0} # NB! keyword 'GLOBAL' is a way to skip this word from the list of files flat_args, spec_args = sort_by_keywords(keywords, args) prefix = spec_args['PREFIX'][0] if 'PREFIX' in spec_args else '' if len(prefix) > 0: build_prefix = '/'.join([BUILD_ROOT, prefix]) else: build_prefix = BUILD_ROOT for arg in flat_args: if arg.startswith(build_prefix): # nothing to do pass elif len(prefix) > 0 and arg.startswith(BUILD_ROOT): unit.oncopy_file([arg, '{}{}'.format(build_prefix, arg[len(BUILD_ROOT):])]) elif arg.startswith(SOURCE_ROOT): unit.oncopy_file([arg, '{}{}'.format(build_prefix, arg[len(SOURCE_ROOT):])]) else: unit.oncopy_file([arg, '{}/{}/{}'.format(build_prefix, unit.get(['MODDIR']), arg)])
def on_run_jbuild_program(unit, *args): args = list(args) """ Custom code generation @link: https://wiki.yandex-team.ru/yatool/java/#kodogeneracijarunjavaprogram """ flat, kv = common.sort_by_keywords( { 'IN': -1, 'IN_DIR': -1, 'OUT': -1, 'OUT_DIR': -1, 'CWD': 1, 'CLASSPATH': -1, 'CP_USE_COMMAND_FILE': 1, 'ADD_SRCS_TO_CLASSPATH': 0 }, args) depends = kv.get('CLASSPATH', []) + kv.get('JAR', []) fake_out = None if depends: # XXX: hack to force ymake to build dependencies fake_out = "fake.out.{}".format(hash(tuple(args))) unit.on_run_java(['TOOL'] + depends + ["OUT", fake_out]) if not kv.get('CP_USE_COMMAND_FILE'): args += [ 'CP_USE_COMMAND_FILE', unit.get(['JAVA_PROGRAM_CP_USE_COMMAND_FILE']) or 'yes' ] if fake_out is not None: args += ['FAKE_OUT', fake_out] prev = unit.get(['RUN_JAVA_PROGRAM_VALUE']) or '' new_val = ( prev + ' ' + base64.b64encode(json.dumps(list(args), encoding='utf-8'))).strip() unit.set(['RUN_JAVA_PROGRAM_VALUE', new_val])
def onregister_yql_python_udf(unit, *args): flat, kv = sort_by_keywords({'NAME': 1, 'RESOURCE_NAME': 1}, args) assert len(flat) == 0 name = get_or_default(kv, 'NAME', 'CustomPython') resource_name = get_or_default(kv, 'RESOURCE_NAME', name) use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') == 'yes' py3 = unit.get('PYTHON3') == 'yes' unit.onyql_abi_version(['2', '9', '0']) unit.onpeerdir(['yql/udfs/common/python/python_udf']) unit.onpeerdir(['yql/library/udf']) if use_arcadia_python: flavor = 'Arcadia' unit.onpeerdir( ['library/python/runtime', 'yql/udfs/common/python/main'] if not py3 else [ 'library/python/runtime_py3', ]) else: flavor = 'System' output_includes = [ 'yql/udfs/common/python/python_udf/python_udf.h', 'yql/library/udf/udf_registrator.h', ] path = name + '.yql_python_udf.cpp' unit.onpython([ 'build/scripts/gen_yql_python_udf.py', flavor, name, resource_name, path, 'OUT', path, 'OUTPUT_INCLUDES', ] + output_includes)
def ytest_base(unit, related_prj_dir, related_prj_name, args): keywords = {"DEPENDS": -1, "DATA": -1} flat_args, spec_args = _common.sort_by_keywords(keywords, args) unit.set(['TEST-NAME', os.path.basename(flat_args[0])]) unit.set(['SCRIPT-REL-PATH', flat_args[1]]) unit.set(['SOURCE-FOLDER-PATH', related_prj_dir]) unit.set(['BUILD-FOLDER-PATH', os.path.join('$B', related_prj_dir)]) unit.set(['TESTED-BINARY-PATH', flat_args[0]]) custom_deps = ' '.join(spec_args["DEPENDS"]) if "DEPENDS" in spec_args else '' unit.set(['CUSTOM-DEPENDENCIES', custom_deps]) data_lst = spec_args.get('DATA', []) + (unit.get(['__test_data']) or '').split(' ') data = '\"' + ';'.join(data_lst) + '\"' if data_lst else '' unit.set(['TEST-DATA', data]) ya_root = unit.get('YA_ROOT') unit.set(['TEST_RUN_SCRIPT', 'devtools/{}/test/node/run_test.py'.format(ya_root)]) related_dirs_list = ['{ARCADIA_ROOT}/devtools/${YA_ROOT}', '${ARCADIA_ROOT}/devtools/${YA_ROOT}', '$RELATED_TARGET_SRCDIR'] related_dirs_value = [] for rel in related_dirs_list: related_dirs_value.extend(['--test-related-path', rel]) unit.set(['RELATED_DIRS', ' '.join(related_dirs_value)]) unit.set(['TEST_KV', '${{kv;hide:"test_related_dirs {}"}}'.format(' '.join(related_dirs_list))])
def ytest_base(unit, related_prj_dir, related_prj_name, args): keywords = {"DEPENDS": -1, "DATA": -1} flat_args, spec_args = _common.sort_by_keywords(keywords, args) unit.set(['TEST-NAME', os.path.basename(flat_args[0])]) unit.set(['SCRIPT-REL-PATH', flat_args[1]]) unit.set(['SOURCE-FOLDER-PATH', related_prj_dir]) unit.set(['BUILD-FOLDER-PATH', os.path.join('$B', related_prj_dir)]) unit.set(['TESTED-BINARY-PATH', flat_args[0]]) custom_deps = ' '.join(spec_args["DEPENDS"]) if "DEPENDS" in spec_args else '' unit.set(['CUSTOM-DEPENDENCIES', custom_deps]) data_lst = spec_args.get('DATA', []) + (unit.get(['__test_data']) or '').split(' ') data = '\"' + ';'.join(data_lst) + '\"' if data_lst else '' unit.set(['TEST-DATA', data]) ya_root = unit.get('YA_ROOT') unit.set(['TEST_RUN_SCRIPT', 'devtools/{}/test/node/run_test.py'.format(ya_root)]) related_dirs_list = ['${ARCADIA_ROOT}/devtools/svn_credentials', '{ARCADIA_ROOT}/devtools/${YA_ROOT}', '${ARCADIA_ROOT}/devtools/${YA_ROOT}', '$RELATED_TARGET_SRCDIR'] related_dirs_value = [] for rel in related_dirs_list: related_dirs_value.extend(['--test-related-path', rel]) unit.set(['RELATED_DIRS', ' '.join(related_dirs_value)]) unit.set(['TEST_KV', '${{kv;hide:"test_related_dirs {}"}}'.format(' '.join(related_dirs_list))])
def onrun_java(unit, *args): flat, kv = common.sort_by_keywords( {'CLASSPATH': -1, 'IN': -1, 'OUT': -1, 'OUT_NOAUTO': -1, 'OUTPUT_INCLUDES': -1, 'DEBUG': 0, 'JAR': 1}, args ) if not (kv.get('CLASSPATH', []) + kv.get('JAR', [])): ymake.report_configure_error('Java program for RUN_JAVA is not specified') if not unit.get('IDE_MSVS_CALL'): for jar in (kv.get('CLASSPATH', []) + kv.get('JAR', [])): unit.oninternal_recurse(jar) classpath = ':'.join(classpath) # Generate java cmd cmd = [] if kv.get('JAR'): cmd += [ '-jar', ':'.join(['$SCARAB_SLIM'] + kv.get('JAR')), ] cmd += [ '-classpath', ':'.join(['$SCARAB'] + kv.get('JAR', []) + kv.get('CLASSPATH', [])), '-Dfile.encoding=UTF-8', ] cmd += flat if 'DEBUG' not in kv: cmd += ['HIDE_OUTPUT'] for k in 'IN', 'OUT', 'OUT_NOAUTO', 'OUTPUT_INCLUDES': if kv.get(k): cmd += [k] + kv[k] unit.on_run_java(cmd)
def onyabs_generate_conf(unit, *args): flat, kv = sort_by_keywords( { 'MODE': 1, 'SCRIPT': 1, 'SRC': 1, 'TOOL': 1, 'CONF_DIR': 1, 'DEST': 1 }, args) src = get_or_default(kv, 'SRC', 'yabs/server/phantom') mode = get_or_default(kv, 'MODE', 'production') script = src + "/" + get_or_default(kv, 'SCRIPT', 'mkconf.py') conf = src + "/" + get_or_default(kv, 'CONF_DIR', 'conf-tmpl') tool = src + "/" + get_or_default(kv, 'TOOL', 'yabs_conf') for name in flat: filename = "/".join([conf, name]) unit.onbuiltin_python([ script, "--cluster-conf-binary", tool, "--mode", mode, "--dest-dir", "${BINDIR}", filename, "IN", filename, "OUT", "${BINDIR}/%s" % name, "TOOL", tool ])
def onacceleo(unit, *args): if unit.get("YMAKE_JAVA_MODULES") == "yes": return flat, kv = common.sort_by_keywords( { 'XSD': -1, 'MTL': -1, 'MTL_ROOT': 1, 'LANG': -1, 'OUT': -1, 'OUT_NOAUTO': -1, 'OUTPUT_INCLUDES': -1, 'DEBUG': 0 }, args) try: mtlroot = kv['MTL_ROOT'][0] except Exception: mtlroot = unit.path().replace('$S/', '') classpath = [ '$SCARAB', ] # XXX special word for ya make to replace following paths with real classpath classpath.append('tools/acceleo') depends = [] if not unit.get('IDE_MSVS_CALL'): for jar in classpath[1:]: depends.append(jar) classpath = ':'.join(classpath) # Generate java cmd cmd = [ '-classpath', classpath, '-Dfile.encoding=UTF-8', 'ru.yandex.se.logsng.tool.Cli', ] for xsd in kv.get('XSD', []): cmd += ['--xsd', xsd] for mtl in kv.get('MTL', []): cmd += ['--mtl', mtl] for lang in kv.get('LANG', []): cmd += ['--lang', lang] cmd += [ '--output-dir', unit.path().replace('$S/', '${ARCADIA_BUILD_ROOT}/') ] cmd += ['--build-root', '${ARCADIA_BUILD_ROOT}'] cmd += ['--source-root', '${ARCADIA_ROOT}'] cmd += ['--mtl-root', mtlroot] # Generate RUN_JAVA args run_java = cmd if 'DEBUG' not in kv: run_java += ['HIDE_OUTPUT'] inputs = kv.get('XSD', []) + kv.get('MTL', []) + kv.get('LANG', []) if inputs: run_java += ['IN'] + inputs for k in 'OUT', 'OUT_NOAUTO', 'OUTPUT_INCLUDES': if kv.get(k): run_java += [k] + kv[k] if depends: run_java += ['TOOL'] + depends unit.on_run_java(run_java)
def onadd_ytest(unit, *args): keywords = {"DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0} flat_args, spec_args = _common.sort_by_keywords(keywords, args) test_data = sorted(_common.filter_out_by_keyword(spec_args.get('DATA', []) + get_norm_paths(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED')) if flat_args[1] == "fuzz.test": unit.ondata("arcadia/fuzzing/{}/corpus.json".format(get_norm_unit_path(unit))) elif flat_args[1] == "go.test": data, _ = get_canonical_test_resources(unit) test_data += data elif flat_args[1] == "coverage.extractor" and not match_coverage_extractor_requirements(unit): # XXX # Current ymake implementation doesn't allow to call macro inside the 'when' body # that's why we add ADD_YTEST(coverage.extractor) to every PROGRAM entry and check requirements later return elif flat_args[1] == "no.test": return fork_mode = [] if 'FORK_SUBTESTS' in spec_args: fork_mode.append('subtests') if 'FORK_TESTS' in spec_args: fork_mode.append('tests') fork_mode = fork_mode or spec_args.get('FORK_MODE', []) or unit.get('TEST_FORK_MODE').split() fork_mode = ' '.join(fork_mode) if fork_mode else '' unit_path = get_norm_unit_path(unit) test_record = { 'TEST-NAME': flat_args[0], 'SCRIPT-REL-PATH': flat_args[1], 'TESTED-PROJECT-NAME': unit.name(), 'TESTED-PROJECT-FILENAME': unit.filename(), 'SOURCE-FOLDER-PATH': unit_path, # TODO get rid of BUILD-FOLDER-PATH 'BUILD-FOLDER-PATH': unit_path, 'BINARY-PATH': "{}/{}".format(unit_path, unit.filename()), 'CUSTOM-DEPENDENCIES': ' '.join(spec_args.get('DEPENDS', []) + get_values_list(unit, 'TEST_DEPENDS_VALUE')), 'TEST-RECIPES': prepare_recipes(unit.get("TEST_RECIPES_VALUE")), 'TEST-ENV': prepare_env(unit.get("TEST_ENV_VALUE")), # 'TEST-PRESERVE-ENV': 'da', 'TEST-DATA': serialize_list(test_data), 'TEST-TIMEOUT': ''.join(spec_args.get('TIMEOUT', [])) or unit.get('TEST_TIMEOUT') or '', 'FORK-MODE': fork_mode, 'SPLIT-FACTOR': ''.join(spec_args.get('SPLIT_FACTOR', [])) or unit.get('TEST_SPLIT_FACTOR') or '', 'SIZE': ''.join(spec_args.get('SIZE', [])) or unit.get('TEST_SIZE_NAME') or '', 'TAG': serialize_list(spec_args.get('TAG', []) + get_values_list(unit, 'TEST_TAGS_VALUE')), 'REQUIREMENTS': serialize_list(spec_args.get('REQUIREMENTS', []) + get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')), 'TEST-CWD': unit.get('TEST_CWD_VALUE') or '', 'FUZZ-DICTS': serialize_list(spec_args.get('FUZZ_DICTS', []) + get_unit_list_variable(unit, 'FUZZ_DICTS_VALUE')), 'FUZZ-OPTS': serialize_list(spec_args.get('FUZZ_OPTS', []) + get_unit_list_variable(unit, 'FUZZ_OPTS_VALUE')), 'YT-SPEC': serialize_list(spec_args.get('YT_SPEC', []) + get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')), 'BLOB': unit.get('TEST_BLOB_DATA') or '', 'SKIP_TEST': unit.get('SKIP_TEST_VALUE') or '', 'TEST_IOS_DEVICE_TYPE': unit.get('TEST_IOS_DEVICE_TYPE_VALUE') or '', 'TEST_IOS_RUNTIME_TYPE': unit.get('TEST_IOS_RUNTIME_TYPE_VALUE') or '', 'ANDROID_APK_TEST_ACTIVITY': unit.get('ANDROID_APK_TEST_ACTIVITY_VALUE') or '', 'TEST_PARTITION': unit.get("TEST_PARTITION") or 'SEQUENTIAL', } if flat_args[1] == 'fuzz.test' and unit.get('FUZZING') == 'yes': test_record['FUZZING'] = '1' # use all cores if fuzzing requested test_record['REQUIREMENTS'] = serialize_list(filter(None, deserialize_list(test_record['REQUIREMENTS']) + ["cpu:all", "ram:all"])) data = dump_test(unit, test_record) if data: unit.set_property(["DART_DATA", data]) save_in_file(unit.get('TEST_DART_OUT_FILE'), data)
def onadd_ytest(unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0 } flat_args, spec_args = _common.sort_by_keywords(keywords, args) if flat_args[1] == "fuzz.test": unit.ondata("arcadia/fuzzing/{}/corpus.json".format( strip_roots(unit.path()))) fork_mode = [] if 'FORK_SUBTESTS' in spec_args: fork_mode.append('subtests') if 'FORK_TESTS' in spec_args: fork_mode.append('tests') fork_mode = fork_mode or spec_args.get( 'FORK_MODE', []) or unit.get('TEST_FORK_MODE').split() fork_mode = ' '.join(fork_mode) if fork_mode else '' test_record = { 'TEST-NAME': flat_args[0], 'SCRIPT-REL-PATH': flat_args[1], 'TESTED-PROJECT-NAME': unit.name(), 'TESTED-PROJECT-FILENAME': unit.filename(), 'SOURCE-FOLDER-PATH': unit.resolve(unit.path()), 'BUILD-FOLDER-PATH': strip_roots(unit.path()), 'BINARY-PATH': strip_roots(os.path.join(unit.path(), unit.filename())), 'CUSTOM-DEPENDENCIES': ' '.join( spec_args.get('DEPENDS', []) + get_values_list(unit, 'TEST_DEPENDS_VALUE')), 'TEST-DATA': serialize_list( spec_args.get('DATA', []) + (unit.get(['__test_data']) or '').split(' ') + get_values_list(unit, 'TEST_DATA_VALUE')), 'TEST-TIMEOUT': ''.join(spec_args.get('TIMEOUT', [])) or unit.get('TEST_TIMEOUT') or '', 'FORK-MODE': fork_mode, 'SPLIT-FACTOR': ''.join(spec_args.get('SPLIT_FACTOR', [])) or unit.get('TEST_SPLIT_FACTOR') or '', 'SIZE': ''.join(spec_args.get('SIZE', [])) or unit.get('TEST_SIZE_NAME') or '', 'TAG': serialize_list( spec_args.get('TAG', []) + (unit.get(['__test_tags']) or '').split(' ')), 'REQUIREMENTS': serialize_list( spec_args.get('REQUIREMENTS', []) + (unit.get(['__test_requirements']) or '').split(' ')), 'TEST-CWD': unit.get('TEST_CWD_VALUE') or '', 'FUZZ-DICTS': serialize_list( spec_args.get('FUZZ_DICTS', []) + get_unit_list_variable(unit, 'FUZZ_DICTS_VALUE')), 'FUZZ-OPTS': serialize_list( spec_args.get('FUZZ_OPTS', []) + get_unit_list_variable(unit, 'FUZZ_OPTS_VALUE')), } # use all cores if fuzzing requested if flat_args[1] == 'fuzz.test' and unit.get('FUZZING') == 'yes': test_record['REQUIREMENTS'] = serialize_list( filter(None, deserialize_list(test_record['REQUIREMENTS']) + ["cpu:all"])) data = dump_test(test_record) if data: unit.set_property(["DART_DATA", data]) save_in_file(unit.get('TEST_DART_OUT_FILE'), data)
def onadd_check(unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return flat_args, spec_args = _common.sort_by_keywords({"DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1}, args) check_type = flat_args[0] test_dir = get_norm_unit_path(unit) test_timeout = '' fork_mode = '' extra_test_data = '' extra_test_dart_data = {} ymake_java_test = unit.get('YMAKE_JAVA_TEST') == 'yes' if check_type in ["flake8.py2", "flake8.py3"]: script_rel_path = check_type fork_mode = unit.get('TEST_FORK_MODE') or '' elif check_type == "black": script_rel_path = check_type fork_mode = unit.get('TEST_FORK_MODE') or '' elif check_type == "JAVA_STYLE": if ymake_java_test and not unit.get('ALL_SRCDIRS') or '': return if len(flat_args) < 2: raise Exception("Not enough arguments for JAVA_STYLE check") check_level = flat_args[1] allowed_levels = { 'base': '/yandex_checks.xml', 'strict': '/yandex_checks_strict.xml', 'extended': '/yandex_checks_extended.xml', 'library': '/yandex_checks_library.xml', } if check_level not in allowed_levels: raise Exception('{} is not allowed in LINT(), use one of {}'.format(check_level, allowed_levels.keys())) flat_args[1] = allowed_levels[check_level] if check_level == 'none': return script_rel_path = "java.style" test_timeout = '120' fork_mode = unit.get('TEST_FORK_MODE') or '' if ymake_java_test: extra_test_data = java_srcdirs_to_data(unit, 'ALL_SRCDIRS') extra_test_dart_data['JDK_RESOURCE'] = 'JDK' + (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION') or '_DEFAULT') elif check_type == "gofmt": script_rel_path = check_type go_files = flat_args[1:] if go_files: test_dir = os.path.dirname(go_files[0]).lstrip("$S/") else: script_rel_path = check_type use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') uid_ext = '' if check_type in ("check.data", "check.resource"): if unit.get("VALIDATE_DATA") == "no": return if check_type == "check.data": uid_ext = unit.get("SBR_UID_EXT").split(" ", 1)[-1] # strip variable name data_re = re.compile(r"sbr:/?/?(\d+)=?.*") data = flat_args[1:] resources = [] for f in data: matched = re.match(data_re, f) if matched: resources.append(matched.group(1)) if resources: test_files = serialize_list(resources) else: return else: test_files = serialize_list(flat_args[1:]) test_record = { 'TEST-NAME': check_type.lower(), 'TEST-TIMEOUT': test_timeout, 'SCRIPT-REL-PATH': script_rel_path, 'TESTED-PROJECT-NAME': os.path.basename(test_dir), 'SOURCE-FOLDER-PATH': test_dir, 'CUSTOM-DEPENDENCIES': " ".join(spec_args.get('DEPENDS', [])), 'TEST-DATA': extra_test_data, "SBR-UID-EXT": uid_ext, 'SPLIT-FACTOR': '', 'TEST_PARTITION': 'SEQUENTIAL', 'FORK-MODE': fork_mode, 'FORK-TEST-FILES': '', 'SIZE': 'SMALL', 'TAG': '', 'REQUIREMENTS': '', 'USE_ARCADIA_PYTHON': use_arcadia_python or '', 'OLD_PYTEST': 'no', 'PYTHON-PATHS': '', # TODO remove FILES, see DEVTOOLS-7052 'FILES': test_files, 'TEST-FILES': test_files, 'NO_JBUILD': 'yes' if ymake_java_test else 'no', } test_record.update(extra_test_dart_data) data = dump_test(unit, test_record) if data: unit.set_property(["DART_DATA", data]) save_in_file(unit.get('TEST_DART_OUT_FILE'), data)
def onyabs_server_build_yson_index(unit, *args): (qxl_dir, json_basename), kv = sort_by_keywords( { 'STATS': -1, 'TIMESTAMP': 1, 'STAGE': -1 }, args) tags, lm_tags, dssm_tags = get_tags(kv.get('STATS')) qxl_prefix = '${ARCADIA_BUILD_ROOT}/' + qxl_dir + '/' pickle_name = qxl_prefix + json_basename + '.pickle' prefix = unit.path().replace('$S', '${ARCADIA_BUILD_ROOT}') + '/' stage = int(kv.get('STAGE', [DEFAULT_STAGE])[0]) timestamp = int(kv.get('TIMESTAMP', [DEFAULT_TIMESTAMP])[0]) deps = [ qxl_dir, os.path.dirname(MKDB_INFO_JSON), ] unit.onpeerdir(deps) unit.ondepends(deps) for tag in tags: yson_name = prefix + tag + '.yson' meta_yson_name = prefix + tag + '.meta.yson' db_name = prefix + tag + '.yabs.' + CODEC unit.onrun_program([ MAKE_YSON_FROM_SQLITE, '--db', pickle_name, '--yson', yson_name, '--meta_yson', meta_yson_name, '--tag', tag, '--stage', str(stage), '--nowtime', str(timestamp), '--mkdb_info', '${ARCADIA_BUILD_ROOT}/' + MKDB_INFO_JSON, 'IN', pickle_name, 'IN', '${ARCADIA_BUILD_ROOT}/' + MKDB_INFO_JSON, 'OUT', yson_name, 'OUT', meta_yson_name, ]) # FIXME: path substitution for TOOLs is bugged, # you cannot pass two different TOOLs as arguments unit.onbuiltin_python([ BUILD_YSON_TEST_BASE, '--mkdb', YT_MKDB, '--yson', yson_name, '--output', db_name, '--tag', tag, '--timestamp', str(timestamp), '--compress', CODEC, 'IN', yson_name, 'OUT', db_name, 'TOOL', YT_MKDB, 'TOOL', UC, ]) for tag in lm_tags: db_name = prefix + tag + '.yabs.' + CODEC unit.onbuiltin_python([ BUILD_LM_TEST_BASE, '--mkdblm', MKDBLM, '--output', db_name, '--tag', tag, '--timestamp', str(timestamp), '--compress', CODEC, 'OUT', db_name, 'TOOL', MKDBLM, 'TOOL', UC, ]) for tag in dssm_tags: db_name = prefix + tag + '.yabs.' + CODEC unit.onbuiltin_python([ BUILD_DSSM_TEST_BASE, '--mkdbdssm', MKDBDSSM, '--output', db_name, '--tag', tag, '--compress', CODEC, 'OUT', db_name, 'TOOL', MKDBDSSM, 'TOOL', UC, ])
def onadd_check(unit, *args): flat_args, spec_args = _common.sort_by_keywords( { "DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1 }, args) check_type = flat_args[0] test_dir = get_norm_unit_path(unit) test_timeout = '' fork_mode = '' if check_type in ["flake8.py2", "flake8.py3"]: script_rel_path = check_type fork_mode = unit.get('TEST_FORK_MODE') or '' elif check_type == "JAVA_STYLE": if len(flat_args) < 2: raise Exception("Not enough arguments for JAVA_STYLE check") check_level = flat_args[1] allowed_levels = { 'base': '/yandex_checks.xml', 'strict': '/yandex_checks_strict.xml', 'extended': '/yandex_checks_extended.xml', 'library': '/yandex_checks_library.xml', } if check_level not in allowed_levels: raise Exception( '{} is not allowed in LINT(), use one of {}'.format( check_level, allowed_levels.keys())) flat_args[1] = allowed_levels[check_level] script_rel_path = "java.style" test_timeout = '120' fork_mode = unit.get('TEST_FORK_MODE') or '' elif check_type == "gofmt": script_rel_path = check_type go_files = flat_args[1:] if go_files: test_dir = os.path.dirname(go_files[0]).lstrip("$S/") else: script_rel_path = check_type use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') test_files = serialize_list(flat_args[1:]) test_record = { 'TEST-NAME': check_type.lower(), 'TEST-TIMEOUT': test_timeout, 'SCRIPT-REL-PATH': script_rel_path, 'TESTED-PROJECT-NAME': os.path.basename(test_dir), 'SOURCE-FOLDER-PATH': test_dir, 'CUSTOM-DEPENDENCIES': " ".join(spec_args.get('DEPENDS', [])), 'TEST-DATA': '', 'SPLIT-FACTOR': '', 'TEST_PARTITION': 'SEQUENTIAL', 'FORK-MODE': fork_mode, 'FORK-TEST-FILES': '', 'SIZE': 'SMALL', 'TAG': '', 'REQUIREMENTS': '', 'USE_ARCADIA_PYTHON': use_arcadia_python or '', 'OLD_PYTEST': 'no', 'PYTHON-PATHS': '', # TODO remove FILES, see DEVTOOLS-7052 'FILES': test_files, 'TEST-FILES': test_files, } data = dump_test(unit, test_record) if data: unit.set_property(["DART_DATA", data]) save_in_file(unit.get('TEST_DART_OUT_FILE'), data)