def onlarge_files(unit, *args):
"""
@usage LARGE_FILES([AUTOUPDATED] Files...)
Use large file ether from working copy or from remote storage via placeholder <File>.external
If <File> is present locally (and not a symlink!) it will be copied to build directory.
Otherwise macro will try to locate <File>.external, parse it retrieve ot during build phase.
"""
args = list(args)
if args and args[0] == 'AUTOUPDATED':
args = args[1:]
for arg in args:
if arg == 'AUTOUPDATED':
unit.message([
"warn",
"Please set AUTOUPDATED argument before other file names"
])
continue
src = unit.resolve_arc_path(arg)
if src.startswith("$S"):
msg = "Used local large file {}. Don't forget to run 'ya upload --update-external' and commit {}.{}".format(
src, src, PLACEHOLDER_EXT)
unit.message(["warn", msg])
unit.oncopy_file([arg, arg])
else:
out_file = strip_roots(os.path.join(unit.path(), arg))
external = "{}.{}".format(arg, PLACEHOLDER_EXT)
from_external_cmd = [external, out_file, 'OUT_NOAUTO', arg]
if os.path.dirname(arg):
from_external_cmd.extend(("RENAME", os.path.basename(arg)))
unit.on_from_external(from_external_cmd)
unit.onadd_check(['check.external', external])
def onadd_check_py_imports(unit, *args):
if unit.get('NO_CHECK_IMPORTS_FOR_VALUE').strip() == "":
return
unit.onpeerdir(['library/python/testing/import_test'])
check_type = "py.imports"
test_dir = unit.resolve(os.path.join(unit.path()))
use_arcadia_python = unit.get('USE_ARCADIA_PYTHON')
test_record = {
'TEST-NAME':
"pyimports",
'TEST-TIMEOUT':
'',
'SCRIPT-REL-PATH':
check_type,
'TESTED-PROJECT-NAME':
os.path.basename(test_dir),
'SOURCE-FOLDER-PATH':
test_dir,
'CUSTOM-DEPENDENCIES':
'',
'TEST-DATA':
'',
'TEST-ENV':
prepare_env(unit.get("TEST_ENV_VALUE")),
'SPLIT-FACTOR':
'',
'TEST_PARTITION':
'SEQUENTIAL',
'FORK-MODE':
'',
'FORK-TEST-FILES':
'',
'SIZE':
'SMALL',
'TAG':
'',
'USE_ARCADIA_PYTHON':
use_arcadia_python or '',
'OLD_PYTEST':
'no',
'PYTHON-PATHS':
'',
'FILES':
serialize_list([
"{}/{}".format(_common.strip_roots(unit.path()), unit.filename())
])
}
if unit.get('NO_CHECK_IMPORTS_FOR_VALUE') != "None":
test_record["NO-CHECK"] = serialize_list(
get_values_list(unit, 'NO_CHECK_IMPORTS_FOR_VALUE') or ["*"])
else:
test_record["NO-CHECK"] = ''
data = dump_test(unit, test_record)
if data:
unit.set_property(["DART_DATA", data])
save_in_file(unit.get('TEST_DART_OUT_FILE'), data)
def onrestrict_path(unit, *args):
if args:
if 'MSG' in args:
pos = args.index('MSG')
paths, msg = args[:pos], args[pos + 1:]
msg = ' '.join(msg)
else:
paths, msg = args, 'forbidden'
if not _common.strip_roots(unit.path()).startswith(paths):
error_msg = "Path '[[imp]]{}[[rst]]' is restricted - [[bad]]{}[[rst]]. Valid path prefixes are: [[unimp]]{}[[rst]]".format(unit.path(), msg, ', '.join(paths))
ymake.report_configure_error(error_msg)
def onjava_test_deps(unit, *args):
assert unit.get('MODULE_TYPE') is not None
assert len(args) == 1
mode = args[0]
path = _common.strip_roots(unit.path())
test_record = {
'SOURCE-FOLDER-PATH':
path,
'TEST-NAME':
'-'.join([
os.path.basename(os.path.dirname(path)),
os.path.basename(path), 'dependencies'
]).strip('-'),
'SCRIPT-REL-PATH':
'java.dependency.test',
'TEST-TIMEOUT':
'',
'TESTED-PROJECT-NAME':
path,
'TEST-DATA':
'',
'TEST_PARTITION':
'SEQUENTIAL',
'FORK-MODE':
'',
'SPLIT-FACTOR':
'',
'CUSTOM-DEPENDENCIES':
' '.join(get_values_list(unit, 'TEST_DEPENDS_VALUE')),
'TAG':
'',
'SIZE':
'SMALL',
'IGNORE_CLASSPATH_CLASH':
' '.join(get_values_list(unit, 'JAVA_IGNORE_CLASSPATH_CLASH_VALUE')),
# JTEST/JTEST_FOR only
'MODULE_TYPE':
unit.get('MODULE_TYPE'),
'UNITTEST_DIR':
'',
'SYSTEM_PROPERTIES':
'',
'TEST-CWD':
'',
}
if mode == 'strict':
test_record['STRICT_CLASSPATH_CLASH'] = 'yes'
data = dump_test(unit, test_record)
unit.set_property(['DART_DATA', data])
def match_coverage_extractor_requirements(unit):
# we shouldn't add test if
return all([
# tests are not requested
unit.get("TESTS_REQUESTED") == "yes",
# build doesn't imply clang coverage, which supports segment extraction from the binaries
unit.get("CLANG_COVERAGE") == "yes",
# contrib wasn't requested
implies(
_common.strip_roots(unit.path()).startswith("contrib/"),
unit.get("ENABLE_CONTRIB_COVERAGE") == "yes"),
])
def onlarge_files(unit, *args):
"""
@usage LARGE_FILES(Files...)
Use alrge file ether from working copy or from remote storage via placeholder <File>.remote
If <File> is presemt locally (and not a symlink!) it will be copied to build directory.
Otherwise macro will try to locate <File>.remote, parse it retrieve ot during build phase.
"""
args = list(args)
for arg in args:
src = unit.resolve_arc_path(arg)
if src.startswith("$S"):
msg = "Used local large file {}. Don't forget to run 'ya upload --update-external' and commit {}.{}".format(
src, src, PLACEHOLDER_EXT)
unit.message(["warn", msg])
unit.oncopy_file([arg, arg])
else:
out_file = strip_roots(os.path.join(unit.path(), arg))
external = "{}.{}".format(arg, PLACEHOLDER_EXT)
unit.on_from_external([external, out_file, 'OUT_NOAUTO', arg])
def _dump_test(
unit,
test_type,
test_files,
timeout,
test_dir,
custom_deps,
test_data,
python_paths,
split_factor,
fork_mode,
test_size,
tags,
requirements,
binary_path='',
old_pytest=False,
test_cwd=None,
runner_bin=None,
yt_spec=None,
data_files=None
):
if test_type == "PY_TEST":
script_rel_path = "py.test"
elif test_type == "PEP8":
script_rel_path = "py.test.pep8"
elif test_type == "PY_FLAKES":
script_rel_path = "py.test.flakes"
else:
script_rel_path = test_type
unit_path = unit.path()
fork_test_files = unit.get('FORK_TEST_FILES_MODE')
fork_mode = ' '.join(fork_mode) if fork_mode else ''
use_arcadia_python = unit.get('USE_ARCADIA_PYTHON')
if test_cwd:
test_cwd = test_cwd.replace("$TEST_CWD_VALUE", "").replace('"MACRO_CALLS_DELIM"', "").strip()
if binary_path:
if fork_test_files == 'on' or unit.get('FORK_TEST_FILES_FILTER') == 'yes':
tests = test_files
else:
tests = [os.path.basename(binary_path)]
else:
tests = test_files
for test_name in tests:
test_record = {
'TEST-NAME': os.path.splitext(test_name)[0],
'TEST-TIMEOUT': timeout,
'SCRIPT-REL-PATH': script_rel_path,
'TESTED-PROJECT-NAME': test_name,
'SOURCE-FOLDER-PATH': test_dir,
'CUSTOM-DEPENDENCIES': " ".join(custom_deps),
'TEST-ENV': prepare_env(unit.get("TEST_ENV_VALUE")),
# 'TEST-PRESERVE-ENV': 'da',
'TEST-DATA': serialize_list(sorted(_common.filter_out_by_keyword(test_data, 'AUTOUPDATED'))),
'TEST-RECIPES': prepare_recipes(unit.get("TEST_RECIPES_VALUE")),
'SPLIT-FACTOR': split_factor,
'TEST_PARTITION': unit.get('TEST_PARTITION') or 'SEQUENTIAL',
'FORK-MODE': fork_mode,
'FORK-TEST-FILES': fork_test_files,
'TEST-FILES': serialize_list(tests),
'SIZE': test_size,
'TAG': serialize_list(tags),
'REQUIREMENTS': serialize_list(requirements),
'USE_ARCADIA_PYTHON': use_arcadia_python or '',
'OLD_PYTEST': 'yes' if old_pytest else 'no',
'PYTHON-PATHS': serialize_list(python_paths),
'TEST-CWD': test_cwd or '',
'SKIP_TEST': unit.get('SKIP_TEST_VALUE') or '',
'BUILD-FOLDER-PATH': _common.strip_roots(unit_path),
'BLOB': unit.get('TEST_BLOB_DATA') or '',
'CANONIZE_SUB_PATH': unit.get('CANONIZE_SUB_PATH') or '',
}
if binary_path:
test_record['BINARY-PATH'] = _common.strip_roots(binary_path)
if runner_bin:
test_record['TEST-RUNNER-BIN'] = runner_bin
if yt_spec:
test_record['YT-SPEC'] = serialize_list(yt_spec)
data = dump_test(unit, test_record)
if data:
unit.set_property(["DART_DATA", data])
save_in_file(unit.get('TEST_DART_OUT_FILE'), data)
def onjava_test(unit, *args):
assert unit.get('MODULE_TYPE') is not None
if unit.get('MODULE_TYPE') == 'JTEST_FOR':
if not unit.get('UNITTEST_DIR'):
ymake.report_configure_error('skip JTEST_FOR in {}: no args provided'.format(unit.path()))
return
java_cp_arg_type = unit.get('JAVA_CLASSPATH_CMD_TYPE_VALUE') or 'MANIFEST'
if java_cp_arg_type not in ('MANIFEST', 'COMMAND_FILE', 'LIST'):
ymake.report_configure_error('{}: TEST_JAVA_CLASSPATH_CMD_TYPE({}) are invalid. Choose argument from MANIFEST, COMMAND_FILE or LIST)'.format(unit.path(), java_cp_arg_type))
return
unit_path = unit.path()
path = _common.strip_roots(unit_path)
test_data = get_norm_paths(unit, 'TEST_DATA_VALUE')
test_data.append('arcadia/build/scripts/unpacking_jtest_runner.py')
test_data.append('arcadia/build/scripts/run_testng.py')
data, data_files = get_canonical_test_resources(unit)
test_data += data
props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE'))
if error_mgs:
ymake.report_configure_error(error_mgs)
return
for prop in props:
if prop['type'] == 'file':
test_data.append(prop['path'].replace('${ARCADIA_ROOT}', 'arcadia'))
props = base64.b64encode(json.dumps(props, encoding='utf-8'))
test_cwd = unit.get('TEST_CWD_VALUE') or '' # TODO: validate test_cwd value
if unit.get('MODULE_TYPE') == 'TESTNG':
script_rel_path = 'testng.test'
elif unit.get('MODULE_TYPE') == 'JUNIT5':
script_rel_path = 'junit5.test'
else:
script_rel_path = 'junit.test'
test_record = {
'SOURCE-FOLDER-PATH': path,
'TEST-NAME': '-'.join([os.path.basename(os.path.dirname(path)), os.path.basename(path)]),
'SCRIPT-REL-PATH': script_rel_path,
'TEST-TIMEOUT': unit.get('TEST_TIMEOUT') or '',
'TESTED-PROJECT-NAME': path,
'TEST-ENV': prepare_env(unit.get("TEST_ENV_VALUE")),
# 'TEST-PRESERVE-ENV': 'da',
'TEST-DATA': serialize_list(sorted(_common.filter_out_by_keyword(test_data, 'AUTOUPDATED'))),
'FORK-MODE': unit.get('TEST_FORK_MODE') or '',
'SPLIT-FACTOR': unit.get('TEST_SPLIT_FACTOR') or '',
'CUSTOM-DEPENDENCIES': ' '.join(get_values_list(unit, 'TEST_DEPENDS_VALUE')),
'TAG': serialize_list(get_values_list(unit, 'TEST_TAGS_VALUE')),
'SIZE': unit.get('TEST_SIZE_NAME') or '',
'REQUIREMENTS': serialize_list(get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')),
'TEST-RECIPES': prepare_recipes(unit.get("TEST_RECIPES_VALUE")),
# JTEST/JTEST_FOR only
'MODULE_TYPE': unit.get('MODULE_TYPE'),
'UNITTEST_DIR': unit.get('UNITTEST_DIR') or '',
'JVM_ARGS': serialize_list(get_values_list(unit, 'JVM_ARGS_VALUE')),
'SYSTEM_PROPERTIES': props,
'TEST-CWD': test_cwd,
'SKIP_TEST': unit.get('SKIP_TEST_VALUE') or '',
'JAVA_CLASSPATH_CMD_TYPE': java_cp_arg_type,
}
data = dump_test(unit, test_record)
if data:
unit.set_property(['DART_DATA', data])
def get_norm_unit_path(unit, extra=None):
path = _common.strip_roots(unit.path())
if extra:
return '{}/{}'.format(path, extra)
return path
def onjava_module(unit, *args):
args_delim = unit.get('ARGS_DELIM')
data = {
'BUNDLE_NAME':
unit.name(),
'PATH':
unit.path(),
'MODULE_TYPE':
unit.get('MODULE_TYPE'),
'MODULE_ARGS':
unit.get('MODULE_ARGS'),
'MANAGED_PEERS':
'${MANAGED_PEERS}',
'MANAGED_PEERS_CLOSURE':
'${MANAGED_PEERS_CLOSURE}',
'NON_NAMAGEABLE_PEERS':
'${NON_NAMAGEABLE_PEERS}',
'TEST_CLASSPATH_MANAGED':
'${TEST_CLASSPATH_MANAGED}',
'EXCLUDE':
extract_macro_calls(unit, 'EXCLUDE_VALUE', args_delim),
'JAVA_SRCS':
extract_macro_calls(unit, 'JAVA_SRCS_VALUE', args_delim),
'JAVAC_FLAGS':
extract_macro_calls(unit, 'JAVAC_FLAGS_VALUE', args_delim),
'ANNOTATION_PROCESSOR':
extract_macro_calls(unit, 'ANNOTATION_PROCESSOR_VALUE', args_delim),
'EXTERNAL_JAR':
extract_macro_calls(unit, 'EXTERNAL_JAR_VALUE', args_delim),
'RUN_JAVA_PROGRAM':
extract_macro_calls2(unit, 'RUN_JAVA_PROGRAM_VALUE'),
'RUN_JAVA_PROGRAM_MANAGED':
'${RUN_JAVA_PROGRAM_MANAGED}',
'ADD_WAR':
extract_macro_calls(unit, 'ADD_WAR_VALUE', args_delim),
'DEPENDENCY_MANAGEMENT':
extract_macro_calls(unit, 'DEPENDENCY_MANAGEMENT_VALUE', args_delim),
'MAVEN_GROUP_ID':
extract_macro_calls(unit, 'MAVEN_GROUP_ID_VALUE', args_delim),
'JAR_INCLUDE_FILTER':
extract_macro_calls(unit, 'JAR_INCLUDE_FILTER_VALUE', args_delim),
'JAR_EXCLUDE_FILTER':
extract_macro_calls(unit, 'JAR_EXCLUDE_FILTER_VALUE', args_delim),
# TODO remove when java test dart is in prod
'UNITTEST_DIR':
unit.get('UNITTEST_DIR'),
'SYSTEM_PROPERTIES':
extract_macro_calls(unit, 'SYSTEM_PROPERTIES_VALUE', args_delim),
'JVM_ARGS':
extract_macro_calls(unit, 'JVM_ARGS_VALUE', args_delim),
'TEST_CWD':
extract_macro_calls(unit, 'TEST_CWD_VALUE', args_delim),
'TEST_DATA':
extract_macro_calls(unit, '__test_data', args_delim),
'TEST_FORK_MODE':
extract_macro_calls(unit, 'TEST_FORK_MODE', args_delim),
'SPLIT_FACTOR':
extract_macro_calls(unit, 'TEST_SPLIT_FACTOR', args_delim),
'TIMEOUT':
extract_macro_calls(unit, 'TEST_TIMEOUT', args_delim),
'TAG':
extract_macro_calls(unit, 'TEST_TAGS_VALUE', args_delim),
'SIZE':
extract_macro_calls(unit, 'TEST_SIZE_NAME', args_delim),
'DEPENDS':
extract_macro_calls(unit, 'TEST_DEPENDS_VALUE', args_delim),
'IDEA_EXCLUDE':
extract_macro_calls(unit, 'IDEA_EXCLUDE_DIRS_VALUE', args_delim),
'IDEA_RESOURCE':
extract_macro_calls(unit, 'IDEA_RESOURCE_DIRS_VALUE', args_delim),
'IDEA_MODULE_NAME':
extract_macro_calls(unit, 'IDEA_MODULE_NAME_VALUE', args_delim),
'GENERATE_SCRIPT':
extract_macro_calls2(unit, 'GENERATE_SCRIPT_VALUE'),
'FAKEID':
extract_macro_calls(unit, 'FAKEID', args_delim),
'TEST_DATA':
extract_macro_calls(unit, 'TEST_DATA_VALUE', args_delim),
'JAVA_FORBIDDEN_LIBRARIES':
extract_macro_calls(unit, 'JAVA_FORBIDDEN_LIBRARIES_VALUE',
args_delim),
}
if unit.get('SAVE_JAVAC_GENERATED_SRCS_DIR') and unit.get(
'SAVE_JAVAC_GENERATED_SRCS_TAR'):
data['SAVE_JAVAC_GENERATED_SRCS_DIR'] = extract_macro_calls(
unit, 'SAVE_JAVAC_GENERATED_SRCS_DIR', args_delim)
data['SAVE_JAVAC_GENERATED_SRCS_TAR'] = extract_macro_calls(
unit, 'SAVE_JAVAC_GENERATED_SRCS_TAR', args_delim)
if unit.get('JAVA_ADD_DLLS_VALUE') == 'yes':
data['ADD_DLLS_FROM_DEPENDS'] = extract_macro_calls(
unit, 'JAVA_ADD_DLLS_VALUE', args_delim)
if unit.get('ERROR_PRONE_VALUE') == 'yes':
data['ERROR_PRONE'] = extract_macro_calls(unit, 'ERROR_PRONE_VALUE',
args_delim)
if unit.get('WITH_KOTLIN_VALUE') == 'yes':
data['WITH_KOTLIN'] = extract_macro_calls(unit, 'WITH_KOTLIN_VALUE',
args_delim)
if unit.get('KOTLIN_JVM_TARGET'):
data['KOTLIN_JVM_TARGET'] = extract_macro_calls(
unit, 'KOTLIN_JVM_TARGET', args_delim)
if unit.get('KOTLINC_FLAGS_VALUE'):
data['KOTLINC_FLAGS'] = extract_macro_calls(
unit, 'KOTLINC_FLAGS_VALUE', args_delim)
if unit.get('WITH_GROOVY_VALUE') == 'yes':
if not common.strip_roots(unit.path()).startswith(
('devtools/dummy_arcadia', 'junk')):
ymake.report_configure_error('Groovy is not allowed here')
data['WITH_GROOVY'] = extract_macro_calls(unit, 'WITH_GROOVY_VALUE',
args_delim)
if unit.get('DIRECT_DEPS_ONLY_VALUE') == 'yes':
data['DIRECT_DEPS_ONLY'] = extract_macro_calls(
unit, 'DIRECT_DEPS_ONLY_VALUE', args_delim)
if unit.get('JAVA_EXTERNAL_DEPENDENCIES_VALUE'):
valid = []
for dep in sum(
extract_macro_calls(unit, 'JAVA_EXTERNAL_DEPENDENCIES_VALUE',
args_delim), []):
if os.path.normpath(dep).startswith('..'):
ymake.report_configure_error(
'{}: {} - relative paths in JAVA_EXTERNAL_DEPENDENCIES is not allowed'
.format(unit.path(), dep))
elif os.path.isabs(dep):
ymake.report_configure_error(
'{}: {} absolute paths in JAVA_EXTERNAL_DEPENDENCIES is not allowed'
.format(unit.path(), dep))
else:
valid.append(dep)
if valid:
data['EXTERNAL_DEPENDENCIES'] = [valid]
if unit.get('MAKE_UBERJAR_VALUE') == 'yes':
if unit.get('MODULE_TYPE') != 'JAVA_PROGRAM':
ymake.report_configure_error(
'{}: UBERJAR supported only for JAVA_PROGRAM module type'.
format(unit.path()))
data['UBERJAR'] = extract_macro_calls(unit, 'MAKE_UBERJAR_VALUE',
args_delim)
data['UBERJAR_PREFIX'] = extract_macro_calls(unit,
'UBERJAR_PREFIX_VALUE',
args_delim)
data['UBERJAR_HIDE_EXCLUDE'] = extract_macro_calls(
unit, 'UBERJAR_HIDE_EXCLUDE_VALUE', args_delim)
data['UBERJAR_PATH_EXCLUDE'] = extract_macro_calls(
unit, 'UBERJAR_PATH_EXCLUDE_VALUE', args_delim)
data['UBERJAR_MANIFEST_TRANSFORMER_MAIN'] = extract_macro_calls(
unit, 'UBERJAR_MANIFEST_TRANSFORMER_MAIN_VALUE', args_delim)
data['UBERJAR_MANIFEST_TRANSFORMER_ATTRIBUTE'] = extract_macro_calls(
unit, 'UBERJAR_MANIFEST_TRANSFORMER_ATTRIBUTE_VALUE', args_delim)
data['UBERJAR_APPENDING_TRANSFORMER'] = extract_macro_calls(
unit, 'UBERJAR_APPENDING_TRANSFORMER_VALUE', args_delim)
data['UBERJAR_SERVICES_RESOURCE_TRANSFORMER'] = extract_macro_calls(
unit, 'UBERJAR_SERVICES_RESOURCE_TRANSFORMER_VALUE', args_delim)
if unit.get('WITH_JDK_VALUE') == 'yes':
if unit.get('MODULE_TYPE') != 'JAVA_PROGRAM':
ymake.report_configure_error(
'{}: JDK export supported only for JAVA_PROGRAM module type'.
format(unit.path()))
data['WITH_JDK'] = extract_macro_calls(unit, 'WITH_JDK_VALUE',
args_delim)
if not data['EXTERNAL_JAR']:
has_processor = extract_macro_calls(unit,
'GENERATE_VCS_JAVA_INFO_NODEP',
args_delim)
data['EMBED_VCS'] = [[
str(has_processor and has_processor[0] and has_processor[0][0])
]]
# FORCE_VCS_INFO_UPDATE is responsible for setting special value of VCS_INFO_DISABLE_CACHE__NO_UID__
macro_val = extract_macro_calls(unit, 'FORCE_VCS_INFO_UPDATE',
args_delim)
macro_str = macro_val[0][
0] if macro_val and macro_val[0] and macro_val[0][0] else ''
if macro_str and macro_str == 'yes':
data['VCS_INFO_DISABLE_CACHE__NO_UID__'] = macro_val
for java_srcs_args in data['JAVA_SRCS']:
external = None
for i in xrange(len(java_srcs_args)):
arg = java_srcs_args[i]
if arg == 'EXTERNAL':
if not i + 1 < len(java_srcs_args):
continue # TODO configure error
ex = java_srcs_args[i + 1]
if ex in ('EXTERNAL', 'SRCDIR', 'PACKAGE_PREFIX', 'EXCLUDE'):
continue # TODO configure error
if external is not None:
continue # TODO configure error
external = ex
if external:
unit.onpeerdir(external)
dep_veto = extract_macro_calls(unit,
'JAVA_DEPENDENCIES_CONFIGURATION_VALUE',
args_delim)
if dep_veto:
dep_veto = set(dep_veto[0])
if (unit.get('IGNORE_JAVA_DEPENDENCIES_CONFIGURATION')
or '').lower() != 'yes':
for veto in map(str.upper, dep_veto):
if veto.upper() == 'FORBID_DIRECT_PEERDIRS':
data['JAVA_DEPENDENCY_DIRECT'] = [['yes']]
elif veto.upper() == 'FORBID_DEFAULT_VERSIONS':
data['JAVA_DEPENDENCY_DEFAULT_VERSION'] = [['yes']]
elif veto.upper() == 'FORBID_CONFLICT':
data['JAVA_DEPENDENCY_CHECK_RESOLVED_CONFLICTS'] = [[
'yes'
]]
elif veto.upper() == 'FORBID_CONFLICT_DM':
data['JAVA_DEPENDENCY_DM_CHECK_DIFFERENT'] = [['yes']]
elif veto.upper() == 'FORBID_CONFLICT_DM_RECENT':
data['JAVA_DEPENDENCY_DM_CHECK_RECENT'] = [['yes']]
elif veto.upper() == 'REQUIRE_DM':
data['JAVA_DEPENDENCY_DM_REQUIRED'] = [['yes']]
else:
ymake.report_configure_error(
'Unknown JAVA_DEPENDENCIES_CONFIGURATION value {} Allowed only [{}]'
.format(
veto, ', '.join([
'FORBID_DIRECT_PEERDIRS',
'FORBID_DEFAULT_VERSIONS', 'FORBID_CONFLICT',
'FORBID_CONFLICT_DM',
'FORBID_CONFLICT_DM_RECENT', 'REQUIRE_DM'
])))
for k, v in data.items():
if not v:
data.pop(k)
dart = 'JAVA_DART: ' + base64.b64encode(
json.dumps(data)) + '\n' + DELIM + '\n'
unit.set_property(['JAVA_DART_DATA', dart])
if unit.get('MODULE_TYPE') in (
'JAVA_PROGRAM', 'JAVA_LIBRARY', 'JTEST', 'TESTNG',
'JUNIT5') and not unit.path().startswith('$S/contrib/java'):
jdeps_val = (unit.get('CHECK_JAVA_DEPS_VALUE') or '').lower()
if jdeps_val and jdeps_val not in ('yes', 'no', 'strict'):
ymake.report_configure_error(
'CHECK_JAVA_DEPS: "yes", "no" or "strict" required')
if jdeps_val and jdeps_val != 'no':
unit.onjava_test_deps(jdeps_val)
if unit.get('LINT_LEVEL_VALUE') != "none":
unit.onadd_check(['JAVA_STYLE', unit.get('LINT_LEVEL_VALUE')])
def onjava_test(unit, *args):
if unit.get("TIDY") == "yes":
# graph changed for clang_tidy tests
return
assert unit.get('MODULE_TYPE') is not None
if unit.get('MODULE_TYPE') == 'JTEST_FOR':
if not unit.get('UNITTEST_DIR'):
ymake.report_configure_error('skip JTEST_FOR in {}: no args provided'.format(unit.path()))
return
java_cp_arg_type = unit.get('JAVA_CLASSPATH_CMD_TYPE_VALUE') or 'MANIFEST'
if java_cp_arg_type not in ('MANIFEST', 'COMMAND_FILE', 'LIST'):
ymake.report_configure_error('{}: TEST_JAVA_CLASSPATH_CMD_TYPE({}) are invalid. Choose argument from MANIFEST, COMMAND_FILE or LIST)'.format(unit.path(), java_cp_arg_type))
return
unit_path = unit.path()
path = _common.strip_roots(unit_path)
test_data = get_norm_paths(unit, 'TEST_DATA_VALUE')
test_data.append('arcadia/build/scripts/run_junit.py')
test_data.append('arcadia/build/scripts/unpacking_jtest_runner.py')
data, data_files = get_canonical_test_resources(unit)
test_data += data
props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE'))
if error_mgs:
ymake.report_configure_error(error_mgs)
return
for prop in props:
if prop['type'] == 'file':
test_data.append(prop['path'].replace('${ARCADIA_ROOT}', 'arcadia'))
props = base64.b64encode(json.dumps(props, encoding='utf-8'))
test_cwd = unit.get('TEST_CWD_VALUE') or '' # TODO: validate test_cwd value
if unit.get('MODULE_TYPE') == 'JUNIT5':
script_rel_path = 'junit5.test'
else:
script_rel_path = 'junit.test'
ymake_java_test = unit.get('YMAKE_JAVA_TEST') == 'yes'
test_record = {
'SOURCE-FOLDER-PATH': path,
'TEST-NAME': '-'.join([os.path.basename(os.path.dirname(path)), os.path.basename(path)]),
'SCRIPT-REL-PATH': script_rel_path,
'TEST-TIMEOUT': unit.get('TEST_TIMEOUT') or '',
'TESTED-PROJECT-NAME': path,
'TEST-ENV': prepare_env(unit.get("TEST_ENV_VALUE")),
# 'TEST-PRESERVE-ENV': 'da',
'TEST-DATA': serialize_list(sorted(_common.filter_out_by_keyword(test_data, 'AUTOUPDATED'))),
'FORK-MODE': unit.get('TEST_FORK_MODE') or '',
'SPLIT-FACTOR': unit.get('TEST_SPLIT_FACTOR') or '',
'CUSTOM-DEPENDENCIES': ' '.join(get_values_list(unit, 'TEST_DEPENDS_VALUE')),
'TAG': serialize_list(_get_test_tags(unit)),
'SIZE': unit.get('TEST_SIZE_NAME') or '',
'REQUIREMENTS': serialize_list(get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')),
'TEST-RECIPES': prepare_recipes(unit.get("TEST_RECIPES_VALUE")),
# JTEST/JTEST_FOR only
'MODULE_TYPE': unit.get('MODULE_TYPE'),
'UNITTEST_DIR': unit.get('UNITTEST_DIR') or '',
'JVM_ARGS': serialize_list(get_values_list(unit, 'JVM_ARGS_VALUE')),
'SYSTEM_PROPERTIES': props,
'TEST-CWD': test_cwd,
'SKIP_TEST': unit.get('SKIP_TEST_VALUE') or '',
'JAVA_CLASSPATH_CMD_TYPE': java_cp_arg_type,
'NO_JBUILD': 'yes' if ymake_java_test else 'no',
'JDK_RESOURCE': 'JDK' + (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION') or '_DEFAULT'),
'JDK_FOR_TESTS': 'JDK' + (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION') or '_DEFAULT') + '_FOR_TESTS',
'YT-SPEC': serialize_list(get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')),
}
test_classpath_origins = unit.get('TEST_CLASSPATH_VALUE')
if test_classpath_origins:
test_record['TEST_CLASSPATH_ORIGINS'] = test_classpath_origins
test_record['TEST_CLASSPATH'] = '${TEST_CLASSPATH_MANAGED}'
elif ymake_java_test:
test_record['TEST_CLASSPATH'] = '${DART_CLASSPATH}'
test_record['TEST_CLASSPATH_DEPS'] = '${DART_CLASSPATH_DEPS}'
if unit.get('UNITTEST_DIR'):
test_record['TEST_JAR'] = '${UNITTEST_MOD}'
else:
test_record['TEST_JAR'] = '{}/{}.jar'.format(unit.get('MODDIR'), unit.get('REALPRJNAME'))
data = dump_test(unit, test_record)
if data:
unit.set_property(['DART_DATA', data])
def onadd_ytest(unit, *args):
keywords = {
"DEPENDS": -1,
"DATA": -1,
"TIMEOUT": 1,
"FORK_MODE": 1,
"SPLIT_FACTOR": 1,
"FORK_SUBTESTS": 0,
"FORK_TESTS": 0
}
flat_args, spec_args = _common.sort_by_keywords(keywords, args)
if flat_args[1] == "fuzz.test":
unit.ondata("arcadia/fuzzing/{}/corpus.json".format(
_common.strip_roots(unit.path())))
elif flat_args[
1] == "coverage.extractor" and not match_coverage_extractor_requirements(
unit):
# XXX
# Current ymake implementation doesn't allow to call macro inside the 'when' body
# that's why we add ADD_YTEST(coverage.extractor) to every PROGRAM entry and check requirements later
return
elif flat_args[1] == "no.test":
return
fork_mode = []
if 'FORK_SUBTESTS' in spec_args:
fork_mode.append('subtests')
if 'FORK_TESTS' in spec_args:
fork_mode.append('tests')
fork_mode = fork_mode or spec_args.get(
'FORK_MODE', []) or unit.get('TEST_FORK_MODE').split()
fork_mode = ' '.join(fork_mode) if fork_mode else ''
test_record = {
'TEST-NAME':
flat_args[0],
'SCRIPT-REL-PATH':
flat_args[1],
'TESTED-PROJECT-NAME':
unit.name(),
'TESTED-PROJECT-FILENAME':
unit.filename(),
'SOURCE-FOLDER-PATH':
unit.resolve(unit.path()),
'BUILD-FOLDER-PATH':
_common.strip_roots(unit.path()),
'BINARY-PATH':
_common.strip_roots(os.path.join(unit.path(), unit.filename())),
'CUSTOM-DEPENDENCIES':
' '.join(
spec_args.get('DEPENDS', []) +
get_values_list(unit, 'TEST_DEPENDS_VALUE')),
'TEST-RECIPES':
prepare_recipes(unit.get("TEST_RECIPES_VALUE")),
'TEST-ENV':
prepare_env(unit.get("TEST_ENV_VALUE")),
# 'TEST-PRESERVE-ENV': 'da',
'TEST-DATA':
serialize_list(
sorted(
_common.filter_out_by_keyword(
spec_args.get('DATA', []) +
get_values_list(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED'))),
'TEST-TIMEOUT':
''.join(spec_args.get('TIMEOUT', [])) or unit.get('TEST_TIMEOUT')
or '',
'FORK-MODE':
fork_mode,
'SPLIT-FACTOR':
''.join(spec_args.get('SPLIT_FACTOR', []))
or unit.get('TEST_SPLIT_FACTOR') or '',
'SIZE':
''.join(spec_args.get('SIZE', [])) or unit.get('TEST_SIZE_NAME') or '',
'TAG':
serialize_list(
spec_args.get('TAG', []) +
get_values_list(unit, 'TEST_TAGS_VALUE')),
'REQUIREMENTS':
serialize_list(
spec_args.get('REQUIREMENTS', []) +
get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')),
'TEST-CWD':
unit.get('TEST_CWD_VALUE') or '',
'FUZZ-DICTS':
serialize_list(
spec_args.get('FUZZ_DICTS', []) +
get_unit_list_variable(unit, 'FUZZ_DICTS_VALUE')),
'FUZZ-OPTS':
serialize_list(
spec_args.get('FUZZ_OPTS', []) +
get_unit_list_variable(unit, 'FUZZ_OPTS_VALUE')),
'YT-SPEC':
serialize_list(
spec_args.get('YT_SPEC', []) +
get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')),
'BLOB':
unit.get('TEST_BLOB_DATA') or '',
'SKIP_TEST':
unit.get('SKIP_TEST_VALUE') or '',
'TEST_IOS_DEVICE_TYPE':
unit.get('TEST_IOS_DEVICE_TYPE_VALUE') or '',
'TEST_IOS_RUNTIME_TYPE':
unit.get('TEST_IOS_RUNTIME_TYPE_VALUE') or '',
'ANDROID_APK_TEST_ACTIVITY':
unit.get('ANDROID_APK_TEST_ACTIVITY_VALUE') or '',
'TEST_PARTITION':
unit.get("TEST_PARTITION") or 'SEQUENTIAL',
}
if flat_args[1] == 'fuzz.test' and unit.get('FUZZING') == 'yes':
test_record['FUZZING'] = '1'
# use all cores if fuzzing requested
test_record['REQUIREMENTS'] = serialize_list(
filter(
None,
deserialize_list(test_record['REQUIREMENTS']) +
["cpu:all", "ram:all"]))
data = dump_test(unit, test_record)
if data:
unit.set_property(["DART_DATA", data])
save_in_file(unit.get('TEST_DART_OUT_FILE'), data)