def _show_tests_summary(self, passed_run_results, failed_run_results):
"""Show tests summary. """
self._show_banner('Testing Summary')
console.info('%d tests scheduled to run by scheduler.' % (len(self.test_jobs)))
if self.skipped_tests:
console.info('%d tests skipped when doing incremental test.' %
len(self.skipped_tests))
console.info('You can specify --full-test to run all tests.')
run_tests = len(passed_run_results) + len(failed_run_results)
if len(passed_run_results) == len(self.test_jobs):
console.notice('All %d tests passed!' % len(passed_run_results))
return
msg = ['total %d tests' % len(self.test_jobs)]
if passed_run_results:
msg.append('%d passed' % len(passed_run_results))
if failed_run_results:
msg.append('%d failed' % len(failed_run_results))
cancelled_tests = len(self.test_jobs) - run_tests
if cancelled_tests:
msg.append('%d cancelled' % cancelled_tests)
console.error(', '.join(msg) + '.')
def update_config(self, section_name, append, user_config):
"""update config section by name. """
section = self.configs.get(section_name)
if section:
if append:
self._append_config(section_name, section, append)
self._replace_config(section_name, section, user_config)
else:
console.error('%s: %s: unknown config section name' % (
self.current_file_name, section_name))
def build(self):
"""Implement the "build" subcommand."""
console.info('Building...')
console.flush()
returncode = ninja_runner.build(self.get_build_dir(),
self.build_script(),
self.build_jobs_num(), self.__options)
if returncode == 0 and not self.verify():
returncode = 1
if returncode != 0:
console.error('Build failure.')
else:
console.info('Build success.')
return returncode
def _show_tests_result(self, passed_run_results, failed_run_results):
"""Show test details and summary according to the options. """
if self.options.show_details:
self._show_banner('Testing Details')
self._show_skipped_tests()
if passed_run_results:
console.info('passed tests:')
self._show_run_results(passed_run_results)
if self.options.show_tests_slower_than is not None:
self._show_slow_tests(passed_run_results, failed_run_results)
if failed_run_results: # Always show details of failed tests
console.error('failed tests:')
self._show_run_results(failed_run_results, is_error=True)
self._show_tests_summary(passed_run_results, failed_run_results)
def try_parse_file(self, filename):
"""load the configuration file and parse."""
try:
self.current_file_name = filename
if os.path.exists(filename):
console.info('Loading config file "%s"' % filename)
with open(filename, 'rb') as f:
content = f.read()
self.__md5.update(content)
exec_file_content(filename, content, _config_globals, None)
except SystemExit:
console.error('Parse error in config file %s' % filename)
finally:
self.current_file_name = ''
def _show_tests_result(self, passed_run_results, failed_run_results):
"""Show test details and summary according to the options. """
if self.options.show_details:
self._show_banner('Testing Details')
self._show_skipped_tests()
if passed_run_results:
console.info('passed tests:')
self._show_run_results(passed_run_results)
if self.options.show_tests_slower_than is not None:
self._show_slow_tests(passed_run_results, failed_run_results)
if failed_run_results: # Always show details of failed tests
console.error('failed tests:')
self._show_run_results(failed_run_results, is_error=True)
self._show_tests_summary(passed_run_results, failed_run_results)
def _check_thrift_srcs_name(self, srcs):
"""Checks whether the thrift file's name ends with 'thrift'. """
error = 0
for src in srcs:
base_name = os.path.basename(src)
pos = base_name.rfind('.')
if pos == -1:
console.error('invalid thrift file name %s' % src)
error += 1
file_suffix = base_name[pos + 1:]
if file_suffix != 'thrift':
console.error('invalid thrift file name %s' % src)
error += 1
if error > 0:
console.error_exit('invalid thrift file names found.')
def _check_thrift_srcs_name(self, srcs):
"""Checks whether the thrift file's name ends with 'thrift'. """
error = 0
for src in srcs:
base_name = os.path.basename(src)
pos = base_name.rfind('.')
if pos == -1:
console.error('invalid thrift file name %s' % src)
error += 1
file_suffix = base_name[pos + 1:]
if file_suffix != 'thrift':
console.error('invalid thrift file name %s' % src)
error += 1
if error > 0:
console.error_exit('invalid thrift file names found.')
def main():
name = sys.argv[1]
exit_code = 0
try:
options, args = util.parse_command_line(sys.argv[2:])
exit_code = _BUILTIN_TOOLS[name](args=args, **options)
if not exit_code:
_verify_outputs()
except Exception as e: # pylint: disable=broad-except
console.error('Blade build tool %s error: %s %s' %
(name, str(e), traceback.format_exc()))
exit_code = 1
finally:
if exit_code:
_cleanup_outputs()
sys.exit(exit_code or 0)
def build(options):
_check_code_style(_TARGETS)
console.info('building...')
console.flush()
if config.get_item('global_config', 'native_builder') == 'ninja':
returncode = _ninja_build(options)
else:
returncode = _scons_build(options)
if returncode != 0:
console.error('building failure.')
return returncode
if not build_manager.instance.verify():
console.error('building failure.')
return 1
console.info('building done.')
return 0
def _append_config(self, section_name, section, append):
"""Append config section items"""
if not isinstance(append, dict):
console.error('%s: %s: Append must be a dict' %
(self.current_file_name, section_name))
for k in append:
if k in section:
if isinstance(section[k], list):
section[k] += var_to_list(append[k])
else:
console.warning('%s: %s: Config item %s is not a list' %
(self.current_file_name, section_name, k))
else:
console.warning('%s: %s: Unknown config item name: %s' %
(self.current_file_name, section_name, k))
def build(options):
_check_code_style(_TARGETS)
console.info('building...')
console.flush()
if config.get_item('global_config', 'native_builder') == 'ninja':
returncode = _ninja_build(options)
else:
returncode = _scons_build(options)
if returncode != 0:
console.error('building failure.')
return returncode
if not build_manager.instance.verify():
console.error('building failure.')
return 1
console.info('building done.')
return 0
def _append_config(self, section_name, section, append):
"""Append config section items"""
if not isinstance(append, dict):
console.error('%s: %s: append must be a dict' %
(self.current_file_name, section_name))
else:
for k in append:
if k in section:
if isinstance(section[k], list):
section[k] += var_to_list(append[k])
else:
console.warning('%s: %s: config item %s is not a list' %
(self.current_file_name, section_name, k))
else:
console.warning('%s: %s: unknown config item name: %s' %
(self.current_file_name, section_name, k))
def build(self):
"""Implement the "build" subcommand."""
console.info('Building...')
console.flush()
start_time = time.time()
returncode = ninja_runner.build(
self.get_build_dir(),
self.build_script(),
self.build_jobs_num(),
targets='', # FIXME: because not all targets has a targets
options=self.__options)
self._write_build_stamp_fime(start_time, returncode)
if returncode != 0:
console.error('Build failure.')
else:
console.info('Build success.')
return returncode
def _show_tests_result(self, passed_run_results, failed_run_results):
"""Show test details and summary according to the options."""
if self.options.show_details:
self._show_banner('Testing Details')
self._show_tests_list(self.unchanged_tests, 'unchanged')
if passed_run_results:
console.info('Passed tests:')
self._show_run_results(passed_run_results)
if self.options.show_tests_slower_than is not None:
self._show_slow_tests(passed_run_results, failed_run_results)
if failed_run_results: # Always show details of failed tests
console.error('Failed tests:')
self._show_run_results(failed_run_results, is_error=True)
self._show_tests_list(self.repaired_tests, 'repaired')
self._show_tests_list(self.new_failed_tests, 'new failed', 'error')
self._show_unrepaired_results()
self._show_tests_summary(passed_run_results, failed_run_results)
def check_job_timeout(self, now):
"""Check whether the job is timeout or not.
This method simply checks job timeout and returns immediately.
The caller should invoke this method repeatedly so that a job
which takes a very long time would be timeout sooner or later.
"""
try:
self.job_lock.acquire()
if (not self.job_is_timeout and self.job_start_time and
self.job_timeout is not None and
self.job_name and self.job_process is not None):
if self.job_start_time + self.job_timeout < now:
self.job_is_timeout = True
console.error('//%s: TIMEOUT\n' % self.job_name)
self.job_process.terminate()
finally:
self.job_lock.release()
def __init__(self, path):
self.path = path
if not os.path.isfile(self.path):
console.error('"%s" is not a valid file.' % self.path)
self.thrift_name = os.path.basename(self.path)[:-7]
# Package name for each language.
self.package_name = {}
# Set to true if there is at least one const definition in the
# thrift file.
self.has_constants = False
self.enums = []
self.structs = []
self.exceptions = []
self.services = []
# Parse the thrift IDL file.
self._parse_file()
def normalize(target, working_dir):
"""Normalize target from command line form into canonical form.
Target canonical form: dir:name
dir: relative to blade_root_dir, use '.' for blade_root_dir
name: name if target is dir:name
'*' if target is dir
'...' if target is dir/...
"""
if target.startswith('//'):
target = target[2:]
elif target.startswith('/'):
console.error('Invalid target "%s" starting from root path.' % target)
target = target[1:] # Try correct to keep going
else: # Relative path
if working_dir != '.':
target = os.path.join(working_dir, target)
path, name = _split(target)
return '%s:%s' % (path, name)
def run(self):
"""Run all the test target programs. """
self._collect_test_jobs()
tests_run_list = []
for target_key in self.test_jobs:
target = self.target_database[target_key]
test_env = self._prepare_env(target)
cmd = [os.path.abspath(self._executable(target))]
cmd += self.options.args
if console.color_enabled():
test_env['GTEST_COLOR'] = 'yes'
else:
test_env['GTEST_COLOR'] = 'no'
test_env['GTEST_OUTPUT'] = 'xml'
test_env['HEAPCHECK'] = target.data.get('heap_check', '')
pprof_path = config.get_item('cc_test_config', 'pprof_path')
if pprof_path:
test_env['PPROF_PATH'] = os.path.abspath(pprof_path)
if self.options.coverage:
test_env['BLADE_COVERAGE'] = 'true'
tests_run_list.append(
(target, self._runfiles_dir(target), test_env, cmd))
console.notice('%d tests to run' % len(tests_run_list))
console.flush()
scheduler = TestScheduler(tests_run_list, self.options.test_jobs)
try:
scheduler.schedule_jobs()
except KeyboardInterrupt:
console.clear_progress_bar()
console.error('KeyboardInterrupt, all tests stopped')
console.flush()
if self.options.coverage:
self._generate_coverage_report()
self._clean_env()
passed_run_results, failed_run_results = scheduler.get_results()
self._save_test_history(passed_run_results, failed_run_results)
self._show_tests_result(passed_run_results, failed_run_results)
return 0 if len(passed_run_results) == len(self.test_jobs) else 1
def run(self):
"""Run all the test target programs. """
self._collect_test_jobs()
tests_run_list = []
for target_key in self.test_jobs:
target = self.target_database[target_key]
test_env = self._prepare_env(target)
cmd = [os.path.abspath(self._executable(target))]
cmd += self.options.args
if console.color_enabled():
test_env['GTEST_COLOR'] = 'yes'
else:
test_env['GTEST_COLOR'] = 'no'
test_env['GTEST_OUTPUT'] = 'xml'
test_env['HEAPCHECK'] = target.data.get('heap_check', '')
pprof_path = config.get_item('cc_test_config', 'pprof_path')
if pprof_path:
test_env['PPROF_PATH'] = os.path.abspath(pprof_path)
if self.options.coverage:
test_env['BLADE_COVERAGE'] = 'true'
tests_run_list.append((target, self._runfiles_dir(target), test_env, cmd))
console.notice('%d tests to run' % len(tests_run_list))
sys.stdout.flush()
scheduler = TestScheduler(tests_run_list, self.options.test_jobs)
try:
scheduler.schedule_jobs()
except KeyboardInterrupt:
console.clear_progress_bar()
console.error('KeyboardInterrupt, all tests stopped')
console.flush()
if self.options.coverage:
self._generate_coverage_report()
self._clean_env()
passed_run_results, failed_run_results = scheduler.get_results()
self._save_test_history(passed_run_results, failed_run_results)
self._show_tests_result(passed_run_results, failed_run_results)
return 0 if len(passed_run_results) == len(self.test_jobs) else 1
def _parse_file(self):
for line in open(self.path):
line = line.strip()
if line.startswith('//') or line.startswith('#'):
continue
pos = line.find('//')
if pos != -1:
line = line[:pos]
pos = line.find('#')
if pos != -1:
line = line[:pos]
matched = re.match('^namespace ([0-9_a-zA-Z]+) ([0-9_a-zA-Z.]+)',
line)
if matched:
lang, package = matched.groups()
self.package_name[lang] = package
continue
matched = re.match(
'(const|struct|service|enum|exception) ([0-9_a-zA-Z]+)', line)
if not matched:
continue
kw, name = matched.groups()
if kw == 'const':
self.has_constants = True
elif kw == 'struct':
self.structs.append(name)
elif kw == 'service':
self.services.append(name)
elif kw == 'enum':
self.enums.append(name)
elif kw == 'exception':
self.exceptions.append(name)
if self.has_constants or self.structs or self.enums or \
self.exceptions or self.services:
pass
else:
console.error('%s is an empty thrift file.' % self.path)
def __init__(self, name, path, code_generation):
self.name = name
self.path = path
assert isinstance(code_generation, dict)
self.code_generation = {}
for language, v in iteritems(code_generation):
if language not in self.__languages:
console.error(
'%s: Language %s is invalid. '
'Protoc plugins in %s are supported by blade currently.' %
(name, language, ', '.join(self.__languages)))
continue
self.code_generation[language] = {}
# Note that each plugin dep should be in the global target format
# since protoc plugin is defined in the global scope
deps = []
for dep in var_to_list(v['deps']):
if dep.startswith('//'):
dep = dep[2:]
if dep not in deps:
deps.append(dep)
self.code_generation[language]['deps'] = deps
def go_package(name,
deps=[],
testdata=[],
visibility=None,
extra_goflags=None):
path = build_manager.instance.get_current_source_path()
srcs, tests = find_go_srcs(path)
if not srcs and not tests:
console.error('Empty go sources in %s' % path)
return
if srcs:
main = False
for src in srcs:
package = extract_go_package(os.path.join(path, src))
if package == 'main':
main = True
break
if main:
go_binary(name=name,
srcs=srcs,
deps=deps,
visibility=visibility,
extra_goflags=extra_goflags)
else:
go_library(name=name,
srcs=srcs,
deps=deps,
visibility=visibility,
extra_goflags=extra_goflags)
if tests:
go_test(name='%s_test' % name,
srcs=tests,
deps=deps,
visibility=visibility,
testdata=testdata,
extra_goflags=extra_goflags)
def _show_tests_summary(self, passed_run_results, failed_run_results):
"""Show tests summary. """
self._show_banner('Testing Summary')
console.info('%d tests scheduled to run by scheduler.' % (len(self.test_jobs)))
if self.skipped_tests:
console.info('%d tests skipped when doing incremental test.' %
len(self.skipped_tests))
console.info('You can specify --full-test to run all tests.')
run_tests = len(passed_run_results) + len(failed_run_results)
if len(passed_run_results) == len(self.test_jobs):
console.notice('All %d tests passed!' % len(passed_run_results))
return
msg = ['total %d tests' % len(self.test_jobs)]
if passed_run_results:
msg.append('%d passed' % len(passed_run_results))
if failed_run_results:
msg.append('%d failed' % len(failed_run_results))
cancelled_tests = len(self.test_jobs) - run_tests
if cancelled_tests:
msg.append('%d cancelled' % cancelled_tests)
console.error(', '.join(msg) + '.')
def _check_code_style(targets):
cpplint = config.get_item('cc_config', 'cpplint')
if not cpplint:
console.info('cpplint disabled')
return 0
changed_files = _get_changed_files(targets, _BLADE_ROOT_DIR, _WORKING_DIR)
if not changed_files:
return 0
console.info('Begin to check code style for changed source code')
p = subprocess.Popen(('%s %s' % (cpplint, ' '.join(changed_files))), shell=True)
try:
p.wait()
if p.returncode != 0:
if p.returncode == 127:
msg = ("Can't execute '{0}' to check style, you can config the "
"'cpplint' option to be a valid cpplint path in the "
"'cc_config' section of blade.conf or BLADE_ROOT, or "
"make sure '{0}' command is correct.").format(cpplint)
else:
msg = 'Please fixing style warnings before submitting the code!'
console.warning(msg)
except KeyboardInterrupt, e:
console.error(str(e))
return 1
def main(blade_path, argv):
exit_code = 0
try:
start_time = time.time()
exit_code = _main(blade_path, argv)
cost_time = time.time() - start_time
console.info('Cost time %s' % format_timedelta(cost_time))
except SystemExit as e:
# pylint misreport e.code as classobj
exit_code = e.code
except KeyboardInterrupt:
console.error('KeyboardInterrupt')
exit_code = -signal.SIGINT
except: # pylint: disable=bare-except
exit_code = 1
console.error(traceback.format_exc())
if exit_code != 0:
console.error('Failure')
return exit_code
def _show_unrepaired_results(self):
"""Show the unrepaired tests"""
if not self.unrepaired_tests:
return
items = self.test_history['items']
console.error('Skipped %d still unrepaired tests:' %
len(self.unrepaired_tests))
for key in self.unrepaired_tests:
test = items[key]
first_fail_time = time.strftime(
'%F %T %A', time.localtime(test.first_fail_time))
duration = datetime.timedelta(seconds=int(time.time() -
test.first_fail_time))
console.error(
' %s: exit(%s), retry %s times, since %s, duration %s' %
(key, test.result.exit_code, test.fail_count, first_fail_time,
duration),
prefix=False)
console.error('You can specify --run-unrepaired-tests to run them',
prefix=False)
def main(blade_path):
exit_code = 0
try:
start_time = time.time()
exit_code = _main(blade_path)
cost_time = int(time.time() - start_time)
if cost_time > 1:
console.info('cost time %s' % format_timedelta(cost_time))
except SystemExit as e:
# pylint misreport e.code as classobj
exit_code = e.code # pylint: disable=redefined-variable-type
except KeyboardInterrupt:
console.error('keyboard interrupted', -signal.SIGINT)
exit_code = -signal.SIGINT
except: # pylint: disable=bare-except
exit_code = 1
console.error(traceback.format_exc())
if exit_code != 0:
console.error('failure')
sys.exit(exit_code)
def main(blade_path):
exit_code = 0
try:
start_time = time.time()
exit_code = _main(blade_path)
cost_time = int(time.time() - start_time)
if cost_time > 1:
console.info('cost time %s' % format_timedelta(cost_time))
except SystemExit as e:
# pylint misreport e.code as classobj
exit_code = e.code # pylint: disable=redefined-variable-type
except KeyboardInterrupt:
console.error('keyboard interrupted', -signal.SIGINT)
exit_code = -signal.SIGINT
except: # pylint: disable=bare-except
exit_code = 1
console.error(traceback.format_exc())
if exit_code != 0:
console.error('failure')
sys.exit(exit_code)
toolchains = {
'scm': generate_scm,
'package': generate_package,
'securecc_object': generate_securecc_object,
'resource_index': generate_resource_index,
'java_jar': generate_java_jar,
'java_resource': generate_java_resource,
'java_test': generate_java_test,
'java_fatjar': generate_fat_jar,
'java_onejar': generate_one_jar,
'java_binary': generate_java_binary,
'scala_test': generate_scala_test,
'shell_test': generate_shell_test,
'shell_testdata': generate_shell_testdata,
'python_library': generate_python_library,
'python_binary': generate_python_binary,
}
if __name__ == '__main__':
name = sys.argv[1]
try:
options, args = parse_command_line(sys.argv[2:])
ret = toolchains[name](args=args, **options)
except Exception as e: # pylint: disable=broad-except
ret = 1
console.error('Blade build tool %s error: %s %s' %
(name, str(e), traceback.format_exc()))
if ret:
sys.exit(ret)
generate_python_binary(args[0], args[1], args[2], args[3:])
toolchains = {
'scm': generate_scm_entry,
'package': generate_package_entry,
'securecc_object': generate_securecc_object_entry,
'resource_index': generate_resource_index_entry,
'java_jar': generate_java_jar_entry,
'java_resource': generate_java_resource_entry,
'java_test': generate_java_test_entry,
'java_fatjar': generate_fat_jar_entry,
'java_onejar': generate_one_jar_entry,
'java_binary': generate_java_binary_entry,
'scala_test': generate_scala_test_entry,
'shell_test': generate_shell_test_entry,
'shell_testdata': generate_shell_testdata_entry,
'python_library': generate_python_library_entry,
'python_binary': generate_python_binary_entry,
}
if __name__ == '__main__':
name = sys.argv[1]
try:
ret = toolchains[name](sys.argv[2:])
except Exception as e: # pylint: disable=broad-except
ret = 1
console.error(str(e))
if ret:
sys.exit(ret)
def error(self, msg):
console.error('%s error: %s' % (source_location(self.current_file_name), msg), prefix=False)
def glob(include, exclude=None, excludes=None, allow_empty=False):
"""This function can be called in BUILD to specify a set of files using patterns.
Args:
include:List(str), file patterns to be matched.
exclude:List[str], file patterns to be removed from the result.
allow_empty:bool: Whether a empty result is a error.
Patterns may contain shell-like wildcards, such as * , ? , or [charset].
Additionally, the path element '**' matches any subpath.
"""
from blade import build_manager
source_dir = Path(build_manager.instance.get_current_source_path())
include = var_to_list(include)
if excludes:
console.warning(
'//%s: "glob.excludes" is deprecated, use "exclude" instead' %
source_dir)
exclude = var_to_list(exclude) + var_to_list(excludes)
def includes_iterator():
results = []
for pattern in include:
for path in source_dir.glob(pattern):
if path.is_file() and not path.name.startswith('.'):
results.append(path.relative_to(source_dir))
return results
def is_special(pattern):
return '*' in pattern or '?' in pattern or '[' in pattern
non_special_excludes = set()
match_excludes = set()
for pattern in exclude:
if is_special(pattern):
match_excludes.add(pattern)
else:
non_special_excludes.add(pattern)
def exclusion(path):
if str(path) in non_special_excludes:
return True
for pattern in match_excludes:
ret = path.match(pattern)
if ret:
return True
return False
result = sorted(
set([str(p) for p in includes_iterator() if not exclusion(p)]))
if not result and not allow_empty:
args = repr(include)
if exclude:
args += ', exclude=%s' % repr(exclude)
console.error(
'//%s: "glob(%s)" got an empty result. If it is the expected behavior, '
'specify "allow_empty=True" to eliminate this error' %
(source_dir, args))
return result
def error(self, msg):
"""Print message with target full name prefix"""
console.error(self._format_message('error', msg), prefix=False)
def _check_error_log(stage):
error_count = console.error_count()
if error_count > 0:
console.error('There are %s errors in the %s stage' %(error_count, stage))
return 1
return 0
def error(symbol):
console.error('%s error: "%s" is not defined in "%s"' %
(src_loc, symbol, name),
prefix=False)
def error(self, msg):
"""Print message with target full name prefix"""
console.error('%s: %s' % (self.source_location, msg), prefix=False)
def error(self, msg):
"""Print message with target full name prefix"""
console.error('//%s: %s' % (self.fullname, msg))
def ninja_rules(self):
console.error('FIXME: fbthrift is still not supported by the ninja backend.')