def set_defaults(docs):
cfg = config.load_config()
defaults = {
'--region' : config.get_item(cfg, ('aws', 'configuration', 'region')),
'--user' : config.get_item(cfg, ('ssh', 'user')),
#'--password' : config.get_item(cfg, ('ssh', 'password')),
'--key' : config.get_item(cfg, ('ssh', 'key_filename')),
'--ait' : config.get_item(cfg, ('aws', 'configuration', 'instance_type')),
'--ami' : config.get_item(cfg, ('aws', 'configuration', 'machine_image')),
'--asg' : ','.join(config.get_item(cfg, ('aws', 'configuration', 'security_groups'))),
'--akp' : config.get_item(cfg, ('aws', 'configuration', 'key_pair')),
'--command' : config.get_item(cfg, ('command', 'command')),
}
for k, v in defaults.iteritems():
if v:
docs = re.sub(r'^(\s*{}=[A-Z]+\s+.+)\s*$'.format(k),
r'\1 [default: {}]'.format(v),
docs, flags=re.MULTILINE)
argv = docopt.docopt(docs)
# split lists
for k in ['--hosts', '--asg']:
if argv.has_key(k):
if argv[k]:
argv[k] = argv[k].split(',')
return argv
def __init__(self,
name,
target_type,
srcs,
deps,
visibility,
blade,
kwargs):
"""Init method.
Init the target.
"""
self.blade = blade
self.build_path = self.blade.get_build_path()
current_source_path = self.blade.get_current_source_path()
self.target_database = self.blade.get_target_database()
self.key = (current_source_path, name)
self.fullname = '%s:%s' % self.key
self.name = name
self.path = current_source_path
self.type = target_type
self.srcs = srcs
self.deps = []
self.expanded_deps = []
self.visibility = 'PUBLIC'
self.env_name = None
self.data = {}
self.data['test_timeout'] = config.get_item('global_config', 'test_timeout')
# Keep track of scons variables defined by scons rules
# generated by this target. Note that one blade target
# may correspond to several scons variables:
# proto_library: static lib/shared lib/jar variables
self.data['targets'] = {}
self.data['default_target'] = ''
self._check_name()
self._check_kwargs(kwargs)
self._check_srcs()
self._check_deps(deps)
self._init_target_deps(deps)
self._init_visibility(visibility)
self.build_rules = []
self.data['generated_hdrs'] = []
def _check_srcs(self):
"""Check source files.
"""
dups = []
srcset = set()
for s in self.srcs:
if s in srcset:
dups.append(s)
else:
srcset.add(s)
if dups:
console.error_exit('%s Duplicate source file paths: %s ' % (
self.fullname, dups))
# Check if one file belongs to two different targets.
action = config.get_item('global_config', 'duplicated_source_action')
for s in self.srcs:
if '..' in s or s.startswith('/'):
console.error_exit('%s Invalid source file path: %s. '
'can only be relative path, and must in current directory '
'or subdirectories.' % (self.fullname, s))
src = os.path.normpath(os.path.join(self.path, s))
target = self.fullname, self._allow_duplicate_source()
if src not in Target.__src_target_map:
Target.__src_target_map[src] = target
else:
target_existed = Target.__src_target_map[src]
if target_existed != target:
# Always preserve the target which disallows
# duplicate source files in the map
if target_existed[1]:
Target.__src_target_map[src] = target
elif target[1]:
pass
else:
message = 'Source file %s belongs to {%s, %s}' % (
s, target_existed[0], target[0])
if action == 'error':
console.error_exit(message)
elif action == 'warning':
console.warning(message)
def _wait_worker_threads(self, threads):
"""Wait for worker threads to complete. """
test_timeout = config.get_item('global_config', 'test_timeout')
try:
while threads:
time.sleep(1) # Check every second
now = time.time()
dead_threads = []
for t in threads:
if t.isAlive():
if test_timeout is not None:
t.check_job_timeout(now)
else:
dead_threads.append(t)
for dt in dead_threads:
threads.remove(dt)
except KeyboardInterrupt:
console.debug('KeyboardInterrupt: Terminate workers...')
for t in threads:
t.terminate()
raise
def _generate_scala_builders(self):
self._add_rule('scons_helper.setup_scala_builders(top_env, "%s")' %
config.get_item('scala_config', 'scala_home'))
def _generate_java_builders(self):
self._add_rule(
'scons_helper.setup_java_builders(top_env, "%s", "%s")' %
(config.get_item('java_config', 'java_home'),
config.get_item('java_binary_config', 'one_jar_boot_jar')))
def generate_cc_rules(self):
build_with_ccache = self.build_environment.ccache_installed
cc = os.environ.get('CC', 'gcc')
cxx = os.environ.get('CXX', 'g++')
ld = os.environ.get('LD', 'g++')
if build_with_ccache:
os.environ[
'CCACHE_BASEDIR'] = self.build_environment.blade_root_dir
os.environ['CCACHE_NOHASHDIR'] = 'true'
cc = 'ccache ' + cc
cxx = 'ccache ' + cxx
self.ccflags_manager.set_cc(cc)
cc_config = config.get_section('cc_config')
cc_library_config = config.get_section('cc_library_config')
cflags, cxxflags = cc_config['cflags'], cc_config['cxxflags']
cppflags, ldflags = self.ccflags_manager.get_flags_except_warning()
cppflags = cc_config['cppflags'] + cppflags
arflags = ''.join(cc_library_config['arflags'])
ldflags = cc_config['linkflags'] + ldflags
includes = cc_config['extra_incs']
includes = includes + ['.', self.build_dir]
includes = ' '.join(['-I%s' % inc for inc in includes])
self.generate_cc_warning_vars()
self.generate_rule(
name='cc',
command='%s -o ${out} -MMD -MF ${out}.d '
'-c -fPIC %s %s ${c_warnings} ${cppflags} '
'%s ${includes} ${in}' %
(cc, ' '.join(cflags), ' '.join(cppflags), includes),
description='CC ${in}',
depfile='${out}.d',
deps='gcc')
self.generate_rule(
name='cxx',
command='%s -o ${out} -MMD -MF ${out}.d '
'-c -fPIC %s %s ${cxx_warnings} ${cppflags} '
'%s ${includes} ${in}' %
(cxx, ' '.join(cxxflags), ' '.join(cppflags), includes),
description='CXX ${in}',
depfile='${out}.d',
deps='gcc')
if config.get_item('cc_config', 'header_inclusion_dependencies'):
preprocess = '%s -o /dev/null -E -H %s %s -w ${cppflags} %s ${includes} ${in} 2>${out}'
self.generate_rule(
name='cchdrs',
command=preprocess %
(cc, ' '.join(cflags), ' '.join(cppflags), includes),
description='CC HDRS ${in}')
self.generate_rule(
name='cxxhdrs',
command=preprocess %
(cxx, ' '.join(cxxflags), ' '.join(cppflags), includes),
description='CXX HDRS ${in}')
securecc = '%s %s' % (cc_config['securecc'], cxx)
self._add_rule('''
build __securecc_phony__ : phony
''')
self.generate_rule(
name='securecccompile',
command='%s -o ${out} -c -fPIC '
'%s %s ${cxx_warnings} ${cppflags} %s ${includes} ${in}' %
(securecc, ' '.join(cxxflags), ' '.join(cppflags), includes),
description='SECURECC ${in}')
self.generate_rule(
name='securecc',
command=self.generate_toolchain_command('securecc_object'),
description='SECURECC ${in}',
restat=True)
self.generate_rule(name='ar',
command='rm -f $out; ar %s $out $in' % arflags,
description='AR ${out}')
self.generate_rule(
name='link',
command='%s -o ${out} %s ${ldflags} ${in} ${extra_ldflags}' %
(ld, ' '.join(ldflags)),
description='LINK ${out}')
self.generate_rule(
name='solink',
command='%s -o ${out} -shared %s ${ldflags} ${in} ${extra_ldflags}'
% (ld, ' '.join(ldflags)),
description='SHAREDLINK ${out}')
def new_build_rules_generator(self):
if config.get_item('global_config', 'native_builder') == 'ninja':
return NinjaRulesGenerator('build.ninja', self.__blade_path, self)
else:
return SconsRulesGenerator('SConstruct', self.__blade_path, self)
def _target_file_path(self):
"""Return package object path according to the standard go directory layout. """
go_home = config.get_item('go_config', 'go_home')
return os.path.join(go_home, 'pkg',
'%s_%s' % (GoTarget._go_os, GoTarget._go_arch),
'%s.a' % self.data['go_package'])
def generate_proto_rules(self):
proto_config = config.get_section('proto_library_config')
protoc = proto_config['protoc']
protoc_java = protoc
if proto_config['protoc_java']:
protoc_java = proto_config['protoc_java']
protobuf_incs = protoc_import_path_option(
proto_config['protobuf_incs'])
protobuf_java_incs = protobuf_incs
if proto_config['protobuf_java_incs']:
protobuf_java_incs = protoc_import_path_option(
proto_config['protobuf_java_incs'])
self._add_rule('''
protocflags =
protoccpppluginflags =
protocjavapluginflags =
protocpythonpluginflags =
protocgoflags =
''')
self.generate_rule(
name='proto',
command='%s --proto_path=. %s -I=`dirname ${in}` '
'--cpp_out=%s ${protocflags} ${protoccpppluginflags} ${in}' %
(protoc, protobuf_incs, self.build_dir),
description='PROTOC ${in}')
self.generate_rule(
name='protojava',
command='%s --proto_path=. %s --java_out=%s/`dirname ${in}` '
'${protocjavapluginflags} ${in}' %
(protoc_java, protobuf_java_incs, self.build_dir),
description='PROTOCJAVA ${in}')
self.generate_rule(name='protopython',
command='%s --proto_path=. %s -I=`dirname ${in}` '
'--python_out=%s ${protocpythonpluginflags} ${in}' %
(protoc, protobuf_incs, self.build_dir),
description='PROTOCPYTHON ${in}')
self.generate_rule(
name='protodescriptors',
command='%s --proto_path=. %s -I=`dirname ${first}` '
'--descriptor_set_out=${out} --include_imports '
'--include_source_info ${in}' % (protoc, protobuf_incs),
description='PROTODESCRIPTORS ${in}')
protoc_go_plugin = proto_config['protoc_go_plugin']
if protoc_go_plugin:
go_home = config.get_item('go_config', 'go_home')
if not go_home:
console.error_exit(
'go_home is not configured in either BLADE_ROOT or BLADE_ROOT.local.'
)
outdir = os.path.join(go_home, 'src')
subplugins = proto_config['protoc_go_subplugins']
if subplugins:
go_out = 'plugins=%s:%s' % ('+'.join(subplugins), outdir)
else:
go_out = outdir
self.generate_rule(
name='protogo',
command='%s --proto_path=. %s -I=`dirname ${in}` '
'--plugin=protoc-gen-go=%s --go_out=%s ${in}' %
(protoc, protobuf_incs, protoc_go_plugin, go_out),
description='PROTOCGOLANG ${in}')
def _target_file_path(self):
"""Return command executable path according to the standard go directory layout. """
go_home = config.get_item('go_config', 'go_home')
return os.path.join(go_home, 'bin',
os.path.basename(self.data['go_package']))
def ninja_protoc_direct_dependencies(self, vars):
if config.get_item('proto_library_config', 'protoc_direct_dependencies'):
dependencies = self.protoc_direct_dependencies()
dependencies += config.get_item('proto_library_config', 'well_known_protos')
vars['protocflags'] = '--direct_dependencies %s' % ':'.join(dependencies)
def _generate_scala_target_platform(self):
target_platform = config.get_item('scala_config', 'target_platform')
if target_platform:
self._write_rule('%s.Append(SCALACFLAGS=["-target:%s"])' %
(self._env_name(), target_platform))
