def setUp(self):
"""
init
"""
sys.argv = ['NOT PLANISH']
module = BrocModule_pb2.Module()
module.name = 'broc'
module.module_cvspath = 'baidu/broc'
module.broc_cvspath = 'baidu/broc/BROC'
module.is_main = True
module.repo_kind = BrocModule_pb2.Module.GIT
module.revision = "1234"
module.last_changed_rev = "1236"
module.dep_level = 0
#get home dir
home = os.environ['HOME']
module.workspace = '%s/unittest_broc/workspace' % home
module.root_path = '%s/unittest_broc/workspace/baidu/broc' % home
module.url = 'https://github.com/baidu/broc'
module.br_kind = BrocModule_pb2.Module.BRANCH
module.br_name = 'trunk'
#module.commit_id = '5d9819900c2781873aa0ffce285d5d3e75b072a8'
self._module = module
Function.RunCommand("mkdir -p %s" % module.root_path, ignore_stderr_when_ok = True)
Function.RunCommand("touch %s/hello.cpp" % module.root_path, ignore_stderr_when_ok = True)
Function.RunCommand("touch %s/hello.h" % module.root_path, ignore_stderr_when_ok = True)
self._env = Environment.Environment(module)
Environment.SetCurrent(self._env)
def test_GLOB(self):
"""
test Syntax.GLOB
"""
#test case of one file
files = Syntax.GLOB("./*.cpp")
self.assertEqual(files, "hello.cpp")
#test case of more files and those files must in the lexicographical order
module = self._module
Function.RunCommand("touch %s/hello1.h" % module.root_path, ignore_stderr_when_ok = True)
Function.RunCommand("touch %s/hello2.h" % module.root_path, ignore_stderr_when_ok = True)
Function.RunCommand("touch %s/hello3.h" % module.root_path, ignore_stderr_when_ok = True)
Function.RunCommand("touch %s/hello10.h" % module.root_path, ignore_stderr_when_ok = True)
files = Syntax.GLOB("./*.h")
self.assertEqual(files, "hello.h hello1.h hello10.h hello2.h hello3.h")
#test case of files not in self module
Function.RunCommand("touch %s/../README" % module.root_path, ignore_stderr_when_ok = True)
flag = False
try:
Syntax.GLOB("../README")
except Syntax.NotInSelfModuleError as e:
flag = True
self.assertTrue(flag)
#test case of no such files
flag = False
try:
Syntax.GLOB("./just_test.cpp")
except Syntax.BrocArgumentIllegalError as e:
flag = True
self.assertTrue(flag)
def test_STATIC_LIBRARY(self):
"""
test Syntax.STATIC_LIBRARY
"""
#set local flags tag
libs = Syntax.Libs("$OUT_ROOT/baidu/broc/libhello.a")
cpptags = Syntax.CppFlags("-DBROC", "-DRELEASE")
src = Syntax.Sources("hello.cpp")
#an error name of application
flag = False
try:
Syntax.STATIC_LIBRARY("^*^&*!*$^", src)
except Syntax.BrocArgumentIllegalError as ex:
flag = True
self.assertTrue(flag)
#an error args of application
flag = False
try:
Syntax.STATIC_LIBRARY("hello", src, cpptags)
except Syntax.BrocArgumentIllegalError as ex:
flag = True
self.assertTrue(flag)
#Libs
Syntax.STATIC_LIBRARY("hello", src, libs)
library = self._env.Targets()[0]
library.Action()
self.assertEqual(library.tag_libs.V(),
["broc_out/baidu/broc/libhello.a"])
#two samename target
flag = False
try:
Syntax.STATIC_LIBRARY("hello", src)
except Syntax.BrocArgumentIllegalError as ex:
flag = True
self.assertTrue(flag)
#library DoCopy
Function.RunCommand("mkdir -p %s/lib" % self._module.root_path, \
ignore_stderr_when_ok = True)
Function.RunCommand("touch %s/lib/libhello1.a" % self._module.root_path, \
ignore_stderr_when_ok = True)
now_dir = os.getcwd()
os.chdir(self._module.workspace)
Syntax.STATIC_LIBRARY("hello1")
lib_paths = os.path.join(self._module.workspace, "broc_out", \
self._module.module_cvspath, "output/lib/libhello1.a")
self.assertTrue(os.path.exists(lib_paths))
os.chdir(now_dir)
def test_PROTO_LIBRARY(self):
"""
test Syntax.PROTO_LIBRARY
"""
#make a new proto file
Function.RunCommand("touch %s/hello.proto" % self._module.root_path, \
ignore_stderr_when_ok = True)
#set local flags
cpptags = Syntax.CppFlags("-DDEBUG_LOCAL", "-DRELEASE_LOCAL")
cxxtags = Syntax.CxxFlags("-Wwrite-strings", "-Wswitch")
incflag = Syntax.Include("")
libflag = Syntax.Libs("$OUT_ROOT/baidu/broc/output/lib/libhello.a")
now_dir = os.getcwd()
os.chdir(self._module.workspace)
protos = Syntax.PROTO_LIBRARY("hello", "*.proto", cpptags, cxxtags, incflag, libflag)
proto_library = self._env.Targets()[0]
src = proto_library.tag_sources.V()[0]
proto_library.Action()
os.chdir(now_dir)
#check result
proto_cmd = """mkdir -p broc_out/baidu/broc && protoc \
--cpp_out=broc_out/baidu/broc -I=baidu/broc \
-I=. baidu/broc/*.proto\n"""
self.assertEqual(' '.join(protos.__str__().split()), ' '.join(proto_cmd.split()))
self.assertEqual(src.cppflags, ["-DDEBUG_LOCAL"])
self.assertEqual(src.cxxflags, ["-Wwrite-strings"])
self.assertEqual(src.includes, [".", "broc_out", 'baidu/broc',
u'broc_out/baidu/broc'])
self.assertEqual(src.infile, "broc_out/baidu/broc/hello.pb.cc")
self.assertEqual(proto_library.tag_libs.V(), \
["broc_out/baidu/broc/output/lib/libhello.a"])
def CalcHeaderFiles(self):
"""
calculate the header files that source file dependends
Returns:
{ ret : True | False, headers : set(), msg : 'error message' }
calculate successfully ret is True; otherwise ret is False and msg contains error message
"""
result = dict()
result['ret'] = False
result['headers'] = set()
result['msg'] = ''
retcode, msg = Function.RunCommand(self._header_cmd,
ignore_stderr_when_ok=True)
if retcode != 0:
result['msg'] = '%s:%s' % (msg, self._header_cmd)
return result
files = msg.split()
for f in files:
if f.endswith(".h"):
if self.workspace in f:
result['headers'].add(f[len(self.workspace) + 1:])
else:
result['headers'].add(f)
result['ret'] = True
return result
def DoCopy(self):
"""
if .a file has been built already before compile action, just copy it from code directory to output directory
Returns:
return True if copy success
return False if fail to copy
"""
if len(self.tag_sources.V()):
Log.Log().LevPrint(
"ERROR", 'StaticLibrary(%s) can not copy because \
its source is not empty' % self.name)
return False
root, _ = os.path.splitext(self.name)
frm = os.path.join(self.env.ModuleCVSPath(), 'lib',
'lib%s%s' % (root, '.a'))
to = os.path.join("broc_out", self.env.ModuleCVSPath(), 'output/lib')
cmd = "mkdir -p %(to)s && cp -Rp %(frm)s %(to)s" % (locals())
Log.Log().LevPrint('MSG', '[PreCopy] %s' % cmd)
ret, msg = Function.RunCommand(cmd)
if ret != 0:
Log.Log().LevPrint("ERROR", "[ERROR] %s\n%s" % (cmd, msg))
return False
else:
return True
def DoPublish(self):
"""
do publish cmd
"""
for cmd in self._publish_cmd:
ret, msg = Function.RunCommand(cmd)
if ret != 0:
return (False, msg)
return (True, '')
def _download_modules(self):
"""
dnowload dependent module from repository
Returns:
download all modules successfully return True, otherwise return False
"""
for k, node in self.planished_nodes.iteritems():
if node.module.repo_kind == BrocModule_pb2.Module.SVN:
# the infos of local code equal Node's info
if os.path.exists(node.module.root_path):
if BrocTree.BrocTree().SameSvnNode(node):
continue
else:
dst = "%s-%f" % (node.module.root_path, time.time())
self.logger.LevPrint(
"WARNING", "local code doesn't match BROC, \
reload it(%s)" % (node.module.origin_config))
Function.MoveFiles(node.module.root_path, dst)
# generate command donwloading code from repository
cmd = None
url = node.module.url
if node.module.revision:
url = "%s -r %s --force" % (url, node.module.revision)
else:
url = "%s --force" % url
cmd = "svn checkout %s %s" % (url, node.module.root_path)
else:
# for git
cmd = "cd %s" % node.module.module_cvspath
if node.module.tag_name:
tag_name = node.module.tag_name
cmd += " && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \
% (tag_name, tag_name, tag_name, tag_name)
elif node.module.br_name:
br_name = node.module.br_name
if br_name != "master":
cmd += " && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \
% (br_name, br_name, br_name, br_name)
else:
cmd += " && git checkout master"
self.logger.LevPrint("MSG", "%s" % cmd)
ret, msg = Function.RunCommand(cmd)
if ret != 0:
self.logger.LevPrint("ERROR", "%s failed(%s)" % (cmd, msg))
return False
return True
def _download_broc(self, node):
"""
download BROC file from repository
Args:
node : the BrocNode object
Returns:
return abs path of BROC file if download success
return None if download failed
"""
broc_path = None
cmd = None
# for svn
# Log.Log().LevPrint("MSG", 'download BROC %s' % node.module.url)
if node.module.repo_kind == BrocModule_pb2.Module.SVN:
hash_value = Function.CalcHash(node.module.url)
broc_url = os.path.join(node.module.url, 'BROC')
broc_path = os.path.join(self._broc_dir, "%s_BROC" % hash_value)
if node.module.revision:
broc_url = "%s -r %s" % (broc_url, node.module.revision)
cmd = "svn export %s %s" % (broc_url, broc_path)
else:
# for GIT
cmd = ""
broc_path = os.path.join(node.module.workspace, node.module.module_cvspath, 'BROC')
broc_dir = os.path.dirname(broc_path)
if not os.path.exists(broc_path):
cmd += "git clone %s %s" \
% ("%s.git" % node.module.url, "%s" % broc_dir)
if node.module.br_name and node.module.br_name != 'master':
br_name = node.module.br_name
cmd += " && cd %s && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \
% (broc_dir, br_name, br_name, br_name, br_name)
elif node.module.tag_name:
tag_name = node.module.tag_name
cmd += " && cd %s && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \
% (broc_dir, tag_name, tag_name, tag_name, tag_name)
if cmd:
Log.Log().LevPrint("MSG", "Getting BROC(%s) ..." % cmd)
ret, msg = Function.RunCommand(cmd)
if ret != 0:
Log.Log().LevPrint("ERROR", msg)
return None
return broc_path
def _check_broc(self, node):
"""
to check BROC file
Args:
node : the object of BrocNode
Returns:
return the abs path of BROC file if check successfully
return None if fail to check BROC file
"""
broc_path = os.path.join(node.module.workspace,
node.module.module_cvspath, 'BROC')
# BROC exists in local file system
if os.path.exists(broc_path):
if node.module.repo_kind == BrocModule_pb2.Module.SVN:
if self.SameSvnNode(node):
return broc_path
else:
return self._download_broc(node)
else:
_broc_dir = os.path.join(node.module.workspace,
node.module.module_cvspath)
if node.module.is_main:
return os.path.join(_broc_dir, "BROC")
cmd = "cd %s " % _broc_dir
if node.module.tag_name:
cmd += " && (git checkout %s || (git fetch --all && git checkout %s))" \
% (node.module.tag_name, node.module.tag_name)
else:
cmd += " && (git checkout %s || (git fetch --all && git checkout %s))" \
% (node.module.br_name, node.module.br_name)
# to run cmd
# self._logger.LevPrint("MSG", "run cmd: %s ..." % cmd)
ret, msg = Function.RunCommand(cmd)
if ret != 0:
self._logger.LevPrint(
"ERROR",
"fail to find BROC(%s) failed(%s)" % (cmd, msg))
# FIX ME, maybe return None is better
raise BrocTreeError("%s\n%s" % (cmd, msg))
else:
return broc_path
# to dowonload BROC
else:
return self._download_broc(node)
def DoBuild(self):
"""
to run build cmd
Returns:
return (True, '') if build successfully
return (False, 'error msg') if fail to build
"""
result = dict()
ret, msg = Function.RunCommand(self.build_cmd,
ignore_stderr_when_ok=False)
if ret != 0:
result['ret'] = False
else:
self.build = False
result['ret'] = True
result['msg'] = self.build_cmd + '\n' + msg
return result
def Run(self):
"""
thread entrence function
"""
while not self._queue.empty():
try:
cmd = self._queue.get(True, 1)
except Queue.Empty:
break
ret, msg = Function.RunCommand(cmd, True)
if ret != 0:
self._logger.LevPrint("ERROR",
"run ut cmd(%s) failed: %s" % (cmd, msg))
self._errors.append(msg)
else:
self._logger.LevPrint("MSG",
"run ut cmd(%s) OK\n%s" % (cmd, msg))
self._queue.task_done()
def _download_broc(self, node):
"""
download BROC from node's repository
Args:
node : BrocNode object of module
Returns:
return the path of BROC file if download successfully;
return None if fail to download BROC
"""
broc_path = None
cmd = None
# for svn
if node.module.repo_kind == BrocModule_pb2.Module.SVN:
_file = Function.CalcMd5(node.module.url)
broc_url = os.path.join(node.module.url, 'BROC')
broc_path = os.path.join(self._broc_dir, "%s_BROC" % _file)
if node.module.revision:
broc_url = "%s -r %s" % (broc_url, node.module.revision)
cmd = "svn export %s %s" % (broc_url, broc_path)
else:
# for GIT
broc_path = os.path.join(node.module.workspace,
node.module.module_cvspath, 'BROC')
cmd = "git clone %s %s && cd %s" \
% (node.module.url, node.module.module_cvspath, node.module.module_cvspath)
if node.module.br_name:
cmd += " && (git checkout %s || git fetch -all && git checkout %s)" \
% (node.module.br_name, node.module.br_name)
elif node.module.tag_name:
cmd += " && git fetch -all && git checkout %s " % node.module.tag_name
else:
self._logger.LevPrint("ERROR", "couldn't find node(%s) branch or tag name" \
% node.module.module_cvspath)
return None
self._logger.LevPrint("MSG", "run command %s" % cmd)
ret, msg = Function.RunCommand(cmd)
if ret != 0:
self._logger.LevPrint("ERROR", "%s" % msg)
self._need_broc_list.append(node.module.url)
return None
return broc_path
def PreAction(self):
"""
parse proto flags and gernerate the command to handle proto file
Returns:
return (True, '') if deal with proto files successfully, otherwise return (False, 'error msg')
"""
proto_dirs = list()
# find the first directory of all proto files
# for example: a/b/c/util.proto, the first directory is a, to handle proto like this because
# https://developers.google.com/protocol-buffers/docs/reference/python-generated#invocation
proto_flags = " ".join(self._tag_protoflags.V())
# add the cvs path of directory of BROC
self._tag_include.AddV(self.env.BrocCVSDir())
cvs_dirs = " ".join(
map(lambda x: "-I=%s " % os.path.normpath(x),
self._tag_include.V()))
#protoc = os.path.join(os.environ['HOME'], "broc/protobuf/bin/protoc")
protoc = 'protoc'
for proto in self._protos.split():
normpath_proto = os.path.normpath(proto)
protos = os.path.join(self.env.BrocCVSDir(), normpath_proto)
out = os.path.normpath(
os.path.join("broc_out", self.env.BrocCVSDir(),
os.path.dirname(normpath_proto)))
cpp_out = os.path.join('broc_out', self.env.BrocCVSDir())
pos = normpath_proto.find('/')
if pos != -1:
cpp_out = os.path.join('broc_out', self.env.BrocCVSDir(),
normpath_proto[:pos])
# the current working directory is $WORKSPACE
cmd = "mkdir -p %(out)s && %(protoc)s --cpp_out=%(cpp_out)s %(proto_flags)s %(cvs_dirs)s \
-I=. %(protos)s" % (locals())
self._proto_cmds.add(cmd)
# run protoc
for cmd in self._proto_cmds:
Log.Log().LevPrint("MSG", "%s" % cmd)
ret, msg = Function.RunCommand(cmd, True)
if ret != 0:
return (False, "%s%s" % (cmd, msg))
return (True, '')
def PreAction(self):
"""
parse proto flags and gernerate the command to handle proto file
Returns:
return (True, '') if deal with proto files successfully, otherwise return (False, 'error msg')
"""
proto_dirs = set()
# find the cvs path of directory of all proto files
for proto in self._protos.split():
proto_dirs.add(
os.path.join(self.env.BrocCVSDir(), os.path.dirname(proto)))
proto_flags = " ".join(self._tag_protoflags.V())
# add protobuf include set from PROTO_LIBRARY
cvs_dirs = " ".join(map(lambda x: "-I=%s " % x, self._tag_include.V()))
# add cvs path of directory of BROC
cvs_dirs += "-I=%s " % self.env.BrocCVSDir()
#protoc = os.path.join(os.environ['HOME'], "broc/protobuf/bin/protoc")
protoc = 'protoc'
for _dir in proto_dirs:
# cvs_out_dir = os.path.normpath(os.path.join('broc_out', _dir))
cvs_out_dir = os.path.normpath(
os.path.join('broc_out', self.env.BrocCVSDir()))
protos = os.path.normpath("%(_dir)s/*.proto" % (locals()))
cmd = "mkdir -p %(cvs_out_dir)s && %(protoc)s --cpp_out=%(cvs_out_dir)s %(proto_flags)s %(cvs_dirs)s \
-I=. %(protos)s" % (locals())
# 执行protoc的目录，在output下
self._proto_cmds.add(cmd)
# run protoc
for cmd in self._proto_cmds:
Log.Log().LevPrint("MSG", "%s" % cmd)
ret, msg = Function.RunCommand(cmd, True)
if ret != 0:
return (False, "%s%s" % (cmd, msg))
return (True, '')
def tearDown(self):
"""
teardown
"""
Function.RunCommand("rm -rf ~/unittest_broc", ignore_stderr_when_ok = True)