def setUp(self): """ init """ sys.argv = ['NOT PLANISH'] module = BrocModule_pb2.Module() module.name = 'broc' module.module_cvspath = 'baidu/broc' module.broc_cvspath = 'baidu/broc/BROC' module.is_main = True module.repo_kind = BrocModule_pb2.Module.GIT module.revision = "1234" module.last_changed_rev = "1236" module.dep_level = 0 #get home dir home = os.environ['HOME'] module.workspace = '%s/unittest_broc/workspace' % home module.root_path = '%s/unittest_broc/workspace/baidu/broc' % home module.url = 'https://github.com/baidu/broc' module.br_kind = BrocModule_pb2.Module.BRANCH module.br_name = 'trunk' #module.commit_id = '5d9819900c2781873aa0ffce285d5d3e75b072a8' self._module = module Function.RunCommand("mkdir -p %s" % module.root_path, ignore_stderr_when_ok = True) Function.RunCommand("touch %s/hello.cpp" % module.root_path, ignore_stderr_when_ok = True) Function.RunCommand("touch %s/hello.h" % module.root_path, ignore_stderr_when_ok = True) self._env = Environment.Environment(module) Environment.SetCurrent(self._env)
def test_GLOB(self): """ test Syntax.GLOB """ #test case of one file files = Syntax.GLOB("./*.cpp") self.assertEqual(files, "hello.cpp") #test case of more files and those files must in the lexicographical order module = self._module Function.RunCommand("touch %s/hello1.h" % module.root_path, ignore_stderr_when_ok = True) Function.RunCommand("touch %s/hello2.h" % module.root_path, ignore_stderr_when_ok = True) Function.RunCommand("touch %s/hello3.h" % module.root_path, ignore_stderr_when_ok = True) Function.RunCommand("touch %s/hello10.h" % module.root_path, ignore_stderr_when_ok = True) files = Syntax.GLOB("./*.h") self.assertEqual(files, "hello.h hello1.h hello10.h hello2.h hello3.h") #test case of files not in self module Function.RunCommand("touch %s/../README" % module.root_path, ignore_stderr_when_ok = True) flag = False try: Syntax.GLOB("../README") except Syntax.NotInSelfModuleError as e: flag = True self.assertTrue(flag) #test case of no such files flag = False try: Syntax.GLOB("./just_test.cpp") except Syntax.BrocArgumentIllegalError as e: flag = True self.assertTrue(flag)
def test_STATIC_LIBRARY(self): """ test Syntax.STATIC_LIBRARY """ #set local flags tag libs = Syntax.Libs("$OUT_ROOT/baidu/broc/libhello.a") cpptags = Syntax.CppFlags("-DBROC", "-DRELEASE") src = Syntax.Sources("hello.cpp") #an error name of application flag = False try: Syntax.STATIC_LIBRARY("^*^&*!*$^", src) except Syntax.BrocArgumentIllegalError as ex: flag = True self.assertTrue(flag) #an error args of application flag = False try: Syntax.STATIC_LIBRARY("hello", src, cpptags) except Syntax.BrocArgumentIllegalError as ex: flag = True self.assertTrue(flag) #Libs Syntax.STATIC_LIBRARY("hello", src, libs) library = self._env.Targets()[0] library.Action() self.assertEqual(library.tag_libs.V(), ["broc_out/baidu/broc/libhello.a"]) #two samename target flag = False try: Syntax.STATIC_LIBRARY("hello", src) except Syntax.BrocArgumentIllegalError as ex: flag = True self.assertTrue(flag) #library DoCopy Function.RunCommand("mkdir -p %s/lib" % self._module.root_path, \ ignore_stderr_when_ok = True) Function.RunCommand("touch %s/lib/libhello1.a" % self._module.root_path, \ ignore_stderr_when_ok = True) now_dir = os.getcwd() os.chdir(self._module.workspace) Syntax.STATIC_LIBRARY("hello1") lib_paths = os.path.join(self._module.workspace, "broc_out", \ self._module.module_cvspath, "output/lib/libhello1.a") self.assertTrue(os.path.exists(lib_paths)) os.chdir(now_dir)
def test_PROTO_LIBRARY(self): """ test Syntax.PROTO_LIBRARY """ #make a new proto file Function.RunCommand("touch %s/hello.proto" % self._module.root_path, \ ignore_stderr_when_ok = True) #set local flags cpptags = Syntax.CppFlags("-DDEBUG_LOCAL", "-DRELEASE_LOCAL") cxxtags = Syntax.CxxFlags("-Wwrite-strings", "-Wswitch") incflag = Syntax.Include("") libflag = Syntax.Libs("$OUT_ROOT/baidu/broc/output/lib/libhello.a") now_dir = os.getcwd() os.chdir(self._module.workspace) protos = Syntax.PROTO_LIBRARY("hello", "*.proto", cpptags, cxxtags, incflag, libflag) proto_library = self._env.Targets()[0] src = proto_library.tag_sources.V()[0] proto_library.Action() os.chdir(now_dir) #check result proto_cmd = """mkdir -p broc_out/baidu/broc && protoc \ --cpp_out=broc_out/baidu/broc -I=baidu/broc \ -I=. baidu/broc/*.proto\n""" self.assertEqual(' '.join(protos.__str__().split()), ' '.join(proto_cmd.split())) self.assertEqual(src.cppflags, ["-DDEBUG_LOCAL"]) self.assertEqual(src.cxxflags, ["-Wwrite-strings"]) self.assertEqual(src.includes, [".", "broc_out", 'baidu/broc', u'broc_out/baidu/broc']) self.assertEqual(src.infile, "broc_out/baidu/broc/hello.pb.cc") self.assertEqual(proto_library.tag_libs.V(), \ ["broc_out/baidu/broc/output/lib/libhello.a"])
def CalcHeaderFiles(self): """ calculate the header files that source file dependends Returns: { ret : True | False, headers : set(), msg : 'error message' } calculate successfully ret is True; otherwise ret is False and msg contains error message """ result = dict() result['ret'] = False result['headers'] = set() result['msg'] = '' retcode, msg = Function.RunCommand(self._header_cmd, ignore_stderr_when_ok=True) if retcode != 0: result['msg'] = '%s:%s' % (msg, self._header_cmd) return result files = msg.split() for f in files: if f.endswith(".h"): if self.workspace in f: result['headers'].add(f[len(self.workspace) + 1:]) else: result['headers'].add(f) result['ret'] = True return result
def DoCopy(self): """ if .a file has been built already before compile action, just copy it from code directory to output directory Returns: return True if copy success return False if fail to copy """ if len(self.tag_sources.V()): Log.Log().LevPrint( "ERROR", 'StaticLibrary(%s) can not copy because \ its source is not empty' % self.name) return False root, _ = os.path.splitext(self.name) frm = os.path.join(self.env.ModuleCVSPath(), 'lib', 'lib%s%s' % (root, '.a')) to = os.path.join("broc_out", self.env.ModuleCVSPath(), 'output/lib') cmd = "mkdir -p %(to)s && cp -Rp %(frm)s %(to)s" % (locals()) Log.Log().LevPrint('MSG', '[PreCopy] %s' % cmd) ret, msg = Function.RunCommand(cmd) if ret != 0: Log.Log().LevPrint("ERROR", "[ERROR] %s\n%s" % (cmd, msg)) return False else: return True
def DoPublish(self): """ do publish cmd """ for cmd in self._publish_cmd: ret, msg = Function.RunCommand(cmd) if ret != 0: return (False, msg) return (True, '')
def _download_modules(self): """ dnowload dependent module from repository Returns: download all modules successfully return True, otherwise return False """ for k, node in self.planished_nodes.iteritems(): if node.module.repo_kind == BrocModule_pb2.Module.SVN: # the infos of local code equal Node's info if os.path.exists(node.module.root_path): if BrocTree.BrocTree().SameSvnNode(node): continue else: dst = "%s-%f" % (node.module.root_path, time.time()) self.logger.LevPrint( "WARNING", "local code doesn't match BROC, \ reload it(%s)" % (node.module.origin_config)) Function.MoveFiles(node.module.root_path, dst) # generate command donwloading code from repository cmd = None url = node.module.url if node.module.revision: url = "%s -r %s --force" % (url, node.module.revision) else: url = "%s --force" % url cmd = "svn checkout %s %s" % (url, node.module.root_path) else: # for git cmd = "cd %s" % node.module.module_cvspath if node.module.tag_name: tag_name = node.module.tag_name cmd += " && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \ % (tag_name, tag_name, tag_name, tag_name) elif node.module.br_name: br_name = node.module.br_name if br_name != "master": cmd += " && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \ % (br_name, br_name, br_name, br_name) else: cmd += " && git checkout master" self.logger.LevPrint("MSG", "%s" % cmd) ret, msg = Function.RunCommand(cmd) if ret != 0: self.logger.LevPrint("ERROR", "%s failed(%s)" % (cmd, msg)) return False return True
def _download_broc(self, node): """ download BROC file from repository Args: node : the BrocNode object Returns: return abs path of BROC file if download success return None if download failed """ broc_path = None cmd = None # for svn # Log.Log().LevPrint("MSG", 'download BROC %s' % node.module.url) if node.module.repo_kind == BrocModule_pb2.Module.SVN: hash_value = Function.CalcHash(node.module.url) broc_url = os.path.join(node.module.url, 'BROC') broc_path = os.path.join(self._broc_dir, "%s_BROC" % hash_value) if node.module.revision: broc_url = "%s -r %s" % (broc_url, node.module.revision) cmd = "svn export %s %s" % (broc_url, broc_path) else: # for GIT cmd = "" broc_path = os.path.join(node.module.workspace, node.module.module_cvspath, 'BROC') broc_dir = os.path.dirname(broc_path) if not os.path.exists(broc_path): cmd += "git clone %s %s" \ % ("%s.git" % node.module.url, "%s" % broc_dir) if node.module.br_name and node.module.br_name != 'master': br_name = node.module.br_name cmd += " && cd %s && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \ % (broc_dir, br_name, br_name, br_name, br_name) elif node.module.tag_name: tag_name = node.module.tag_name cmd += " && cd %s && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \ % (broc_dir, tag_name, tag_name, tag_name, tag_name) if cmd: Log.Log().LevPrint("MSG", "Getting BROC(%s) ..." % cmd) ret, msg = Function.RunCommand(cmd) if ret != 0: Log.Log().LevPrint("ERROR", msg) return None return broc_path
def _check_broc(self, node): """ to check BROC file Args: node : the object of BrocNode Returns: return the abs path of BROC file if check successfully return None if fail to check BROC file """ broc_path = os.path.join(node.module.workspace, node.module.module_cvspath, 'BROC') # BROC exists in local file system if os.path.exists(broc_path): if node.module.repo_kind == BrocModule_pb2.Module.SVN: if self.SameSvnNode(node): return broc_path else: return self._download_broc(node) else: _broc_dir = os.path.join(node.module.workspace, node.module.module_cvspath) if node.module.is_main: return os.path.join(_broc_dir, "BROC") cmd = "cd %s " % _broc_dir if node.module.tag_name: cmd += " && (git checkout %s || (git fetch --all && git checkout %s))" \ % (node.module.tag_name, node.module.tag_name) else: cmd += " && (git checkout %s || (git fetch --all && git checkout %s))" \ % (node.module.br_name, node.module.br_name) # to run cmd # self._logger.LevPrint("MSG", "run cmd: %s ..." % cmd) ret, msg = Function.RunCommand(cmd) if ret != 0: self._logger.LevPrint( "ERROR", "fail to find BROC(%s) failed(%s)" % (cmd, msg)) # FIX ME, maybe return None is better raise BrocTreeError("%s\n%s" % (cmd, msg)) else: return broc_path # to dowonload BROC else: return self._download_broc(node)
def DoBuild(self): """ to run build cmd Returns: return (True, '') if build successfully return (False, 'error msg') if fail to build """ result = dict() ret, msg = Function.RunCommand(self.build_cmd, ignore_stderr_when_ok=False) if ret != 0: result['ret'] = False else: self.build = False result['ret'] = True result['msg'] = self.build_cmd + '\n' + msg return result
def Run(self): """ thread entrence function """ while not self._queue.empty(): try: cmd = self._queue.get(True, 1) except Queue.Empty: break ret, msg = Function.RunCommand(cmd, True) if ret != 0: self._logger.LevPrint("ERROR", "run ut cmd(%s) failed: %s" % (cmd, msg)) self._errors.append(msg) else: self._logger.LevPrint("MSG", "run ut cmd(%s) OK\n%s" % (cmd, msg)) self._queue.task_done()
def _download_broc(self, node): """ download BROC from node's repository Args: node : BrocNode object of module Returns: return the path of BROC file if download successfully; return None if fail to download BROC """ broc_path = None cmd = None # for svn if node.module.repo_kind == BrocModule_pb2.Module.SVN: _file = Function.CalcMd5(node.module.url) broc_url = os.path.join(node.module.url, 'BROC') broc_path = os.path.join(self._broc_dir, "%s_BROC" % _file) if node.module.revision: broc_url = "%s -r %s" % (broc_url, node.module.revision) cmd = "svn export %s %s" % (broc_url, broc_path) else: # for GIT broc_path = os.path.join(node.module.workspace, node.module.module_cvspath, 'BROC') cmd = "git clone %s %s && cd %s" \ % (node.module.url, node.module.module_cvspath, node.module.module_cvspath) if node.module.br_name: cmd += " && (git checkout %s || git fetch -all && git checkout %s)" \ % (node.module.br_name, node.module.br_name) elif node.module.tag_name: cmd += " && git fetch -all && git checkout %s " % node.module.tag_name else: self._logger.LevPrint("ERROR", "couldn't find node(%s) branch or tag name" \ % node.module.module_cvspath) return None self._logger.LevPrint("MSG", "run command %s" % cmd) ret, msg = Function.RunCommand(cmd) if ret != 0: self._logger.LevPrint("ERROR", "%s" % msg) self._need_broc_list.append(node.module.url) return None return broc_path
def PreAction(self): """ parse proto flags and gernerate the command to handle proto file Returns: return (True, '') if deal with proto files successfully, otherwise return (False, 'error msg') """ proto_dirs = list() # find the first directory of all proto files # for example: a/b/c/util.proto, the first directory is a, to handle proto like this because # https://developers.google.com/protocol-buffers/docs/reference/python-generated#invocation proto_flags = " ".join(self._tag_protoflags.V()) # add the cvs path of directory of BROC self._tag_include.AddV(self.env.BrocCVSDir()) cvs_dirs = " ".join( map(lambda x: "-I=%s " % os.path.normpath(x), self._tag_include.V())) #protoc = os.path.join(os.environ['HOME'], "broc/protobuf/bin/protoc") protoc = 'protoc' for proto in self._protos.split(): normpath_proto = os.path.normpath(proto) protos = os.path.join(self.env.BrocCVSDir(), normpath_proto) out = os.path.normpath( os.path.join("broc_out", self.env.BrocCVSDir(), os.path.dirname(normpath_proto))) cpp_out = os.path.join('broc_out', self.env.BrocCVSDir()) pos = normpath_proto.find('/') if pos != -1: cpp_out = os.path.join('broc_out', self.env.BrocCVSDir(), normpath_proto[:pos]) # the current working directory is $WORKSPACE cmd = "mkdir -p %(out)s && %(protoc)s --cpp_out=%(cpp_out)s %(proto_flags)s %(cvs_dirs)s \ -I=. %(protos)s" % (locals()) self._proto_cmds.add(cmd) # run protoc for cmd in self._proto_cmds: Log.Log().LevPrint("MSG", "%s" % cmd) ret, msg = Function.RunCommand(cmd, True) if ret != 0: return (False, "%s%s" % (cmd, msg)) return (True, '')
def PreAction(self): """ parse proto flags and gernerate the command to handle proto file Returns: return (True, '') if deal with proto files successfully, otherwise return (False, 'error msg') """ proto_dirs = set() # find the cvs path of directory of all proto files for proto in self._protos.split(): proto_dirs.add( os.path.join(self.env.BrocCVSDir(), os.path.dirname(proto))) proto_flags = " ".join(self._tag_protoflags.V()) # add protobuf include set from PROTO_LIBRARY cvs_dirs = " ".join(map(lambda x: "-I=%s " % x, self._tag_include.V())) # add cvs path of directory of BROC cvs_dirs += "-I=%s " % self.env.BrocCVSDir() #protoc = os.path.join(os.environ['HOME'], "broc/protobuf/bin/protoc") protoc = 'protoc' for _dir in proto_dirs: # cvs_out_dir = os.path.normpath(os.path.join('broc_out', _dir)) cvs_out_dir = os.path.normpath( os.path.join('broc_out', self.env.BrocCVSDir())) protos = os.path.normpath("%(_dir)s/*.proto" % (locals())) cmd = "mkdir -p %(cvs_out_dir)s && %(protoc)s --cpp_out=%(cvs_out_dir)s %(proto_flags)s %(cvs_dirs)s \ -I=. %(protos)s" % (locals()) # 执行protoc的目录,在output下 self._proto_cmds.add(cmd) # run protoc for cmd in self._proto_cmds: Log.Log().LevPrint("MSG", "%s" % cmd) ret, msg = Function.RunCommand(cmd, True) if ret != 0: return (False, "%s%s" % (cmd, msg)) return (True, '')
def tearDown(self): """ teardown """ Function.RunCommand("rm -rf ~/unittest_broc", ignore_stderr_when_ok = True)