class AutoGenWorkerInProcess(mp.Process):
def __init__(self, module_queue, data_pipe_file_path, feedback_q,
file_lock, cache_q, log_q, error_event):
mp.Process.__init__(self)
self.module_queue = module_queue
self.data_pipe_file_path = data_pipe_file_path
self.data_pipe = None
self.feedback_q = feedback_q
self.PlatformMetaFileSet = {}
self.file_lock = file_lock
self.cache_q = cache_q
self.log_q = log_q
self.error_event = error_event
def GetPlatformMetaFile(self, filepath, root):
try:
return self.PlatformMetaFileSet[(filepath, root)]
except:
self.PlatformMetaFileSet[(filepath, root)] = filepath
return self.PlatformMetaFileSet[(filepath, root)]
def run(self):
try:
taskname = "Init"
with self.file_lock:
try:
self.data_pipe = MemoryDataPipe()
self.data_pipe.load(self.data_pipe_file_path)
except:
self.feedback_q.put(taskname + ":" +
"load data pipe %s failed." %
self.data_pipe_file_path)
EdkLogger.LogClientInitialize(self.log_q)
loglevel = self.data_pipe.Get("LogLevel")
if not loglevel:
loglevel = EdkLogger.INFO
EdkLogger.SetLevel(loglevel)
target = self.data_pipe.Get("P_Info").get("Target")
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
archlist = self.data_pipe.Get("P_Info").get("ArchList")
active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")
workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")
PackagesPath = os.getenv("PACKAGES_PATH")
mws.setWs(workspacedir, PackagesPath)
self.Wa = WorkSpaceInfo(workspacedir, active_p, target, toolchain,
archlist)
self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")
GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")
GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")
os.environ._data = self.data_pipe.Get("Env_Var")
GlobalData.gWorkspace = workspacedir
GlobalData.gDisableIncludePathCheck = False
GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache")
GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource")
GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest")
GlobalData.gPlatformHashFile = self.data_pipe.Get(
"PlatformHashFile")
GlobalData.gModulePreMakeCacheStatus = dict()
GlobalData.gModuleMakeCacheStatus = dict()
GlobalData.gHashChainStatus = dict()
GlobalData.gCMakeHashFile = dict()
GlobalData.gModuleHashFile = dict()
GlobalData.gFileHashDict = dict()
GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get(
"EnableGenfdsMultiThread")
GlobalData.file_lock = self.file_lock
CommandTarget = self.data_pipe.Get("CommandTarget")
pcd_from_build_option = []
for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
pcd_id = ".".join((pcd_tuple[0], pcd_tuple[1]))
if pcd_tuple[2].strip():
pcd_id = ".".join((pcd_id, pcd_tuple[2]))
pcd_from_build_option.append("=".join((pcd_id, pcd_tuple[3])))
GlobalData.BuildOptionPcd = pcd_from_build_option
module_count = 0
FfsCmd = self.data_pipe.Get("FfsCommand")
if FfsCmd is None:
FfsCmd = {}
GlobalData.FfsCmd = FfsCmd
PlatformMetaFile = self.GetPlatformMetaFile(
self.data_pipe.Get("P_Info").get("ActivePlatform"),
self.data_pipe.Get("P_Info").get("WorkspaceDir"))
while True:
if self.module_queue.empty():
break
if self.error_event.is_set():
break
module_count += 1
module_file, module_root, module_path, module_basename, module_originalpath, module_arch, IsLib = self.module_queue.get_nowait(
)
modulefullpath = os.path.join(module_root, module_file)
taskname = " : ".join((modulefullpath, module_arch))
module_metafile = PathClass(module_file, module_root)
if module_path:
module_metafile.Path = module_path
if module_basename:
module_metafile.BaseName = module_basename
if module_originalpath:
module_metafile.OriginalPath = PathClass(
module_originalpath, module_root)
arch = module_arch
target = self.data_pipe.Get("P_Info").get("Target")
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
Ma = ModuleAutoGen(self.Wa, module_metafile, target, toolchain,
arch, PlatformMetaFile, self.data_pipe)
Ma.IsLibrary = IsLib
# SourceFileList calling sequence impact the makefile string sequence.
# Create cached SourceFileList here to unify its calling sequence for both
# CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
RetVal = Ma.SourceFileList
if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [
None, "", "all"
]:
try:
CacheResult = Ma.CanSkipbyPreMakeCache()
except:
CacheResult = False
self.feedback_q.put(taskname)
if CacheResult:
self.cache_q.put(
(Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True))
continue
else:
self.cache_q.put(
(Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False))
Ma.CreateCodeFile(False)
Ma.CreateMakeFile(False,
GenFfsList=FfsCmd.get(
(Ma.MetaFile.Path, Ma.Arch), []))
if GlobalData.gBinCacheSource and CommandTarget in [
None, "", "all"
]:
try:
CacheResult = Ma.CanSkipbyMakeCache()
except:
CacheResult = False
self.feedback_q.put(taskname)
if CacheResult:
self.cache_q.put(
(Ma.MetaFile.Path, Ma.Arch, "MakeCache", True))
continue
else:
self.cache_q.put(
(Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))
except Empty:
pass
except:
self.feedback_q.put(taskname)
finally:
self.feedback_q.put("Done")
self.cache_q.put("CacheDone")
def printStatus(self):
print("Processs ID: %d Run %d modules in AutoGen " %
(os.getpid(), len(AutoGen.Cache())))
print("Processs ID: %d Run %d modules in AutoGenInfo " %
(os.getpid(), len(AutoGenInfo.GetCache())))
groupobj = {}
for buildobj in BuildDB.BuildObject.GetCache().values():
if str(buildobj).lower().endswith("dec"):
try:
groupobj['dec'].append(str(buildobj))
except:
groupobj['dec'] = [str(buildobj)]
if str(buildobj).lower().endswith("dsc"):
try:
groupobj['dsc'].append(str(buildobj))
except:
groupobj['dsc'] = [str(buildobj)]
if str(buildobj).lower().endswith("inf"):
try:
groupobj['inf'].append(str(buildobj))
except:
groupobj['inf'] = [str(buildobj)]
print("Processs ID: %d Run %d pkg in WDB " %
(os.getpid(), len(groupobj.get("dec", []))))
print("Processs ID: %d Run %d pla in WDB " %
(os.getpid(), len(groupobj.get("dsc", []))))
print("Processs ID: %d Run %d inf in WDB " %
(os.getpid(), len(groupobj.get("inf", []))))
class AutoGenWorkerInProcess(mp.Process):
def __init__(self, module_queue, data_pipe_file_path, feedback_q,
file_lock, share_data, log_q, error_event):
mp.Process.__init__(self)
self.module_queue = module_queue
self.data_pipe_file_path = data_pipe_file_path
self.data_pipe = None
self.feedback_q = feedback_q
self.PlatformMetaFileSet = {}
self.file_lock = file_lock
self.share_data = share_data
self.log_q = log_q
self.error_event = error_event
def GetPlatformMetaFile(self, filepath, root):
try:
return self.PlatformMetaFileSet[(filepath, root)]
except:
self.PlatformMetaFileSet[(filepath, root)] = filepath
return self.PlatformMetaFileSet[(filepath, root)]
def run(self):
try:
taskname = "Init"
with self.file_lock:
if not os.path.exists(self.data_pipe_file_path):
self.feedback_q.put(taskname + ":" +
"load data pipe %s failed." %
self.data_pipe_file_path)
self.data_pipe = MemoryDataPipe()
self.data_pipe.load(self.data_pipe_file_path)
EdkLogger.LogClientInitialize(self.log_q)
loglevel = self.data_pipe.Get("LogLevel")
if not loglevel:
loglevel = EdkLogger.INFO
EdkLogger.SetLevel(loglevel)
target = self.data_pipe.Get("P_Info").get("Target")
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
archlist = self.data_pipe.Get("P_Info").get("ArchList")
active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")
workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")
PackagesPath = os.getenv("PACKAGES_PATH")
mws.setWs(workspacedir, PackagesPath)
self.Wa = WorkSpaceInfo(workspacedir, active_p, target, toolchain,
archlist)
self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")
GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")
GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")
os.environ._data = self.data_pipe.Get("Env_Var")
GlobalData.gWorkspace = workspacedir
GlobalData.gDisableIncludePathCheck = False
GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
pcd_from_build_option = []
for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
pcd_id = ".".join((pcd_tuple[0], pcd_tuple[1]))
if pcd_tuple[2].strip():
pcd_id = ".".join((pcd_id, pcd_tuple[2]))
pcd_from_build_option.append("=".join((pcd_id, pcd_tuple[3])))
GlobalData.BuildOptionPcd = pcd_from_build_option
module_count = 0
FfsCmd = self.data_pipe.Get("FfsCommand")
if FfsCmd is None:
FfsCmd = {}
PlatformMetaFile = self.GetPlatformMetaFile(
self.data_pipe.Get("P_Info").get("ActivePlatform"),
self.data_pipe.Get("P_Info").get("WorkspaceDir"))
libConstPcd = self.data_pipe.Get("LibConstPcd")
Refes = self.data_pipe.Get("REFS")
while True:
if self.module_queue.empty():
break
if self.error_event.is_set():
break
module_count += 1
module_file, module_root, module_path, module_basename, module_originalpath, module_arch, IsLib = self.module_queue.get_nowait(
)
modulefullpath = os.path.join(module_root, module_file)
taskname = " : ".join((modulefullpath, module_arch))
module_metafile = PathClass(module_file, module_root)
if module_path:
module_metafile.Path = module_path
if module_basename:
module_metafile.BaseName = module_basename
if module_originalpath:
module_metafile.OriginalPath = PathClass(
module_originalpath, module_root)
arch = module_arch
target = self.data_pipe.Get("P_Info").get("Target")
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
Ma = ModuleAutoGen(self.Wa, module_metafile, target, toolchain,
arch, PlatformMetaFile, self.data_pipe)
Ma.IsLibrary = IsLib
if IsLib:
if (Ma.MetaFile.File, Ma.MetaFile.Root, Ma.Arch,
Ma.MetaFile.Path) in libConstPcd:
Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,
Ma.MetaFile.Root, Ma.Arch,
Ma.MetaFile.Path)]
if (Ma.MetaFile.File, Ma.MetaFile.Root, Ma.Arch,
Ma.MetaFile.Path) in Refes:
Ma.ReferenceModules = Refes[(Ma.MetaFile.File,
Ma.MetaFile.Root, Ma.Arch,
Ma.MetaFile.Path)]
Ma.CreateCodeFile(False)
Ma.CreateMakeFile(False,
GenFfsList=FfsCmd.get(
(Ma.MetaFile.File, Ma.Arch), []))
except Empty:
pass
except:
traceback.print_exc(file=sys.stdout)
self.feedback_q.put(taskname)
finally:
self.feedback_q.put("Done")
def printStatus(self):
print("Processs ID: %d Run %d modules in AutoGen " %
(os.getpid(), len(AutoGen.Cache())))
print("Processs ID: %d Run %d modules in AutoGenInfo " %
(os.getpid(), len(AutoGenInfo.GetCache())))
groupobj = {}
for buildobj in BuildDB.BuildObject.GetCache().values():
if str(buildobj).lower().endswith("dec"):
try:
groupobj['dec'].append(str(buildobj))
except:
groupobj['dec'] = [str(buildobj)]
if str(buildobj).lower().endswith("dsc"):
try:
groupobj['dsc'].append(str(buildobj))
except:
groupobj['dsc'] = [str(buildobj)]
if str(buildobj).lower().endswith("inf"):
try:
groupobj['inf'].append(str(buildobj))
except:
groupobj['inf'] = [str(buildobj)]
print("Processs ID: %d Run %d pkg in WDB " %
(os.getpid(), len(groupobj.get("dec", []))))
print("Processs ID: %d Run %d pla in WDB " %
(os.getpid(), len(groupobj.get("dsc", []))))
print("Processs ID: %d Run %d inf in WDB " %
(os.getpid(), len(groupobj.get("inf", []))))