def configure(self):
"""configure the target"""
self.enterBuildDir()
configure = Arguments([self.sourceDir() / (self.subinfo.options.configure.projectFile or "configure")])
self.shell.environment["CFLAGS"] = self.subinfo.options.configure.cflags + " " + self.shell.environment["CFLAGS"]
self.shell.environment["CXXFLAGS"] = self.subinfo.options.configure.cxxflags + " " + self.shell.environment["CXXFLAGS"]
self.shell.environment["LDFLAGS"] = self.subinfo.options.configure.ldflags + " " + self.shell.environment["LDFLAGS"]
self.shell.environment["MAKE"] = self.makeProgram
autogen = self.sourceDir() / "autogen.sh"
if self.subinfo.options.configure.bootstrap and autogen.exists():
self.shell.execute(self.sourceDir(), autogen)
elif self.subinfo.options.configure.autoreconf:
includesArgs = Arguments()
if self.subinfo.options.configure.useDefaultAutoreconfIncludes:
includes = []
for i in [CraftCore.standardDirs.craftRoot() / "dev-utils/cmake/share", CraftCore.standardDirs.locations.data]:
aclocalDir = i / "aclocal"
if aclocalDir.is_dir():
includes += [f"-I{self.shell.toNativePath(aclocalDir)}"]
includesArgs += includes
self.shell.execute(self.sourceDir(), "autoreconf", Arguments(self.subinfo.options.configure.autoreconfArgs) + includesArgs)
with utils.ScopedEnv({"CLICOLOR_FORCE": None}):
return self.shell.execute(self.buildDir(), configure, self.configureOptions(self))
def __init__(self, package):
self._cachedFromParent = {}
self._package = package
_register = self.registerOption
_convert = self._convert
# cachability is handled by the version comparison
_register("version", str, persist=False, compatible=True)
_register("patchLevel", int, persist=False, compatible=True)
_register("branch", str, persist=False)
_register("revision", str, persist=False)
_register("ignored", bool, persist=False, compatible=True)
_register("buildTests", True, persist=False, compatible=True)
_register("buildStatic",bool, persist=False)
_register("buildType", CraftCore.settings.get("Compile", "BuildType"), persist=False, compatible=True) # cachability already handled by cache behaviour
_register("args", Arguments(), persist=False)
_register("featureArguments",Arguments(), persist=False)
settings = UserOptions.instance().settings
if settings.has_section(package.path):
_registered = UserOptions.instance().registeredOptions[package.path]
for key, value in settings[package.path].items():
if key in _registered:
value = _convert(_registered[key].value, value)
setattr(self, key, value)
def __init__(self):
BuildSystemBase.__init__(self, "autotools")
self._shell = BashShell()
self.platform = ""# hope for auto detection
if CraftCore.compiler.isGCC() and not CraftCore.compiler.isNative() and CraftCore.compiler.isX86():
self.platform = Arguments(["--host=i686-pc-linux-gnu"])
elif CraftCore.compiler.isWindows:
if CraftCore.compiler.isX86():
self.platform = Arguments(["--host=i686-w64-mingw32", "--build=i686-w64-mingw32", "--target=i686-w64-mingw32"])
else:
self.platform = Arguments(["--host=x86_64-w64-mingw32", "--build=x86_64-w64-mingw32", "--target=x86_64-w64-mingw32"])
def __init__(self, dynamic):
## with this option additional arguments could be added to the configure commmand line
self.args = Arguments(dynamic.args)
## with this option additional arguments could be added to the configure commmand line (for static builds)
self.staticArgs = Arguments()
## set source subdirectory as source root for the configuration tool.
# Sometimes it is required to take a subdirectory from the source tree as source root
# directory for the configure tool, which could be enabled by this option. The value of
# this option is added to sourceDir() and the result is used as source root directory.
self.configurePath = None
# add the cmake defines that are needed to build tests here
self.testDefine = None
## run autogen in autotools
self.bootstrap = False
## run "autoreconf -vfi" in autotools
self.autoreconf = True
## optional arguments for autoreconf
self.autoreconfArgs = Arguments(["-vfi"])
## Whether to add the default -I flags when running autoreconf
## This is needed since some packages fail if we pass -I to autoreconf
self.useDefaultAutoreconfIncludes = True
# do not use default include path
self.noDefaultInclude = False
## do not use default lib path
self.noDefaultLib = False
## set this attribute in case a non standard configuration
# tool is required (supported currently by QMakeBuildSystem only)
self.tool = False
# cflags currently only used for autotools
self.cflags = ""
# cxxflags currently only used for autotools
self.cxxflags = ""
# ldflags currently only used for autotools
self.ldflags = ""
# the project file, this is either a .pro for qmake or a sln for msbuild
self.projectFile = None
# whether to not pass --datarootdir configure
self.noDataRootDir = False
# whether to not pass --libdir configure
self.noLibDir = False
def execute(self, path, cmd, args="", **kwargs):
# try to locate the command
if CraftCore.compiler.isWindows:
tmp = CraftCore.cache.findApplication(cmd)
if tmp:
cmd = tmp
command = Arguments([
self._findBash(), "-c",
str(Arguments([self.toNativePath(cmd), args]))
])
else:
command = Arguments([cmd, args])
env = dict(os.environ)
env.update(self.environment)
env.update(kwargs.get("env", {}))
return utils.system(command, cwd=path, env=env, **kwargs)
def makeOptions(self, args):
"""return options for make command line"""
defines = Arguments()
if self.subinfo.options.make.ignoreErrors:
defines.append("-i")
makeProgram = self.makeProgram
if makeProgram == "ninja":
if CraftCore.debug.verbose() > 0:
defines.append("-v")
else:
if CraftCore.debug.verbose() > 0:
defines += ["VERBOSE=1", "V=1"]
if self.subinfo.options.make.supportsMultijob and makeProgram != "nmake":
if makeProgram not in {
"ninja", "jom"
} or ("Compile", "Jobs") in CraftCore.settings:
defines += [
"-j",
str(
CraftCore.settings.get("Compile", "Jobs",
multiprocessing.cpu_count()))
]
if args:
defines.append(args)
return defines
def configureOptions(self, defines=""):
"""return options for configure command line"""
defines = Arguments(defines)
defines += self.subinfo.options.configure.args
if self.supportsCCACHE and CraftCore.cache.findApplication("ccache"):
defines += self.ccacheOptions()
if CraftCore.compiler.isClang() and self.supportsClang:
defines += self.clangOptions()
return defines
def configureOptions(self, defines=""):
"""returns default configure options"""
craftRoot = OsUtils.toUnixPath(CraftCore.standardDirs.craftRoot())
options = Arguments([defines])
options += [
"-DBUILD_TESTING={testing}".format(
testing="ON" if self.buildTests else "OFF"),
BuildSystemBase.configureOptions(self),
f"-DCMAKE_INSTALL_PREFIX={craftRoot}",
f"-DCMAKE_PREFIX_PATH={craftRoot}"
]
if self.buildType() is not None:
options.append(f"-DCMAKE_BUILD_TYPE={self.buildType()}")
#if CraftCore.compiler.isGCC() and not CraftCore.compiler.isNative():
# options += " -DCMAKE_TOOLCHAIN_FILE=%s" % os.path.join(CraftStandardDirs.craftRoot(), "craft", "bin", "toolchains", "Toolchain-cross-mingw32-linux-%s.cmake" % CraftCore.compiler.architecture)
if CraftCore.settings.getboolean("CMake", "KDE_L10N_AUTO_TRANSLATIONS",
False):
options.append("-DKDE_L10N_AUTO_TRANSLATIONS=ON")
if CraftCore.compiler.isWindows:
# people use InstallRequiredSystemLibraries.cmake wrong and unconditionally install the
# msvc crt...
options.append("-DCMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP=ON")
elif CraftCore.compiler.isMacOS:
options += [
f"-DKDE_INSTALL_BUNDLEDIR={OsUtils.toUnixPath(CraftCore.standardDirs.craftRoot())}/Applications/KDE",
"-DAPPLE_SUPPRESS_X11_WARNING=ON"
]
elif CraftCore.compiler.isLinux:
# use the same lib dir on all distributions
options += ["-DCMAKE_INSTALL_LIBDIR=lib"]
if CraftCore.compiler.isWindows or CraftCore.compiler.isMacOS:
options.append("-DKDE_INSTALL_USE_QT_SYS_PATHS=ON")
if self.subinfo.options.buildTools:
options += self.subinfo.options.configure.toolsDefine
if self.subinfo.options.buildStatic and self.subinfo.options.configure.staticArgs:
options += self.subinfo.options.configure.staticArgs
if CraftCore.compiler.isIntel():
# this is needed because otherwise it'll detect the MSVC environment
options += " -DCMAKE_CXX_COMPILER=\"%s\" " % os.path.join(
os.getenv("BIN_ROOT"), os.getenv("ARCH_PATH"),
"icl.exe").replace("\\", "/")
options += " -DCMAKE_C_COMPILER=\"%s\" " % os.path.join(
os.getenv("BIN_ROOT"), os.getenv("ARCH_PATH"),
"icl.exe").replace("\\", "/")
options += " -DCMAKE_LINKER=\"%s\" " % os.path.join(
os.getenv("BIN_ROOT"), os.getenv("ARCH_PATH"),
"xilink.exe").replace("\\", "/")
options += ["-S", self.configureSourceDir()]
return options
def main():
#加载日志
logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt = '%m/%d/%Y %H:%M:%S',
level = logging.INFO)
logger = logging.getLogger(__name__)
# 'opt' means options
opt = None
parser = argparse.ArgumentParser(description='multi-qa')
parser.add_argument('mode', help='mode: train')
parser.add_argument('conf_file', help='path to conf file.')
parser.add_argument('dataset', help='dataset for train')
#得到参数实例
args = parser.parse_args()
mode = args.mode # train mode
#得到conf文件的参数
conf_file = args.conf_file
dataset = args.dataset
conf_args = Arguments(conf_file)
opt = conf_args.readArguments()
opt['cuda'] = torch.cuda.is_available()
opt['confFile'] = conf_file
opt['datadir'] = os.path.dirname(conf_file) # .
for key, val in args.__dict__.items():
if val is not None and key not in ['command', 'conf_file']:
opt[key] = val
#使用GPU
# opt['cuda'] = False
device = torch.device("cuda" if opt['cuda'] else 'cpu')
logger.info("device %s is using for training!", device)
model = ConvQA_CN_NetTrainer(opt)
print("Select mode----", mode)
print("Using dataset ----", dataset)
model.train()
def main(conf_file, model_file, data_file, output_path, mask, indicator,
answer_span_in_context, no_ans_bit):
conf_args = Arguments(conf_file)
opt = conf_args.readArguments()
opt['cuda'] = torch.cuda.is_available()
opt['confFile'] = conf_file
opt['datadir'] = os.path.dirname(conf_file)
opt['PREV_ANS_MASK'] = mask
opt['PREV_ANS_INDICATOR'] = indicator
opt['OFFICIAL'] = True
opt['OFFICIAL_TEST_FILE'] = data_file
if answer_span_in_context:
opt['ANSWER_SPAN_IN_CONTEXT_FEATURE'] = None
if no_ans_bit:
opt['NO_PREV_ANS_BIT'] = None
trainer = SDNetTrainer(opt)
test_data = trainer.preproc.preprocess('test')
predictions, confidence, final_json = trainer.official(
model_file, test_data)
with output_path.open(mode='w') as f:
json.dump(final_json, f)
import torch
from Models.SDNetTrainer import SDNetTrainer
from Utils.Arguments import Arguments
if __name__ == "__main__":
# multiprocessing.set_start_method('spawn')
opt = None
parser = argparse.ArgumentParser(description='SDNet')
parser.add_argument('--command', default='train', help='Command: train')
parser.add_argument('--conf_file', default='conf_stvqa', help='Path to conf file.')
parser.add_argument('--log_file', default='', help='Path to log file.')
cmdline_args = parser.parse_args()
command = cmdline_args.command
conf_file = cmdline_args.conf_file
conf_args = Arguments(conf_file)
opt = conf_args.readArguments()
opt['cuda'] = torch.cuda.is_available()
opt['confFile'] = conf_file
opt['datadir'] = os.path.dirname(conf_file) # conf_file specifies where the data folder is
if cmdline_args.log_file != '':
if not os.path.exists('myLog'):
os.makedirs('myLog')
file_handle = logging.FileHandler(os.path.join('myLog', cmdline_args.log_file +'.txt'))
log.addHandler(file_handle)
for key,val in cmdline_args.__dict__.items():
if val is not None and key not in ['command', 'conf_file']:
opt[key] = val
def configure(self):
"""configure the target"""
self.enterBuildDir()
configure = Arguments([
self.sourceDir() /
(self.subinfo.options.configure.projectFile or "configure")
])
self.shell.environment[
"CFLAGS"] = self.subinfo.options.configure.cflags + " " + self.shell.environment[
"CFLAGS"]
self.shell.environment[
"CXXFLAGS"] = self.subinfo.options.configure.cxxflags + " " + self.shell.environment[
"CXXFLAGS"]
self.shell.environment[
"LDFLAGS"] = self.subinfo.options.configure.ldflags + " " + self.shell.environment[
"LDFLAGS"]
self.shell.environment["MAKE"] = self.makeProgram
env = {"CLICOLOR_FORCE": None}
if self.supportsCCACHE:
cxx = CraftCore.standardDirs.craftRoot(
) / "dev-utils/ccache/bin" / Path(os.environ["CXX"]).name
if CraftCore.compiler.isWindows and not cxx.suffix:
cxx = Path(str(cxx) + CraftCore.compiler.executableSuffix)
if cxx.exists():
env["CXX"] = OsUtils.toMSysPath(cxx)
env["CC"] = OsUtils.toMSysPath(cxx.parent /
Path(os.environ["CC"]).name)
with utils.ScopedEnv(env):
autogen = self.sourceDir() / "autogen.sh"
if self.subinfo.options.configure.bootstrap and autogen.exists():
mode = os.stat(autogen).st_mode
if mode & stat.S_IEXEC == 0:
os.chmod(autogen, mode | stat.S_IEXEC)
self.shell.execute(self.sourceDir(), autogen)
elif self.subinfo.options.configure.autoreconf and (
(self.sourceDir() / "configure.ac").exists() or
(self.sourceDir() / "configure.in").exists()):
includesArgs = Arguments()
if self.subinfo.options.configure.useDefaultAutoreconfIncludes:
includes = []
dataDirs = [
CraftCore.standardDirs.craftRoot() /
"dev-utils/cmake/share"
]
if CraftCore.compiler.isWindows:
# on Windows data location lies outside of the autotools prefix (msys)
dataDirs.append(CraftCore.standardDirs.locations.data)
for i in dataDirs:
aclocalDir = i / "aclocal"
if aclocalDir.is_dir():
includes += [
f"-I{self.shell.toNativePath(aclocalDir)}"
]
includesArgs += includes
if not self.shell.execute(
self.sourceDir(), "autoreconf",
Arguments(
self.subinfo.options.configure.autoreconfArgs) +
includesArgs):
return False
return self.shell.execute(self.buildDir(), configure,
self.configureOptions(self))
def configureOptions(self, defines=""):
"""returns default configure options"""
craftRoot = OsUtils.toUnixPath(CraftCore.standardDirs.craftRoot())
options = Arguments([defines])
options += [
"-DBUILD_TESTING={testing}".format(
testing="ON" if self.buildTests else "OFF"),
"-DBUILD_SHARED_LIBS={shared}".format(
shared="OFF" if self.subinfo.options.buildStatic else "ON"),
BuildSystemBase.configureOptions(self),
f"-DCMAKE_INSTALL_PREFIX={craftRoot}",
f"-DCMAKE_PREFIX_PATH={craftRoot}"
]
if self.buildType() is not None:
options.append(f"-DCMAKE_BUILD_TYPE={self.buildType()}")
#if CraftCore.compiler.isGCC() and not CraftCore.compiler.isNative():
# options += " -DCMAKE_TOOLCHAIN_FILE=%s" % os.path.join(CraftStandardDirs.craftRoot(), "craft", "bin", "toolchains", "Toolchain-cross-mingw32-linux-%s.cmake" % CraftCore.compiler.architecture)
if CraftCore.settings.getboolean("CMake", "KDE_L10N_AUTO_TRANSLATIONS",
False):
options.append("-DKDE_L10N_AUTO_TRANSLATIONS=ON")
if CraftCore.compiler.isWindows:
# people use InstallRequiredSystemLibraries.cmake wrong and unconditionally install the
# msvc crt...
options.append("-DCMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP=ON")
elif CraftCore.compiler.isMacOS:
options += [
f"-DKDE_INSTALL_BUNDLEDIR={OsUtils.toUnixPath(CraftCore.standardDirs.craftRoot())}/Applications/KDE",
"-DAPPLE_SUPPRESS_X11_WARNING=ON"
]
elif CraftCore.compiler.isLinux:
# use the same lib dir on all distributions
options += ["-DCMAKE_INSTALL_LIBDIR=lib"]
elif CraftCore.compiler.isAndroid:
nativeToolingRoot = CraftCore.settings.get("General",
"KF5HostToolingRoot",
"/opt/nativetooling")
nativeToolingCMake = CraftCore.settings.get(
"General", "KF5HostToolingCMakePath",
"/opt/nativetooling/lib/x86_64-linux-gnu/cmake/")
additionalFindRoots = ";".join(
filter(None, [
CraftCore.settings.get(
"General", "AndroidAdditionalFindRootPath", ""),
craftRoot
]))
options += [
f"-DCMAKE_TOOLCHAIN_FILE={nativeToolingRoot}/share/ECM/toolchain/Android.cmake",
f"-DECM_ADDITIONAL_FIND_ROOT_PATH='{additionalFindRoots}'",
f"-DKF5_HOST_TOOLING={nativeToolingCMake}",
f"-DANDROID_APK_OUTPUT_DIR={self.packageDestinationDir()}",
f"-DANDROID_FASTLANE_METADATA_OUTPUT_DIR={self.packageDestinationDir()}"
]
self.__autodetectAndroidApkTargets()
if self.androidApkTargets != None and len(
self.androidApkTargets) > 0:
options += [
f"-DQTANDROID_EXPORTED_TARGET={';'.join(self.androidApkTargets)}",
f"-DANDROID_APK_DIR={';'.join(self.androidApkDirs)}"
]
if self.buildType() == "Release" or self.buildType(
) == "MinSizeRel":
options += ["-DANDROIDDEPLOYQT_EXTRA_ARGS=--release"]
if CraftCore.compiler.isWindows or CraftCore.compiler.isMacOS:
options.append("-DKDE_INSTALL_USE_QT_SYS_PATHS=ON")
if self.subinfo.options.buildTools:
options += self.subinfo.options.configure.toolsDefine
if self.subinfo.options.buildStatic and self.subinfo.options.configure.staticArgs:
options += self.subinfo.options.configure.staticArgs
if CraftCore.compiler.isIntel():
# this is needed because otherwise it'll detect the MSVC environment
options += " -DCMAKE_CXX_COMPILER=\"%s\" " % os.path.join(
os.getenv("BIN_ROOT"), os.getenv("ARCH_PATH"),
"icl.exe").replace("\\", "/")
options += " -DCMAKE_C_COMPILER=\"%s\" " % os.path.join(
os.getenv("BIN_ROOT"), os.getenv("ARCH_PATH"),
"icl.exe").replace("\\", "/")
options += " -DCMAKE_LINKER=\"%s\" " % os.path.join(
os.getenv("BIN_ROOT"), os.getenv("ARCH_PATH"),
"xilink.exe").replace("\\", "/")
options += ["-S", self.configureSourceDir()]
return options
def __init__(self):
## options passed to make on install
self.args = Arguments(["install"])
def __init__(self):
## ignore make error
self.ignoreErrors = None
## options for the make tool
self.args = Arguments()
self.supportsMultijob = True