def unpack( self ):
svnpath = self.kdeSvnPath()
utils.cleanDirectory( self.workdir )
if svnpath:
if not self.kdeSvnUnpack( svnpath, "scripts" ):
return False
for pkg in self.subinfo.languages.split():
if not self.kdeSvnUnpack( svnpath, pkg ):
return False
else:
print "Unpacked %s" % pkg
autogen = os.path.join( self.packagedir , "autogen.py" )
svnpath = os.path.join( self.kdesvndir, svnpath )
# execute autogen.py and generate the CMakeLists.txt files
cmd = "cd %s && python %s %s" % \
(svnpath , autogen, self.subinfo.languages )
utils.system( cmd )
else:
filenames = []
for pkg in self.subinfo.languages.split():
if not self.subinfo.buildTarget in self.subinfo.targets.keys():
return False
tgt = self.subinfo.buildTarget
filenames.append( 'kde-l10n-' + pkg + '-' + tgt + '.tar.bz2' )
if not utils.unpackFiles( self.downloaddir, filenames, self.workdir ):
return False
# no need for autogen.py - CMake scripts are already created
return True
def run_once(self):
# check if the screensaver process is running and alive
try:
utils.system('export DISPLAY=:0.0 && xscreensaver-command -version')
except error.CmdError, e:
logging.debug(e)
raise error.TestFail('xscreensaver is not alive')
def run_once(self, binary = '/opt/google/chrome/chrome'):
# Try to kill all running instances of the binary.
try:
utils.system('pkill -KILL -f %s' % binary)
except error.CmdError, e:
logging.debug(e)
raise error.TestFail('%s is not running before kill' % binary)
def failure_epilogue (self):
if expected:
print("Could not stat {} - was expected to be present".format(rootfs))
else:
print("Sliver rootfs {} still present - this is unexpected".format(rootfs))
utils.system(self.test_ssh.actual_command("ls -l {rootfs}; du -hs {rootfs}".format(**locals()),
dry_run=self.dry_run, timeout=20))
def unpack( self ):
svnpath = self.kdeSvnPath()
utils.cleanDirectory( self.workdir )
if svnpath:
if not self.kdeSvnUnpack( svnpath, "scripts" ):
return False
for pkg in self.subinfo.languages.split():
if not self.kdeSvnUnpack( svnpath, pkg ):
return False
autogen = os.path.join( self.packagedir , "autogen.py" )
svnpath = os.path.join( self.kdesvndir, svnpath )
# execute autogen.py and generate the CMakeLists.txt files
cmd = [ "python", autogen, self.subinfo.languages ]
# FIXME comment out once libxslt works again
cmd.append( "--disable-docs" )
utils.system( cmd )
else:
filenames = []
for pkg in self.subinfo.languages.split():
if not self.subinfo.buildTarget in self.subinfo.targets.keys():
return False
tgt = self.subinfo.buildTarget
filenames.append( 'kde-l10n-' + pkg + '-' + tgt + '.tar.bz2' )
if not utils.unpackFiles( self.downloaddir, filenames, self.workdir ):
return False
# no need for autogen.py - CMake scripts are already created
return True
def dld_genome_data( fname, url_prefix='ftp://ftp.ncbi.nih.gov/genomes/Bacteria/' ):
bname = os.path.basename( fname ) ## e.g. NC_002937
try:
os.makedirs( 'GENOMES' )
except:
pass
if not os.path.isfile( 'GENOMES/' + bname + '.gff' ):
try:
os.remove('GENOMES/' + bname + '.gff'); os.remove('GENOMES/' + bname + '.fna');
except:
pass
os.chdir( 'GENOMES' )
cmd = "wget '" + url_prefix + fname + ".gff'"
tmp = ut.system( cmd )
cmd = "wget '" + url_prefix + fname + ".fna'"
tmp = ut.system( cmd )
myfile = open( bname + '.fna', 'r' )
line = myfile.readline()
myfile.close()
cmd = "../rpl '" + line.strip() + "' '>" + line.split('|')[3] + "' " + bname + '.fna'
tmp = ut.system( cmd )
ut.system( "awk '($3==\"gene\"){print}' " + bname + ".gff > " + bname + "_genes.gff" )
os.chdir( '..' )
def qemu_local_init(self):
"all nodes : init a clean local directory for holding node-dep stuff like iso image..."
utils.system("rm -rf {}".format(self.nodedir()))
utils.system("mkdir {}".format(self.nodedir()))
if not self.is_qemu():
return True
return utils.system("rsync -v -a --exclude .svn template-qemu/ {}/"\
.format(self.nodedir())) == 0
def make( self ):
build, bt, toolsetSwitches = self.getBuildSettings();
return utils.system("msbuild /target:lib_mpir_gc \"%s\" /p:Configuration=%s %s" %
(os.path.join(self.sourceDir(), build, "mpir.sln"), bt, toolsetSwitches)
) and utils.system("msbuild /target:dll_mpir_gc \"%s\" /p:Configuration=%s %s" %
(os.path.join(self.sourceDir(), build, "mpir.sln"), bt, toolsetSwitches)
)
def compile( self ):
if self.compiler == "mingw":
compiler = "mingw32"
else:
compiler = "do_ms"
workdir = os.path.join( self.workdir, self.instsrcdir )
cmd = 'cd %s && ms\\%s' % ( workdir, compiler )
print cmd
utils.system( cmd ) or utils.die( "failed on command: %s" % cmd )
return True
def compile( self ):
cmd = 'cd %s && python configure.py -b %s -w' % ( os.path.join( self.workdir, self.instsrcdir ),
os.path.join( self.imagedir, "bin" ) )
if self.compiler == "mingw":
cmd += " -p win32-g++"
utils.system( cmd ) or utils.die( "failed to execute %s" % cmd )
sedcommand = r""" -e "s/""" + os.path.join( self.rootdir, "tmp", "qt" ).replace('\\', '\\/') + "*\\/image-mingw\\/" + """/""" + self.rootdir.replace('\\', '\\/') + """/g" """
utils.sedFile( os.path.join( self.workdir, self.instsrcdir ), "pyqtconfig.py", sedcommand )
cmd = 'cd %s && %s' % ( os.path.join( self.workdir, self.instsrcdir ), self.cmakeMakeProgramm )
utils.system( cmd ) or utils.die( "failed to execute %s" % cmd )
return True
def __init__( self, parent=None ):
self.parent = self
if parent:
self.parent = parent
self.output = False
self.outFile = None
self.graphVizInstallPath = None
if not self.isInstalled():
utils.system("emerge.bat graphviz")
if not self.isInstalled():
utils.die("could not find installed graphviz package, you may download and install it from http://www.graphviz.org/Download.php")
def install( self ):
if self.isTargetBuild():
# Configuring Qt for WinCE ignores the -prefix option,
# so we have to do the job manually...
# delete this because it is not working for windows
utils.deleteFile( os.path.join( self.buildDir(), "plugin", "bearer", "qnmbearerd4.dll" ))
utils.deleteFile( os.path.join( self.buildDir(), "plugin", "bearer", "qnmbearer4.dll" ))
# syncqt expects qconfig.h to be in the install dir and fails if not
utils.createDir( os.path.join( self.installDir(), "src", "corelib", "global") )
utils.copyFile( os.path.join( self.buildDir(), "src", "corelib", "global", "qconfig.h" ), os.path.join( self.installDir(), "src", "corelib" , "global", "qconfig.h" ) )
# headers need to be copied using syncqt because of the relative paths
utils.prependPath(self.sourceDir(), "bin")
command = os.path.join(self.sourceDir(), "bin", "syncqt.bat")
command += " -base-dir \"" + self.sourceDir() + "\""
command += " -outdir \"" + self.installDir() + "\""
command += " -copy"
command += " -quiet"
utils.system( command )
utils.copySrcDirToDestDir( os.path.join( self.buildDir(), "bin" ) , os.path.join( self.installDir(), "bin" ) )
utils.copySrcDirToDestDir( os.path.join( self.buildDir(), "lib" ) , os.path.join( self.installDir(), "lib" ) )
# the dlls must be copied to the bin dir too
for file in os.listdir( os.path.join( self.installDir(), "lib" ) ):
if file.endswith( ".dll" ):
utils.copyFile( os.path.join( self.installDir(), "lib" , file ), os.path.join( self.installDir(), "bin" , file ) )
utils.copySrcDirToDestDir( os.path.join( self.buildDir(), "mkspecs" ) , os.path.join( self.installDir(), "mkspecs" ) )
utils.copySrcDirToDestDir( os.path.join( self.buildDir(), "plugins" ) , os.path.join( self.installDir(), "plugins" ) )
# create qt.conf
utils.copyFile( os.path.join( self.packageDir(), "qt.conf" ), os.path.join( self.installDir(), "bin", "qt.conf" ) )
return True
if not QMakeBuildSystem.install(self):
return False
# Workaround QTBUG-12034
utils.copySrcDirToDestDir( os.path.join( self.buildDir(), "plugins", "imageformats" ) ,
os.path.join( self.installDir(), "bin", "imageformats" ) )
# create qt.conf
utils.copyFile( os.path.join( self.packageDir(), "qt.conf" ), os.path.join( self.installDir(), "bin", "qt.conf" ) )
# install msvc debug files if available
if self.buildType() == "Debug" and (self.compiler() == "msvc2005" or self.compiler() == "msvc2008" or self.compiler() == "msvc2010"):
srcdir = os.path.join( self.buildDir(), "lib" )
destdir = os.path.join( self.installDir(), "lib" )
filelist = os.listdir( srcdir )
for file in filelist:
if file.endswith( ".pdb" ):
utils.copyFile( os.path.join( srcdir, file ), os.path.join( destdir, file ) )
return True
def some_zeroed_input_file(self, prefix, mbytes):
name = os.path.join(self.workdir,
'%s%d' % (prefix, self.input_file_count))
self.input_file_count += 1
# TODO: use actual disk file size, avoid rebuilding across iterations
old_mbytes = self.existing_input_files.get(name, 0)
if mbytes > old_mbytes:
cmd = ('/bin/dd if=/dev/zero of=%s bs=1M seek=%d count=%d'
% (name, old_mbytes, mbytes-old_mbytes))
utils.system(cmd)
self.existing_input_files[name] = mbytes
return name
def patch():
os.chdir(utils.mojo_root_dir)
utils.filter_file("build/landmines.py", lambda line: not "gyp" in line)
utils.commit("filter gyp out of build/landmines.py")
utils.filter_file("gpu/BUILD.gn", lambda line: not "//gpu/ipc" in line)
utils.commit("filter gpu/ipc out of gpu/BUILD.gn")
for p in utils.find(["*.patch"], os.path.dirname(os.path.realpath(__file__))):
print "applying patch %s" % os.path.basename(p)
utils.system(["git", "apply", p])
utils.commit("applied patch %s" % os.path.basename(p))
def make_package( self ):
self.svnpath = os.path.join( self.kdesvndir, self.subinfo.svnTargets['svnHEAD'] )
dstpath = self.packageDestinationDir(withBuildType=False)
if not utils.test4application( "kdewin-packager" ):
utils.die( "kdewin-packager not found - please make sure it is in your path" )
for pkg in self.subinfo.languages.split():
workdir = os.path.join( self.workdir, pkg )
cmd = "kdewin-packager -name kde-l10n-%s -version %s -hashfirst -compression 2 -root %s/%s -destdir %s" % \
( pkg, self.buildTarget, self.imagedir, pkg, dstpath )
utils.system( cmd )
return True
def kill_slower_workers(fast_pid, cpu_cgroup, pids_file):
moved_pids_file = pids_file + '.moved'
try:
os.rename(pids_file, moved_pids_file)
except OSError:
return
logging.debug('fastest worker pid %d of container %s'
' killing all slower workers',
fast_pid, cpu_cgroup.path)
for line in open(moved_pids_file):
pid = int(line.rstrip())
if pid != fast_pid:
utils.system('kill %d' % pid, ignore_status=True)
def install(self):
srcdir = os.path.join(self.workdir, os.getenv("KDECOMPILER") + "-" + self.buildType, "wincursors")
destdir = os.path.join(self.cmakeInstallPrefix, "share", "icons")
utils.copySrcDirToDestDir(srcdir, destdir)
srcreg = os.path.join(self.packagedir, "cursor.reg")
dstreg = os.path.join(self.workdir, "cursor.reg")
shutil.copy(srcreg, dstreg)
utils.sedFile(
self.workdir, "cursor.reg", "s/CHANGEME/%s/g" % destdir.replace("\\", "\\\\\\\\").replace("/", "\\\\\\\\")
)
utils.system("regedit /S %s" % os.path.join(self.workdir, "cursor.reg"))
return True
def sedFile(self, directory, fileName, sedcommand ):
""" runs the given sed command on the given file """
olddir = os.getcwd()
try:
os.chdir( directory )
backup = "%s.orig" % fileName
if( os.path.isfile( backup ) ):
os.remove( backup )
command = "sed -i.orig %s %s" % ( sedcommand, fileName )
utils.system( command )
finally:
os.chdir( olddir )
def configure( self, buildType=None, defines=""):
thirdparty_dir = os.path.join( self.buildDir(), "3rdparty" )
utils.prependPath(self.buildDir(), "bin")
configure = os.path.join( self.sourceDir(), "configure.exe" ).replace( "/", "\\" )
self.enterBuildDir()
# so that the mkspecs can be found, when -prefix is set
os.putenv( "QMAKEPATH", self.sourceDir() )
# configure qt
# prefix = os.path.join( self.rootdir, "qt" ).replace( "\\", "/" )
prefix = self.installDir()
platform = ""
libtmp = os.getenv( "LIB" )
inctmp = os.getenv( "INCLUDE" )
if self.compiler() == "msvc2005" or self.compiler() == "msvc2008":
platform = "win32-%s" % self.compiler()
elif self.compiler() == "mingw":
os.environ[ "LIB" ] = ""
os.environ[ "INCLUDE" ] = ""
platform = "win32-g++"
else:
exit( 1 )
os.environ[ "USERIN" ] = "y"
userin = "y"
command = r"echo %s | %s -opensource -platform %s -prefix %s " \
"-qt-gif -qt-libpng -qt-libjpeg -qt-libtiff -plugin-sql-mysql " \
"-no-phonon -qdbus -openssl -dbus-linked " \
"-fast -no-vcproj -no-dsp -plugin-sql-mysql " \
"-nomake demos -nomake examples -nomake docs " \
"-I \"%s\" -L \"%s\" " % \
( userin, configure, platform, prefix,
os.path.join( thirdparty_dir, "include" ),
os.path.join( thirdparty_dir, "lib" ) )
if self.buildType() == "Debug":
command = command + " -debug "
else:
command = command + " -release "
print "command: ", command
utils.system( command )
if( not libtmp == None ):
os.environ[ "LIB" ] = libtmp
if( not inctmp == None ):
os.environ[ "INCLUDE" ] = inctmp
return True
def unittest( self ):
os.chdir( os.path.join( self.sourceDir(), 'build.vc12') )
if self.buildType() == "Debug":
bt = "Debug"
else:
bt = "Release"
toolsetSwitches = ""
if compiler.isMSVC2012():
toolsetSwitches = "/property:PlatformToolset=v110"
elif compiler.isMSVC2013():
toolsetSwitches = "/tv:12.0 /property:PlatformToolset=v120"
return utils.system("msbuild \"%s\" /p:Configuration=%s %s" %
(os.path.join(self.sourceDir(), "build.vc12", "mpir-tests.sln"), bt, toolsetSwitches)
) and utils.system(os.path.join("mpir-tests", "run-tests.py"));
def do_ssh_slice_once(self, options, command, expected):
# locate a key
private_key=self.locate_private_key()
if not private_key :
utils.header("WARNING: Cannot find a valid key for slice {}".format(self.name()))
return False
# convert nodenames to real hostnames
slice_spec = self.slice_spec
nodename=slice_spec['nodenames'][0]
site_spec, node_spec = self.test_plc.locate_node(nodename)
hostname=node_spec['node_fields']['hostname']
if expected:
msg="{} to return TRUE from ssh".format(command)
else:
msg="{} to return FALSE from ssh".format(command)
utils.header("checking {} -- slice {} on node {}".format(msg, self.name(), hostname))
site_spec, node_spec = self.test_plc.locate_hostname(hostname)
test_ssh = TestSsh (hostname, key=private_key, username=self.name())
full_command = test_ssh.actual_command(command)
retcod = utils.system (full_command, silent=True, timeout=10)
if getattr(options, 'dry_run', None):
return True
if expected:
success = retcod==0
else:
success = retcod!=0
if not success:
utils.header ("WRONG RESULT for {}".format(msg))
return success
def msysCompile( self, bOutOfSource = True ):
"""run configure and make for Autotools based stuff"""
config = os.path.join( self.workdir, self.instsrcdir, "configure" )
build = os.path.join( self.workdir )
if( bOutOfSource ):
# otherwise $srcdir is very long and a conftest may fail (like it's the
# case in libgmp-4.2.4)
config = os.path.join( "..", self.instsrcdir, "configure" )
build = os.path.join( build, self.instsrcdir + "-build" )
utils.cleanDirectory( build )
else:
build = os.path.join( build, self.instsrcdir )
sh = os.path.join( self.msysdir, "bin", "sh.exe" )
# todo use msysexecute
cmd = "%s --login -c \"cd %s && %s %s && make -j2" % \
( sh, self.__toMSysPath( build ), self.__toMSysPath( config ), \
self.msysConfigureFlags() )
if utils.verbose() > 1:
cmd += " VERBOSE=1"
cmd += "\""
if utils.verbose() > 0:
print "msys compile: %s" % cmd
if not utils.system(cmd):
utils.die( "msys compile failed. cmd: %s" % cmd )
return True
def run(self, silent):
command = self.test_ssh.actual_command(self.command)
retcod = utils.system(command, silent=silent, dry_run=self.dry_run)
if self.expected:
return retcod == 0
else:
return retcod != 0
def createPortablePackage( self ):
"""create portable 7z package with digest files located in the manifest subdir"""
if not self.packagerExe:
utils.die("could not find 7za in your path!")
if self.package.endswith( "-package" ):
shortPackage = self.package[ : -8 ]
else:
shortPackage = self.package
if not "setupname" in self.defines or not self.defines[ "setupname" ]:
self.defines[ "setupname" ] = shortPackage
if self.subinfo.options.package.withArchitecture:
self.defines[ "setupname" ] += "-" + os.getenv("EMERGE_ARCHITECTURE")
self.defines[ "setupname" ] += "-" + self.buildTarget + ".7z"
if not "srcdir" in self.defines or not self.defines[ "srcdir" ]:
self.defines[ "srcdir" ] = self.imageDir()
for f in self.scriptnames:
utils.copyFile(f,os.path.join(self.defines[ "srcdir" ],os.path.split(f)[1]))
# make absolute path for output file
if not os.path.isabs( self.defines[ "setupname" ] ):
dstpath = self.packageDestinationDir()
self.defines[ "setupname" ] = os.path.join( dstpath, self.defines[ "setupname" ] )
utils.deleteFile(self.defines[ "setupname" ])
cmd = "cd %s && %s a -r %s %s" % (self.defines[ "srcdir" ], self.packagerExe,self.defines[ "setupname" ], '*')
if not utils.system(cmd):
utils.die( "while packaging. cmd: %s" % cmd )
def make(self):
self.enterSourceDir()
if self.buildType() == "Debug":
bt = "Debug"
else:
bt = "Release"
datafile = os.path.join(os.path.dirname(os.path.abspath(__file__)), "icudt55l.dat")
if os.path.exists(datafile):
datafileDestination = os.path.join(self.sourceDir(), "data", "in", "icudt55l.dat")
if os.path.exists(datafileDestination):
os.remove(datafileDestination)
utils.copyFile( datafile, datafileDestination)
toolsetSwitches = ""
if compiler.isMSVC2012():
toolsetSwitches = "/property:PlatformToolset=v110"
elif compiler.isMSVC2013():
toolsetSwitches = "/tv:12.0 /property:PlatformToolset=v120"
elif compiler.isMSVC2015():
toolsetSwitches = "/tv:14.0 /property:PlatformToolset=v140"
return utils.system("msbuild /m /t:Rebuild \"%s\" /p:Configuration=%s %s" %
(os.path.join(self.sourceDir(), "allinone", "allinone.sln"), bt, toolsetSwitches)
)
def kdeConfigureInternal( self, buildType, kdeCustomDefines ):
"""Using cmake"""
builddir = "%s" % ( self.COMPILER )
if( not buildType == None ):
buildtype = "-DCMAKE_BUILD_TYPE=%s" % buildType
builddir = "%s-%s" % ( builddir, buildType )
if( not self.buildNameExt == None ):
builddir = "%s-%s" % ( builddir, self.buildNameExt )
os.chdir( self.workdir )
if ( not os.path.exists( builddir) ):
os.mkdir( builddir )
if not self.noClean:
utils.cleanDirectory( builddir )
os.chdir( builddir )
command = r"""cmake -G "%s" %s %s %s""" % \
( self.cmakeMakefileGenerator, \
self.kdeDefaultDefines(), \
kdeCustomDefines, \
buildtype )
if utils.verbose() > 0:
print "configuration command: %s" % command
if not utils.system(command):
utils.die( "while CMake'ing. cmd: %s" % command )
return True
def run(self, command, message=None, background=False, dry_run=False):
local_command = self.actual_command(command, dry_run=dry_run)
if dry_run:
utils.header("DRY RUN " + local_command)
return 0
else:
self.header(message)
return utils.system(local_command, background)
def qmerge(self):
if not BinaryPackageBase.qmerge(self):
return False
tmpFile = tempfile.TemporaryFile()
git = os.path.join(self.rootdir,"dev-utils","git","bin","git")
utils.system( "%s config --global --get url.git://anongit.kde.org/.insteadof" % git,
stdout=tmpFile, stderr=tmpFile )
tmpFile.seek( 0 )
for line in tmpFile:
if str(line,'UTF-8').find("kde:")>-1:
return True
utils.debug( "adding kde related settings to global git config file",1 )
utils.system( "%s config --global url.git://anongit.kde.org/.insteadOf kde:" % git)
utils.system( "%s config --global url.ssh://[email protected]/.pushInsteadOf kde:" % git)
utils.system( "%s config --global core.autocrlf false" % git)
utils.system( "%s config --system core.autocrlf false" % git)
return True
def unpack( self ):
# print "gpgme-qt unpack called"
# do the svn fetch/update
repo = "svn://cvs.gnupg.org/gpgme/trunk/gpgme"
utils.cleanDirectory( self.workdir )
self.svnFetch( repo )
self.kdeSvnUnpack()
srcdir = os.path.join( self.svndir )
destdir = os.path.join( self.workdir, "kdepimlibs", "gpgme-qt" )
utils.copySrcDirToDestDir( srcdir, destdir )
os.chdir( self.workdir )
utils.system( "cd %s && patch -p0 < %s" % ( self.workdir, os.path.join( self.packagedir, "gpgme-qt.patch" ) ) )
return True
def configure( self ):
self.enterSourceDir()
cmd = "python configure.py"
cmd += self.subinfo.options.configure.defines
cmd += " --bindir %s/bin " % self.installDir()
cmd += " --destdir %s/Lib/site-packages " % self.installDir()
cmd += " --plugin-destdir %s/plugins " % self.installDir()
sipcfg = sipconfig.Configuration()
sipdir = os.path.splitdrive(sipcfg.default_sip_dir)[1]
sipdir = os.path.join(sipdir,'PyQt4')
cmd += " --sipdir %s%s " % (self.installDir(), sipdir)
utils.system(cmd) or utils.die("command: %s failed" % (cmd))
return True
def curlFile(url, destdir, filename, quiet):
"""download file with curl from 'url' into 'destdir', if filename is given to the file specified"""
curl = CraftCore.cache.findApplication("curl")
command = [curl, "-C", "-", "--retry", "10", "-L", "--ftp-ssl", "--fail"]
cert = os.path.join(CraftCore.standardDirs.etcDir(), "cacert.pem")
if os.path.exists(cert):
command += ["--cacert", cert]
# the default of 20 might not be enough for sourceforge ...
command += ["--max-redirs", "50"]
command += ["-o", os.path.join(destdir, filename)]
command += [url]
CraftCore.log.debug("curlfile called")
if CraftCore.debug.verbose() < 1:
if quiet:
with io.StringIO() as tmp:
ciMode = CraftCore.settings.getboolean("ContinuousIntegration",
"Enabled", False)
if ciMode:
command += ["-v"]
if not utils.system(command,
logCommand=ciMode,
stdout=tmp,
stderr=subprocess.STDOUT):
CraftCore.log.warning(tmp.getvalue())
return False
if ciMode:
loc = re.findall(r"Host: ([^\s]+)", tmp.getvalue())
if loc:
CraftCore.log.info(f"Downloaded from: {loc[-1]}")
return True
elif CraftCore.cache.checkCommandOutputFor(curl,
"--progress-bar",
helpCommand="--help all"):
command += ["--progress-bar"]
CraftCore.log.info(f"curl {url}")
return utils.system(command,
displayProgress=True,
logCommand=False,
stderr=subprocess.STDOUT)
command += ["-v"]
return utils.system(command)
def get_public_key():
"""
Return a valid string ssh public key for the user executing autoserv or
autotest. If there's no DSA or RSA public key, create a DSA keypair with
ssh-keygen and return it.
:returns: a ssh public key
:rtype: str
"""
ssh_conf_path = os.path.expanduser('~/.ssh')
dsa_public_key_path = os.path.join(ssh_conf_path, 'id_dsa.pub')
dsa_private_key_path = os.path.join(ssh_conf_path, 'id_dsa')
rsa_public_key_path = os.path.join(ssh_conf_path, 'id_rsa.pub')
rsa_private_key_path = os.path.join(ssh_conf_path, 'id_rsa')
has_dsa_keypair = (os.path.isfile(dsa_public_key_path)
and os.path.isfile(dsa_private_key_path))
has_rsa_keypair = (os.path.isfile(rsa_public_key_path)
and os.path.isfile(rsa_private_key_path))
if has_dsa_keypair:
logging.info('DSA keypair found, using it')
public_key_path = dsa_public_key_path
elif has_rsa_keypair:
logging.info('RSA keypair found, using it')
public_key_path = rsa_public_key_path
else:
logging.info('Neither RSA nor DSA keypair found, '
'creating DSA ssh key pair')
utils.system('ssh-keygen -t dsa -q -N "" -f %s' % dsa_private_key_path)
public_key_path = dsa_public_key_path
public_key = open(public_key_path, 'r')
public_key_str = public_key.read()
public_key.close()
return public_key_str
def run_once(self):
cpuType = utils.get_cpu_arch()
logging.debug("cpu type is %s" % cpuType)
# test return values
result = utils.system("rootdev -s")
logging.debug("Rootdev test res: %d", result)
if (result != 0):
raise error.TestFail("Rootdev failed")
result = utils.system("rootdev -s -d")
logging.debug("Rootdev test -d switch res: %d", result)
if (result != 0):
raise error.TestFail("Rootdev failed -s -d")
# test with -d Results should be without the partition device number
text = utils.system_output("rootdev -s -d 2>&1")
text = text.strip()
logging.debug("Rootdev -s -d txt is *%s*", text)
self.test_no_partition(text, cpuType)
def importCert(cert, pwKey):
pw = CraftChoicePrompt.promptForPassword(
message=
f"Enter the password for certificate: {Path(cert).name}",
key=pwKey)
return utils.system([
"security", "import", cert, "-k", self.loginKeychain, "-P",
pw, "-T", "/usr/bin/codesign", "-T", "/usr/bin/productsign"
],
stdout=subprocess.DEVNULL,
secret=[password])
def check_hash(path, opti, arch, comp):
cmd = "find {0} -type f -executable".format(path)
cmd += ' -name "*{0}_{1}_{2}*" '.format(comp, arch, opti)
cmd += "| sort | xargs md5sum"
lines = system(cmd).splitlines()
hashes = defaultdict(list)
for line in lines:
h, f = line.split()
hashes[h].append(f)
return hashes
def run_worker(cmd, cpu_cgroup, blkio_cgroup, pids_file):
# main of new process for running an independent worker shell
logging.debug('Worker running command: %s' % cmd)
logging.debug('Moving to cpu_cgroup: %s' % cpu_cgroup.path)
logging.debug('Moving to blkio_cgroup: %s' % blkio_cgroup.path)
cpu_cgroup.move_my_task_here()
blkio_cgroup.move_my_task_here()
p = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True)
if pids_file:
utils.system('echo %d >> %s' % (p.pid, pids_file))
logging.debug('running "%s" in container %s and io cgroup %s as pid %d',
cmd, cpu_cgroup.path, blkio_cgroup.path, p.pid)
p.wait()
if pids_file:
kill_slower_workers(p.pid, cpu_cgroup, pids_file)
logging.debug(p.stdout.read())
def next(self, c, f, data):
if data['setup'] == 'raid1':
self.enamel.setup['disktype'] = 'raid'
if data['setup'] == 'noraid':
self.enamel.setup['disktype'] = 'single'
def ret(*_):
return url.root.child('DiskSelect')
# Stop any existing raid devices (interferes with stuff)
return utils.system('mdadm --stop --scan').addBoth(ret)
def roll(target_version):
mojoms_gs_path = "gs://mojo/network/%s/mojoms.zip" % (target_version, )
network_service_path = os.path.join(mojo_root_dir, "mojo", "services",
"network")
mojoms_path = os.path.join(network_service_path, "public", "interfaces")
version_path = os.path.join(network_service_path, "VERSION")
with tempfile.NamedTemporaryFile() as temp_zip_file:
gs.download_from_public_bucket(mojoms_gs_path, temp_zip_file.name)
with zipfile.ZipFile(temp_zip_file.name) as z:
z.extractall(mojoms_path)
with open(version_path, 'w') as stamp_file:
stamp_file.write(target_version)
system(["git", "add", "public"], cwd=network_service_path)
system(["git", "add", "VERSION"], cwd=network_service_path)
chromium_rev = chromium_rev_number(target_version)
commit("Roll the network service to https://crrev.com/" + chromium_rev,
cwd=mojo_root_dir)
return 0
def createPackage(self):
""" create a package """
CraftCore.log.debug("packaging using the MacDMGPackager")
defines = self.setDefaults(self.defines)
if not "pkgproj" in defines:
CraftCore.log.error(
"Cannot not create .pkg because no .pkgproj was defined.")
return False
if not self.internalCreatePackage(
defines, seperateSymbolFiles=True, packageSymbols=True):
return False
packageDest = Path(defines["setupname"])
if packageDest.exists():
utils.deleteFile(packageDest)
pkgprojPath = defines["pkgproj"]
# set output file basename
packagesutil = CraftCore.cache.findApplication("packagesutil")
if not utils.system([
packagesutil, '--file', pkgprojPath, 'set', 'project', 'name',
packageDest.stem
]):
return False
packagesbuild = CraftCore.cache.findApplication("packagesbuild")
if not utils.system([
packagesbuild, "-v", '--reference-folder',
os.path.dirname(self.getMacAppPath(defines)), '--build-folder',
packageDest.parent, pkgprojPath
]):
return False
if not CodeSign.signMacPackage(packageDest):
return False
CraftHash.createDigestFiles(packageDest)
return True
def _test_udev_report_atmel(self):
"""Test that atmel trackpad failure can trigger udev crash report."""
DRIVER_DIR = '/sys/bus/i2c/drivers/atmel_mxt_ts'
has_atmel_device = False
if os.path.exists(DRIVER_DIR):
for filename in os.listdir(DRIVER_DIR):
if os.path.isdir(os.path.join(DRIVER_DIR, filename)):
has_atmel_device = True
if not has_atmel_device:
logging.info('No atmel device, skip the test')
return None
self._set_consent(True)
# Use udevadm to trigger a fake udev event representing atmel driver
# failure. The uevent match rule in 99-crash-reporter.rules is
# ACTION=="change", SUBSYSTEM=="i2c", DRIVER=="atmel_mxt_ts",
# ENV{ERROR}=="1" RUN+="/sbin/crash_reporter
# --udev=SUBSYSTEM=i2c-atmel_mxt_ts:ACTION=change"
utils.system('udevadm control --property=ERROR=1', ignore_status=True)
utils.system(
'udevadm trigger '
'--action=change '
'--subsystem-match=i2c '
'--attr-match=driver=atmel_mxt_ts',
ignore_status=True)
utils.system('udevadm control --property=ERROR=0', ignore_status=True)
utils.poll_for_condition(
self.CheckAtmelCrashes,
timeout=60,
exception=error.TestFail('No valid Atmel crash reports'))
def exec_stop(self):
fwmark = config.global_config["transproxy"]["fwmark"]
ip_route_table = config.global_config["transproxy"]["ip_route_table"]
self.iptables.del_all()
utils.system("ip route del local default dev lo table %s" %
ip_route_table)
utils.system("ip rule del fwmark %s lookup %s" %
(fwmark, ip_route_table))
self.stopped = True
if not self.process:
utils.log(
"Not killing ss-redir process, since self.process is None")
elif type(self.process.returncode) != int:
utils.log("Killing ss-redir process")
self.kill()
else:
utils.log(
"Not killing ss-redir process, since self.process.returncode is int"
)
def updateSharedMimeInfo(package) -> bool:
if OsUtils.isWin():
dataDir = os.path.join("bin", "data", "mime")
else:
dataDir = os.path.join("share", "mime")
# only check in imageDir, if installDir differs from imageDir it is irrelevant for us
if not os.path.isdir(os.path.join(package.imageDir(), dataDir)):
return True
dataDir = os.path.join(CraftCore.standardDirs.craftRoot(), dataDir)
flags = []
if CraftCore.debug.verbose() > 0:
flags += ["-V"]
return utils.system(["update-mime-database"] + flags + [dataDir])
def form_singleConfig(self, data):
def gotDisks(disks):
print "HERE"
print disks
f = form.Form()
myDisks = {}
# Split on new lines, and remove the * that signifies bootable partitions
for n in disks.replace('*','').split('\n'):
if not n.strip():
continue
parts = n.split()
device = parts[0].strip('/dev/')
cyls = int(parts[3].strip('-').strip('+'))
myDisks[device] = cyls
self.disks = []
cnt = 0
for dev, cyls in myDisks.items():
cnt += 1
size = (int(cyls) * 8225280)/(1024*1024)
self.disks.append(dev)
f.addField(dev, form.String(),
form.widgetFactory(form.SelectChoice,
options=(
('/', '/'),
('/var', '/var'),
('/storage', '/storage')
)
),
label = [tags.strong[dev], " (%sMB)" % size]
)
if cnt == 1:
f.data[dev] = '/'
if cnt ==2:
f.data[dev] = '/var'
if cnt ==3:
f.data[dev] = '/storage'
# If more than one drive exists..
f.addAction(self.nextSingle)
return f
return utils.system('sfdisk -R -l 2>&1| grep "^/dev/" | grep "Linux$"').addBoth(gotDisks)
def execute(self, path, cmd, args="", **kwargs):
# try to locate the command
tmp = CraftCore.cache.findApplication(cmd)
if tmp:
cmd = tmp
if CraftCore.compiler.isWindows:
command = f"{self._findBash()} -c \"{self.toNativePath(cmd)} {args}\""
else:
command = f"{self.toNativePath(cmd)} {args}"
env = dict(os.environ)
env.update(self.environment)
env.update(kwargs.get("env", {}))
return utils.system(command, cwd=path, env=env, **kwargs)
def patch(dest_dir, relative_patches_dir=os.curdir):
"""Applies the *.patch files in |relative_patches_dir|.
Args:
relative_patches_dir: A directory path relative to the current directory.
Defaults to the directory of this file.
Raises:
subprocess.CalledProcessError if the patch couldn't be applied.
"""
patches_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
relative_patches_dir)
assert os.path.isdir(patches_dir)
os.chdir(dest_dir)
for p in utils.find(["*.patch"], patches_dir):
print "applying patch %s" % os.path.basename(p)
try:
utils.system(["git", "apply", p])
utils.commit("applied patch %s" % os.path.basename(p))
except subprocess.CalledProcessError:
print "ERROR: patch %s failed to apply" % os.path.basename(p)
raise
def run_once(self):
errors = 0
test_dir = '/tmp/chromium_lsm_test_dir'
os.mkdir(test_dir, 0700)
mnt_target = '%s/mount_point' % (test_dir)
os.mkdir(mnt_target, 0700)
sym_target = '%s/symlink' % (test_dir)
os.symlink('mount_point', sym_target)
# Mounting should succeed (no symbolic link in mount path).
errors += self._check_mount(mnt_target, True,
'Unable to mount on a directory')
# Mounting should fail as we used a mount path with a symbolic link.
errors += self._check_mount(sym_target, False,
'Unexpectedly mounted on a symlink')
utils.system('rm -rf ' + test_dir)
# If self.error is not zero, there were errors.
if errors > 0:
raise error.TestFail('Failed %d tests' % errors)
def system(cmdline, result='output'):
"""Run a command and return its output.
If result == 'status', the exit status of the command is returned.
If result == 'output', the output of the command is returned.
If result == 'both', a tuple of status and output is returned.
"""
sta, out, err = utils.system(cmdline)
if result == 'status':
return sta
elif result == 'output':
return out
elif result == 'both':
return sta, out
def signMacPackage(packagePath: str):
if not CraftCore.settings.getboolean("CodeSigning", "Enabled", False):
return True
# special case, two independent setups of craft might want to sign at the same time and only one keychain can be unlocked at a time
with _MacSignScope() as scope:
packagePath = Path(packagePath)
devID = CraftCore.settings.get("CodeSigning", "MacDeveloperId")
if packagePath.name.endswith(".dmg"):
# sign dmg
if not utils.system([
"codesign", "--force", "--keychain", scope.loginKeychain,
"--sign", f"Developer ID Application: {devID}", packagePath
]):
return False
# TODO: this step would require notarisation
# verify dmg signature
utils.system([
"spctl", "-a", "-t", "open", "--context",
"context:primary-signature", packagePath
])
else:
# sign pkg
packagePathTmp = f"{packagePath}.sign"
if not utils.system([
"productsign", "--keychain", scope.loginKeychain, "--sign",
f"Developer ID Installer: {devID}", packagePath,
packagePathTmp
]):
return False
utils.moveFile(packagePathTmp, packagePath)
return True
def system(self, command, errorMessage="", debuglevel=1, **kw):
"""convencience function for running system commands.
This method prints a debug message and then runs a system command.
If the system command returns with errors the method prints an error
message and exits if @ref self.subinfo.options.exitOnErrors is true"""
if utils.system(command, **kw):
return True
if self.subinfo.options.exitOnErrors:
EmergeDebug.warning("while running %s cmd: %s" %
(errorMessage, str(command)))
else:
EmergeDebug.warning("while running %s cmd: %s" %
(errorMessage, str(command)))
return False
def execute(self, path, cmd, args="", **kwargs):
# try to locate the command
if CraftCore.compiler.isWindows:
tmp = CraftCore.cache.findApplication(cmd)
if tmp:
cmd = tmp
command = Arguments([
self._findBash(), "-c",
str(Arguments([self.toNativePath(cmd), args]))
])
else:
command = Arguments([cmd, args])
env = dict(os.environ)
env.update(self.environment)
env.update(kwargs.get("env", {}))
return utils.system(command, cwd=path, env=env, **kwargs)
def signWindows(fileNames: [str]) -> bool:
if not CraftCore.settings.getboolean("CodeSigning", "Enabled", False):
return True
if not CraftCore.compiler.isWindows:
CraftCore.log.warning(
"Code signing is currently only supported on Windows")
return True
signTool = CraftCore.cache.findApplication("signtool", forceCache=True)
if not signTool:
env = SetupHelper.getMSVCEnv()
signTool = CraftCore.cache.findApplication("signtool",
env["PATH"],
forceCache=True)
if not signTool:
CraftCore.log.warning(
"Code signing requires a VisualStudio installation")
return False
command = [
signTool, "sign", "/tr", "http://timestamp.digicert.com", "/td",
"SHA256", "/fd", "SHA256", "/a"
]
certFile = CraftCore.settings.get("CodeSigning", "Certificate", "")
subjectName = CraftCore.settings.get("CodeSigning", "CommonName", "")
certProtected = CraftCore.settings.getboolean("CodeSigning", "Protected",
False)
kwargs = dict()
if certFile:
command += ["/f", certFile]
if subjectName:
command += ["/n", subjectName]
if certProtected:
password = CraftChoicePrompt.promptForPassword(
message='Enter the password for your package signing certificate',
key="WINDOWS_CODE_SIGN_CERTIFICATE_PASSWORD")
command += ["/p", password]
kwargs["secret"] = [password]
if True or CraftCore.debug.verbose() > 0:
command += ["/v"]
else:
command += ["/q"]
for args in utils.limitCommandLineLength(command, fileNames):
if not utils.system(args, **kwargs):
return False
return True
def _createShortPath(longPath) -> str:
if not CraftShortPath._useShortpaths:
return longPath
import utils
utils.createDir(CraftCore.standardDirs.junctionsDir())
longPath = OsUtils.toNativePath(longPath)
path = CraftCore.standardDirs.junctionsDir() / hex(
zlib.crc32(bytes(longPath, "UTF-8")))[2:]
delta = len(longPath) - len(str(path))
if delta <= 0:
CraftCore.debug.log.debug(
f"Using junctions for {longPath} wouldn't save characters returning original path"
)
CraftCore.debug.log.debug(f"{longPath}\n" f"{path}, gain: {delta}")
return longPath
utils.createDir(longPath)
if not os.path.isdir(path):
if OsUtils.isUnix():
ok = utils.createSymlink(longPath,
path,
useAbsolutePath=True,
targetIsDirectory=True)
else:
# note: mklink is a CMD command => needs shell
ok = utils.system(["mklink", "/J", path, longPath],
shell=True,
stdout=subprocess.DEVNULL,
logCommand=False)
if not ok:
CraftCore.debug.log.critical(
f"Could not create shortpath {path}, for {longPath}")
return longPath
else:
if not os.path.samefile(path, longPath):
CraftCore.debug.log.critical(
f"Existing short path {path}, did not match {longPath}")
return longPath
CraftCore.debug.log.debug(f"Mapped \n"
f"{longPath} to\n"
f"{path}, gained {delta}")
return path
def __signMacApp(appPath : Path, scope : _MacSignScope):
CraftCore.log.info(f"Sign {appPath}")
devID = CraftCore.settings.get("CodeSigning", "MacDeveloperId")
bundlePattern = re.compile(r".*(\.app|\.framework)$", re.IGNORECASE)
# get all bundles, as we specify handleAppBundleAsFile we will not yet get nested bundles
for bun in utils.filterDirectoryContent(appPath,
whitelist=lambda x, root: bundlePattern.match(x.path),
blacklist=lambda x, root: True,
handleAppBundleAsFile = True):
if not __signMacApp(Path(bun), scope):
return False
# all files in the bundle
bundeFilter = lambda x, root: not Path(x.path).is_symlink() and not bundlePattern.match(x.path)
filter = bundeFilter
# we can only sign non binary files in Resources, else they get stored in the
# extended attributes and might get lost during deployment
# TODO: allow for dmg?
# https://github.com/packagesdev/packages/issues/65
if "Contents/Resources" not in str(appPath):
filter = lambda x, root: bundeFilter(x, root) and utils.isBinary(x.path)
binaries = list(utils.filterDirectoryContent(appPath,
whitelist=lambda x, root: filter(x, root),
blacklist=lambda x, root: True,
handleAppBundleAsFile = True))
mainApp = appPath / "Contents/MacOS" / appPath.name.split(".")[0]
if str(mainApp) in binaries:
binaries.remove(str(mainApp))
signCommand = ["codesign", "--keychain", scope.loginKeychain, "--sign", f"Developer ID Application: {devID}", "--force", "--preserve-metadata=identifier,entitlements", "--options", "runtime", "--verbose=99", "--timestamp"]
for command in utils.limitCommandLineLength(signCommand, binaries):
if not utils.system(command):
return False
if not utils.system(signCommand + ["--deep", appPath]):
return False
## Verify signature
if not utils.system(["codesign", "--display", "--verbose", appPath]):
return False
if not utils.system(["codesign", "--verify", "--verbose", "--strict", "--deep", appPath]):
return False
# TODO: this step might require notarisation
utils.system(["spctl", "-a", "-t", "exec", "-vv", appPath])
return True
def createPackage(self):
""" create a package """
CraftCore.log.debug("packaging using the MacDMGPackager")
defines = self.setDefaults(self.defines)
# TODO: provide an image with dbg files
if not self.internalCreatePackage(defines):
return False
appPath = self.getMacAppPath(defines)
archive = os.path.normpath(self.archiveDir())
CraftCore.log.info(f"Packaging {appPath}")
dmgDest = defines["setupname"]
if os.path.exists(dmgDest):
utils.deleteFile(dmgDest)
appName = defines['appname'] + ".app"
if not utils.system([
"create-dmg",
"--volname",
os.path.basename(dmgDest),
# Add a drop link to /Applications:
"--icon",
appName,
"140",
"150",
"--app-drop-link",
"350",
"150",
dmgDest,
appPath
]):
return False
if not CodeSign.signMacPackage(dmgDest):
return False
CraftHash.createDigestFiles(dmgDest)
return True
def exec_stop(self):
self.stopped = True
if self.started:
fwmark = config.global_config["transproxy"]["fwmark"]
ip_route_table = config.global_config["transproxy"][
"ip_route_table"]
interface_name = config.global_config["transproxy"]["wireguard"][
"interface_name"]
self.iptables.del_all()
utils.system("ip route del table %d default dev %s" %
(ip_route_table, interface_name))
utils.system("ip rule del fwmark %d table %d" %
(fwmark, ip_route_table))
utils.system("ip link delete %s" % interface_name)
def _createShortPath(longPath) -> str:
import utils
longPath = OsUtils.toNativePath(longPath)
if not CraftShortPath._useShortpaths:
return longPath
if not os.path.isdir(CraftCore.standardDirs.junctionsDir()):
os.makedirs(CraftCore.standardDirs.junctionsDir())
path = OsUtils.toNativePath(os.path.join(CraftCore.standardDirs.junctionsDir(), hex(zlib.crc32(bytes(longPath, "UTF-8")))[2:]))
if len(longPath) < len(path):
CraftCore.debug.log.debug(f"Using junctions for {longPath} wouldn't save characters returning original path")
CraftCore.debug.log.debug(f"{longPath}\n"
f"{path}, gain:{len(longPath) - len(path)}")
return longPath
os.makedirs(longPath, exist_ok=True)
if not os.path.isdir(path):
# note: mklink is a CMD command => needs shell
if not utils.system(["mklink", "/J", path, longPath], shell=True, stdout=subprocess.DEVNULL, logCommand=False):
CraftCore.debug.log.critical(f"Could not create shortpath {path}, for {longPath}")
return longPath
else:
if not os.path.samefile(path, longPath):
CraftCore.debug.log.critical(f"Existing short path {path}, did not match {longPath}")
return longPath
return path
def _createShortPath(longPath) -> str:
if not os.path.isdir(CraftStandardDirs.junctionsDir()):
os.makedirs(CraftStandardDirs.junctionsDir())
path = os.path.join(CraftStandardDirs.junctionsDir(),
str(zlib.crc32(bytes(longPath, "UTF-8"))))
if len(longPath) < len(path):
craftDebug.log.info(
f"Using junctions for {longPath} wouldn't save characters returning original path"
)
return longPath
if not os.path.isdir(path):
if not os.path.isdir(longPath):
os.makedirs(longPath)
# note: mklink is a CMD command => needs shell
if not utils.system(["mklink", "/J", path, longPath], shell=True):
craftDebug.log.critical(
f"Could not create shortpath {path}, for {longPath}")
return longPath
else:
if not os.path.samefile(path, longPath):
craftDebug.log.critical(
f"Existing short path {path}, did not match {longPath}")
return longPath
return path
def roll(target_version):
try:
chromium_rev = chromium_rev_number(target_version)
except urllib2.HTTPError:
print ("Failed to identify a Chromium revision associated with %s. "
"Ensure that target_version is a Chromium origin/master "
"commit.") % (target_version)
return 1
mojoms_gs_path = "gs://mojo/network_service/%s/mojoms.zip" % (target_version,)
network_service_path = os.path.join(
mojo_root_dir, "mojo", "services", "network")
mojoms_path = os.path.join(network_service_path, "public", "interfaces")
version_path = os.path.join(network_service_path, "VERSION")
try:
with tempfile.NamedTemporaryFile() as temp_zip_file:
gs.download_from_public_bucket(mojoms_gs_path, temp_zip_file.name)
try:
system(["git", "rm", "-r", mojoms_path], cwd=mojo_root_dir)
except subprocess.CalledProcessError:
print ("Could not remove %s. "
"Ensure your local tree is in a clean state." % mojoms_path)
return 1
with zipfile.ZipFile(temp_zip_file.name) as z:
z.extractall(mojoms_path)
# pylint: disable=C0302,bare-except
except:
print ("Failed to download the mojom files associated with %s. Ensure that "
"the corresponding network service artifacts were uploaded to "
"Google Storage.") % (target_version)
return 1
with open(version_path, 'w') as stamp_file:
stamp_file.write(target_version)
system(["git", "add", "public"], cwd=network_service_path)
system(["git", "add", "VERSION"], cwd=network_service_path)
commit("Roll the network service to https://crrev.com/" + chromium_rev,
cwd=mojo_root_dir)
return 0
import matplotlib.pyplot as plt
from ftocp import FTOCP
from nlp import NLP
from matplotlib import rc
from numpy import linalg as la
# rc('font',**{'family':'sans-serif','sans-serif':['Helvetica']})
# rc('text', usetex=True)
# Aaron added
import pdb
# =============================
# Initialize system parameters
x0 = np.zeros(4)
dt = 0.1 # Discretization time
sys = system(x0, dt) # initialize system object
maxTime = 14 # Simulation time
goal = np.array([10,10,0,np.pi/2])
# Initialize mpc parameters
N = 20; n = 4; d = 2;
Q = 1*np.eye(n)
R = 1*np.eye(d)
Qf = 1000*np.eye(n)
# =================================================================
# ======================== Subsection: Nonlinear MPC ==============
# First solve the nonlinear optimal control problem as a Non-Linear Program (NLP)
printLevel = 1
xub = np.array([15, 15, 15, 15])
uub = np.array([10, 0.5])
def environment(self):
if not self._environment:
mergeroot = self.toNativePath(CraftCore.standardDirs.craftRoot())
ldflags = f" -L{mergeroot}/lib "
cflags = f" -I{mergeroot}/include "
if CraftCore.compiler.isMacOS:
# Only look for includes/libraries in the XCode SDK on MacOS to avoid errors with
# libraries installed by homebrew (causes errors e.g. with iconv since headers will be
# found in /usr/local/include first but libraries are searched for in /usr/lib before
# /usr/local/lib. See https://langui.sh/2015/07/24/osx-clang-include-lib-search-paths/
# Ensure that /usr/include comes before /usr/local/include in the header search path to avoid
# pulling in headers from /usr/local/include (e.g. installed by homebrew) that will cause
# linker errors later.
sdkPath = CraftCore.cache.getCommandOutput(
"xcrun", "--show-sdk-path")[1].strip()
deploymentFlag = f"-mmacosx-version-min={os.environ['MACOSX_DEPLOYMENT_TARGET']}"
cflags = f" -isysroot {sdkPath} {deploymentFlag} {cflags} -isystem /usr/include"
ldflags = f" -isysroot {sdkPath} {deploymentFlag} {ldflags}"
# TODO: well that sounded like a good idea, but is broken with recent xcode
# when fixed we probably should also set that flag for the rest too?
# See https://github.com/Homebrew/homebrew-core/issues/2674 for the -no_weak_imports flag
#ldflags = f" -Wl,-no_weak_imports {ldflags}"
if CraftCore.compiler.isMSVC():
# based on Windows-MSVC.cmake
if self.buildType == "Release":
cflags += " -MD -O2 -Ob2 -DNDEBUG "
elif self.buildType == "RelWithDebInfo":
cflags += " -MD -Zi -O2 -Ob1 -DNDEBUG "
ldflags += " -debug "
elif self.buildType == "Debug":
cflags += " -MDd -Zi -Ob0 -Od "
ldflags += " -debug -pdbtype:sept "
else:
if self.buildType == "Release":
cflags += " -O3 -DNDEBUG "
if self.buildType == "RelWithDebInfo":
cflags += " -O2 -g -DNDEBUG "
elif self.buildType == "Debug":
cflags += " -O0 -g3 "
if OsUtils.isWin():
def convertPath(path: str):
return ":".join([
str(self.toNativePath(p))
for p in path.split(os.path.pathsep)
])
path = "/usr/local/bin:/usr/bin:/bin:/usr/bin/site_perl:/usr/bin/vendor_perl:/usr/bin/core_perl"
if CraftCore.compiler.isMinGW():
gcc = shutil.which("gcc")
if gcc:
path = f"{self.toNativePath(os.path.dirname(gcc))}:{path}"
elif CraftCore.compiler.isMSVC():
path = f"{self.toNativePath(os.path.dirname(shutil.which('cl')))}:{path}"
self._environment[
"PATH"] = f"{path}:{convertPath(os.environ['PATH'])}"
self._environment["PKG_CONFIG_PATH"] = convertPath(
os.environ["PKG_CONFIG_PATH"])
if "make" in self._environment:
del self._environment["make"]
# MSYSTEM is used by uname
if CraftCore.compiler.isMinGW():
self._environment[
"MSYSTEM"] = f"MINGW{CraftCore.compiler.bits}_CRAFT"
elif CraftCore.compiler.isMSVC():
self._environment[
"MSYSTEM"] = f"MSYS{CraftCore.compiler.bits}_CRAFT"
if self.useMSVCCompatEnv and CraftCore.compiler.isMSVC():
automake = []
for d in os.scandir(
os.path.join(os.path.dirname(self._findBash()),
"..", "share")):
if d.name.startswith("automake"):
automake += [(d.name.rsplit("-")[1],
os.path.realpath(d.path))]
automake.sort(key=lambda x: CraftVersion(x[0]))
latestAutomake = automake[-1][1]
if False:
cl = "clang-cl"
else:
cl = "cl"
clWrapper = self.toNativePath(
os.path.join(latestAutomake, "compile"))
self._environment[
"AR"] = f"{self.toNativePath(os.path.join(latestAutomake, 'ar-lib'))} lib"
self._environment["LD"] = "link -nologo"
self._environment["CC"] = f"{clWrapper} {cl} -nologo"
self._environment["CXX"] = self._environment["CC"]
self._environment["CPP"] = f"{cl} -nologo -EP"
self._environment["CXXCPP"] = self._environment["CPP"]
self._environment["NM"] = "dumpbin -symbols"
self._environment[
"RC"] = f"windres -O COFF --target={'pe-i386' if CraftCore.compiler.isX86() else 'pe-x86-64'} --preprocessor='cl -nologo -EP -DRC_INVOKED -DWINAPI_FAMILY=0'"
self._environment["STRIP"] = ":"
self._environment["RANLIB"] = ":"
self._environment["F77"] = "no"
self._environment["FC"] = "no"
cflags += (
" -GR -W3 -EHsc" # dynamic and exceptions enabled
" -D_USE_MATH_DEFINES -DWIN32_LEAN_AND_MEAN -DNOMINMAX -D_CRT_SECURE_NO_WARNINGS"
" -wd4005" # don't warn on redefine
" -wd4996" # The POSIX name for this item is deprecated.
)
if CraftCore.compiler.getMsvcPlatformToolset() > 120:
cflags += " -FS"
if CraftCore.compiler.isAndroid:
toolchainPath = os.path.join(
CraftCore.standardDirs.tmpDir(),
f"android-{CraftCore.compiler.architecture}-toolchain")
utils.system([
"python3",
os.path.join(os.environ.get("ANDROID_NDK_ROOT"),
"build/tools/make_standalone_toolchain.py"),
"--install-dir", toolchainPath, "--arch",
CraftCore.compiler.architecture, "--api",
CraftCore.compiler.androidApiLevel()
])
self._environment["PATH"] = os.path.join(
toolchainPath, "bin") + ":" + os.environ["PATH"]
self._environment["AR"] = "ar"
self._environment["AS"] = "clang"
self._environment["CC"] = "clang"
self._environment["CXX"] = "clang++"
self._environment["LD"] = "ld"
self._environment["STRIP"] = "strip"
self._environment["RANLIB"] = "ranlib"
self._environment["CFLAGS"] = os.environ.get("CFLAGS", "").replace(
"$", "$$") + cflags
self._environment["CXXFLAGS"] = os.environ.get(
"CXXFLAGS", "").replace("$", "$$") + cflags
self._environment["LDFLAGS"] = os.environ.get(
"LDFLAGS", "").replace("$", "$$") + ldflags
return self._environment
