def installFromList(filename, srcdir, destdir, overwrite=True, link=False, optimizeCopy=False,developerOnly=False):
"""Takes a list of src/dest files and directories. Each line has a source,
and a destination. Lines starting with "#" are comments.
This method is use for the post-build step on both unix
and windows.
A line may be preceded by "@unix" or "@win32" to indicate that
it should only be copied for that architecture
"""
substitutions = dict(
PythonVersion="%d.%d" % (sys.version_info[0], sys.version_info[1]),
arch=getArch())
text = open(filename).read()
t = string.Template(text)
text = t.substitute(substitutions)
lines = text.split("\n")
# strip out comments and newlines
lines = [l.strip() for l in lines]
lines = [l for l in lines if l != ""]
lines = [l for l in lines if l[0] != "#"]
for line in lines:
elements = line.split()
arch = elements[0]
if len(elements) == 3:
if arch == "@win32":
if getArch() != "win32": continue
elif arch == "@unix":
if getArch() == "win32": continue
elif arch == "@darwin86":
if getArch() != "darwin86": continue
elif arch == "@darwin64":
if getArch() != "darwin64": continue
elif arch == "@darwin":
# both darwin86 and darwin64
if getArch() not in ["darwin86", "darwin64"]: continue
elif arch == "@linux64":
if getArch() != "linux64": continue
elif arch == "@linux32":
if getArch() != "linux32": continue
elif arch == "@linux32arm":
if getArch() != "linux32arm": continue
elif arch == "@linux32armv7":
if getArch() != "linux32armv7": continue
else:
raise Exception("Unknown architecture type %s in file %s" % (arch, filename))
src = elements[1]
dest = elements[2]
elif len(elements) == 2:
src = elements[0]
dest = elements[1]
else:
raise Exception("Line in unknown format: '%s'" % line)
src = os.path.join(srcdir, src)
dest = os.path.join(destdir, dest)
smartCopy(src, dest, overwrite=overwrite, link=link, optimizeCopy=optimizeCopy, developerOnly=developerOnly)
def installFromList(filename, srcdir, destdir, overwrite=True, link=False, optimizeCopy=False,developerOnly=False):
"""Takes a list of src/dest files and directories. Each line has a source,
and a destination. Lines starting with "#" are comments.
This method is use for the post-build step on both unix
and windows.
A line may be preceded by "@unix" or "@win32" to indicate that
it should only be copied for that architecture
"""
substitutions = dict(
PythonVersion="%d.%d" % (sys.version_info[0], sys.version_info[1]),
arch=getArch())
text = open(filename).read()
t = string.Template(text)
text = t.substitute(substitutions)
lines = text.split("\n")
# strip out comments and newlines
lines = [l.strip() for l in lines]
lines = [l for l in lines if l != ""]
lines = [l for l in lines if l[0] != "#"]
for line in lines:
elements = line.split()
arch = elements[0]
if len(elements) == 3:
if arch == "@win32":
if getArch() != "win32": continue
elif arch == "@unix":
if getArch() == "win32": continue
elif arch == "@darwin86":
if getArch() != "darwin86": continue
elif arch == "@darwin64":
if getArch() != "darwin64": continue
elif arch == "@darwin":
# both darwin86 and darwin64
if getArch() not in ["darwin86", "darwin64"]: continue
elif arch == "@linux64":
if getArch() != "linux64": continue
elif arch == "@linux32":
if getArch() != "linux32": continue
else:
raise Exception("Unknown architecture type %s in file %s" % (arch, filename))
src = elements[1]
dest = elements[2]
elif len(elements) == 2:
src = elements[0]
dest = elements[1]
else:
raise Exception("Line in unknown format: '%s'" % line)
src = os.path.join(srcdir, src)
dest = os.path.join(destdir, dest)
smartCopy(src, dest, overwrite=overwrite, link=link, optimizeCopy=optimizeCopy, developerOnly=developerOnly)
def createCustomerRelease(releaseType, releaseSpecDir, destDir, fromDir, releaseName, allArchitectures=True, allArchiveTypes=False, allowSymbolicLinks=False, parentTmpDir=None):
"""Create a customer release tarball in destdir from the
directories engReleaseDir and releaseSpecDir
releaseType is one of biaryReleaseTypes or sourceReleaseTypes
fromDir is engineering build for the binary release,
source dir for a source release
releaseSpecDir is directory containing manifests
Returns a path to the tarball
Throws an exception on error"""
if not os.path.exists(destDir):
utils.createDir(destDir)
if releaseType in binaryReleaseTypes:
isSourceRelease = False
elif releaseType in sourceReleaseTypes:
isSourceRelease = True
else:
raise Exception("Unknown release type '%s'" % releaseType)
if not isSourceRelease:
allArchitectures=False
manifestFile = os.path.join(releaseSpecDir, releaseType + "_release",
"manifests", releaseType + "_release.manifest")
log.info("Creating customer release of type %s", releaseType)
log.info("specDir: %s", releaseSpecDir)
log.info("fromDir: %s", fromDir)
tempdir = utils.createTemporaryDirectory("assemble_release", parentDir=parentTmpDir)
try:
releaseImageDir = os.path.join(tempdir, releaseName)
installFromManifest(manifestFile, fromDir, releaseImageDir,
level=0, overwrite=False, destdirExists=False,
allArchitectures=allArchitectures, allowSymbolicLinks=allowSymbolicLinks)
if isSourceRelease:
# run autogen equivalent on source trees
build.prepareSourceTree(releaseImageDir, customerRelease=True)
# Need to create both zip and tgz source releases on the master build system
if allArchiveTypes == True:
# sanity checks
if not isSourceRelease:
raise Exception("Attempt to create both zip and tar file for a binary release. Probably an error")
if getArch() == "win32":
raise Exception("Attempt to create .tgz source release on a windows system. Would not include autogen.sh")
zipFile = utils.createArchive(releaseImageDir, destDir, type="zip")
tarFile = utils.createArchive(releaseImageDir, destDir, type="tar")
return (tarFile, zipFile)
else:
release = utils.createArchive(releaseImageDir, destDir)
return release
finally:
utils.remove(tempdir)
def __init__(self, addr):
self.calls = {}
self.rets = {}
self.callDsts = {}
self.funcAddrs = {}
ah = getArch(ARCH)
self.BIT = ah.getBit() // 4
self.RET = ah.getRets()
tracerFile = addr.split('/')[-1] + '.tracer'
if (not self.load(tracerFile, addr)):
self.analyze(addr)
self.save(tracerFile, addr)
def __init__(self, p, funcName):
self.p = p
self.func = funcName
self.sourceFolder = SOURCEFOLDER
ah = getArch(ARCH)
self.eip = ah.getEip()
self.max_brkTime = 30 # If a breakpoint is triggered more than this number, it will be removed
self.ed = '394743516231415926' # A random number as end of output of a session
print('loading %s' % ASMFILE)
cur_time = time.clock()
self.asm = asmAnalyser(ASMFILE)
print('finished loading, cost %ss' % str(time.clock() - cur_time))
if (not self.asm.funcExist(funcName)):
print('ERROR! Function %s not exist!' % funcName)
return
p.stdin.write(bytes('target remote:%s\n' % GDBPORT, encoding='utf8'))
def getReleaseName(stamp, releaseType, debug=False):
arch = getArch()
if debug:
prefix = "nupic-debug"
else:
prefix = "nupic"
if releaseType == "binary":
releaseName = "%s-%s-%s" % (prefix, stamp, arch)
elif releaseType == "vitamind":
releaseName = "%s-vitamind-%s-%s" % (prefix, stamp, arch)
elif releaseType == "npp":
releaseName = "%s-npp-%s-%s" % (prefix, stamp, arch)
elif releaseType in sourceReleaseTypes:
releaseName = "%s-%s-%s" % (prefix, stamp, releaseType)
else:
raise Exception("getReleaseName: unknown release type '%s'" % releaseType)
return releaseName
import os
import sys
import re
from pprint import pprint as pp
import operator
scriptDir = os.path.dirname(__file__)
externalDir = os.path.abspath(os.path.join(scriptDir, '../../external'))
pybuildDir = os.path.abspath(os.path.join(scriptDir, '../../build_system/pybuild'))
langPyDir = os.path.abspath(os.path.join(scriptDir, '../py'))
# Get the architecture first
sys.path = [os.path.join(pybuildDir)] + sys.path
import arch
a = arch.getArch()
# Get yaml from external (it is not installed)
pythonVersion = '%d.%d' % sys.version_info[:2]
sys.path = [os.path.join(externalDir, a,
'lib/python%s/site-packages' % pythonVersion)] + sys.path
import yaml
# Restore sys.path
sys.path = sys.path[2:]
# The directory that contains the NuPIC 2 C++ object model
baseDir = os.path.abspath(os.path.join(scriptDir, '../../nta/engine'))
assert os.path.isdir(baseDir)
def getBlock(lines):
def installFromManifest(manifestFileName, srcdir, destdir, level, overwrite, destdirExists, allArchitectures=False, arch=None, allowSymbolicLinks=False):
# use an additional two spaces of indentation for each submanifest level
indent = ""
for i in xrange(0, level):
indent = indent + " "
nfailures = 0
if arch is None:
myarch = getArch()
else:
myarch = arch
log.info(indent + "Installing from %s" % os.path.basename(manifestFileName))
try:
try:
manifest = open(manifestFileName)
except:
log.error(indent + "Unable to open manifest file '%s'" % manifestFileName)
raise
if not os.path.isdir(srcdir):
nfailures = nfailures + 1
raise Exception(indent + 'Source directory "%s" does not exist or is not a directory' % srcdir)
if os.path.exists(destdir) and not destdirExists:
raise Exception(indent + 'Destination directory "%s" already exists. Delete before installing manifest.' % destdir)
#
# by default, files in the manifest file are specified by a pathname
# relative to the source root. There are two exceptions
# 1. pathnames to a manifest file (beginning with '@include') are relative to the path
# of the directory containing the current manifest file
# 2. pathnames specified with @local are relative to the directory containing the current
# manifest file (typically "../files/<filename>").
fullManifestPath = os.path.abspath(manifestFileName)
manifestDirPath = os.path.dirname(fullManifestPath)
if not destdirExists:
os.makedirs(destdir, mode=0755)
nfiles = 0
nsubmanifests = 0
# Go through the manifest file line by line. Primitive parsing is ok our
# simple format, though it is a bit clunky.
lines = manifest.readlines()
for line in lines:
# ignore comments and blank lines
line = line.strip()
if line == "":
continue
if line[0] == "#":
continue
# @all@ expands to one file per architecture
if line.find("@all@") != -1:
if allArchitectures:
arches = ["linux64", "linux32", "darwin86", "win32"]
else:
arches = [myarch]
for a in arches:
newline = line.replace("@all@", a)
# Do the substitutions right here, rather than on the second
# pass, because this is where we know the architecture type
newline = doSubstitutions(newline, a)
log.debug("Adding rule '%s' based on '%s'", newline, line)
lines.append(newline)
continue
# @allunix@ expands to one file per unix architecture
elif line.find("@allunix@") != -1:
if allArchitectures:
arches = ["linux64", "linux32", "darwin86"]
elif myarch != "win32":
arches = [myarch]
else:
arches = list()
for a in arches:
newline = line.replace("@allunix@", a)
# Do the substitutions right here, rather than on the second
# pass, because this is where we know the architecture type
newline = doSubstitutions(newline, a)
lines.append(newline)
log.debug("Adding rule '%s' based on '%s'",newline, line)
continue
else:
line = doSubstitutions(line, myarch)
fields = line.split()
if len(fields) == 3:
# @arch <archname> <filename>
# This is an architecture-specific filename. If allArchitectures is false,
# it is only copied if myarch matches. If allArchitectures is true,
# then the file is copied for that specific architecture.
# archname may be a specific architecture or may be "win32"
# @include arch <manifestfile>
# @local <srcpath> <destpath>
if fields[0] == "@arch":
if allArchitectures or \
myarch == fields[1] or \
(fields[1] == "unix" and myarch != "win32"):
srcpath = fields[2]
fullsrcpath = os.path.join(srcdir, srcpath)
destpath = srcpath
else:
log.debug(indent + "Skipping line for different architecture: %s" % line)
continue
elif fields[0] == "@include":
arch = fields[1]
submanifest = fields[2]
if allArchitectures or (arch == "unix" and myarch != "win32") or arch == myarch:
# @todo consolidate submanifest code with @include below?
if not os.path.isabs(submanifest):
submanifest = os.path.abspath(os.path.join(manifestDirPath, submanifest))
log.info(indent + "Installing files from sub-manifest %s" % submanifest)
nsubmanifests = nsubmanifests + 1
try:
installFromManifest(submanifest, srcdir, destdir,
level=level+1, overwrite=overwrite,
destdirExists=True, allArchitectures=allArchitectures,
arch=myarch, allowSymbolicLinks=allowSymbolicLinks)
except KeyboardInterrupt:
nfailures = nfailures + 1
raise Exception("Keyboard interrupt")
except:
nfailures = nfailures + 1
log.error(indent + "Installation from sub-manifest %s failed", submanifest)
continue
else:
log.debug("Skipping line for different achitecture: %s" % line)
continue
elif fields[0] == "@local":
srcpath = fields[1]
fullsrcpath = os.path.abspath(os.path.join(manifestDirPath, fields[1]))
destpath = fields[2]
else:
log.error(indent + 'Format error in manifest file. Lines with three fields must begin with @local or @arch')
nfailures = nfailures+1
continue
elif len(fields) == 2:
# @include <manifestfile>
# Include another manifest file. Path is relative to current manifest file
#
# <srcpath> <destpath>
# Copy from srcpath in source tree to destpath in dest tree
if fields[0] == "@include":
submanifest = fields[1]
if not os.path.isabs(submanifest):
submanifest = os.path.abspath(os.path.join(manifestDirPath, submanifest))
log.info(indent + "Installing files from sub-manifest %s" % submanifest)
nsubmanifests = nsubmanifests + 1
try:
installFromManifest(submanifest, srcdir, destdir,
level=level+1, overwrite=overwrite,
destdirExists=True, allArchitectures=allArchitectures,
arch=myarch,allowSymbolicLinks=allowSymbolicLinks)
except KeyboardInterrupt:
nfailures = nfailures + 1
raise Exception("Keyboard interrupt")
except:
nfailures = nfailures + 1
log.error(indent + "Installation from sub-manifest %s failed", submanifest)
continue
else:
srcpath = fields[0]
fullsrcpath = os.path.join(srcdir, srcpath)
destpath = fields[1]
elif len(fields) == 1:
# <path>
# Copy from source to dest, using same relative path
srcpath = fields[0]
destpath = srcpath
fullsrcpath = os.path.join(srcdir, srcpath)
else:
log.error(indent + "Format error in manifest file. ")
log.error(indent + "Bad line: '%s'" % line)
nfailures = nfailures + 1
continue
if not os.path.exists(fullsrcpath):
log.error(indent + "Source path %s does not exist" % fullsrcpath)
nfailures = nfailures + 1
continue
fulldestpath = os.path.join(destdir, destpath)
fulldestdir = os.path.dirname(fulldestpath)
if not os.path.exists(fulldestdir):
log.debug(indent + "Creating directory %s" % fulldestdir)
os.makedirs(fulldestdir, 0755)
if os.path.exists(fulldestpath):
if not overwrite:
log.error(indent + "File %s already exists and overwrite flag is not set" % (fulldestpath))
nfailures = nfailures + 1
else:
log.debug(indent + "File %s already exists -- overwriting" % (fulldestpath))
if overwrite and not os.path.exists(fulldestpath):
log.warning(indent + "File %s does not exist and overwrite flag is set (not fatal)" % (fulldestpath))
# Complain if the file is a symbolic link but ok if allowSymbolicLinks is specified
if os.path.islink(fullsrcpath):
if allowSymbolicLinks:
target = os.readlink(fullsrcpath)
if os.path.isabs(target):
fullsrcpath = target
else:
fullsrcpath = os.path.normpath(os.path.join(os.path.dirname(fullsrcpath), target))
log.warn(indent + "Copying target of symbolic link %s" % fullsrcpath)
else:
log.error(indent + "Skipping symbolic link %s" % fullsrcpath)
nfailures = nfailures + 1
continue
# copy the file
log.debug(indent + "Copying %s -> %s" % (srcpath, destpath))
nfiles = nfiles + 1
try:
shutil.copy2(fullsrcpath, fulldestpath)
except Exception, e:
nfailures = nfailures + 1
log.error(indent + "Unable to copy %s to %s. Error: %s", srcpath, destpath, e)
### while(newline != "")
if nfailures == 0:
if nsubmanifests == 0:
log.info(indent + "Done. %d files copied" % (nfiles))
else:
log.info(indent + "Done. %d files copied %s submanifests processed" % (nfiles, nsubmanifests))
else:
log.info(indent + "Done. %d copy attempts; %d failures; %d submanifests" % (nfiles, nfailures, nsubmanifests))
raise Exception("install from manifest failed")
def installFromManifest(
manifestFileName,
srcdir,
destdir,
level,
overwrite,
destdirExists,
allArchitectures=False,
arch=None,
allowSymbolicLinks=False,
):
# use an additional two spaces of indentation for each submanifest level
indent = ""
for i in xrange(0, level):
indent = indent + " "
nfailures = 0
if arch is None:
myarch = getArch()
else:
myarch = arch
log.info(indent + "Installing from %s" % os.path.basename(manifestFileName))
try:
try:
manifest = open(manifestFileName)
except:
log.error(indent + "Unable to open manifest file '%s'" % manifestFileName)
raise
if not os.path.isdir(srcdir):
nfailures = nfailures + 1
raise Exception(indent + 'Source directory "%s" does not exist or is not a directory' % srcdir)
if os.path.exists(destdir) and not destdirExists:
raise Exception(
indent + 'Destination directory "%s" already exists. Delete before installing manifest.' % destdir
)
#
# by default, files in the manifest file are specified by a pathname
# relative to the source root. There are two exceptions
# 1. pathnames to a manifest file (beginning with '@include') are relative to the path
# of the directory containing the current manifest file
# 2. pathnames specified with @local are relative to the directory containing the current
# manifest file (typically "../files/<filename>").
fullManifestPath = os.path.abspath(manifestFileName)
manifestDirPath = os.path.dirname(fullManifestPath)
if not destdirExists:
os.makedirs(destdir, mode=0755)
nfiles = 0
nsubmanifests = 0
# Go through the manifest file line by line. Primitive parsing is ok our
# simple format, though it is a bit clunky.
lines = manifest.readlines()
for line in lines:
# ignore comments and blank lines
line = line.strip()
if line == "":
continue
if line[0] == "#":
continue
# @all@ expands to one file per architecture
if line.find("@all@") != -1:
if allArchitectures:
arches = ["linux64", "linux32", "darwin86", "win32"]
else:
arches = [myarch]
for a in arches:
newline = line.replace("@all@", a)
# Do the substitutions right here, rather than on the second
# pass, because this is where we know the architecture type
newline = doSubstitutions(newline, a)
log.debug("Adding rule '%s' based on '%s'", newline, line)
lines.append(newline)
continue
# @allunix@ expands to one file per unix architecture
elif line.find("@allunix@") != -1:
if allArchitectures:
arches = ["linux64", "linux32", "darwin86"]
elif myarch != "win32":
arches = [myarch]
else:
arches = list()
for a in arches:
newline = line.replace("@allunix@", a)
# Do the substitutions right here, rather than on the second
# pass, because this is where we know the architecture type
newline = doSubstitutions(newline, a)
lines.append(newline)
log.debug("Adding rule '%s' based on '%s'", newline, line)
continue
else:
line = doSubstitutions(line, myarch)
fields = line.split()
if len(fields) == 3:
# @arch <archname> <filename>
# This is an architecture-specific filename. If allArchitectures is false,
# it is only copied if myarch matches. If allArchitectures is true,
# then the file is copied for that specific architecture.
# archname may be a specific architecture or may be "win32"
# @include arch <manifestfile>
# @local <srcpath> <destpath>
if fields[0] == "@arch":
if allArchitectures or myarch == fields[1] or (fields[1] == "unix" and myarch != "win32"):
srcpath = fields[2]
fullsrcpath = os.path.join(srcdir, srcpath)
destpath = srcpath
else:
log.debug(indent + "Skipping line for different architecture: %s" % line)
continue
elif fields[0] == "@include":
arch = fields[1]
submanifest = fields[2]
if allArchitectures or (arch == "unix" and myarch != "win32") or arch == myarch:
# @todo consolidate submanifest code with @include below?
if not os.path.isabs(submanifest):
submanifest = os.path.abspath(os.path.join(manifestDirPath, submanifest))
log.info(indent + "Installing files from sub-manifest %s" % submanifest)
nsubmanifests = nsubmanifests + 1
try:
installFromManifest(
submanifest,
srcdir,
destdir,
level=level + 1,
overwrite=overwrite,
destdirExists=True,
allArchitectures=allArchitectures,
arch=myarch,
allowSymbolicLinks=allowSymbolicLinks,
)
except KeyboardInterrupt:
nfailures = nfailures + 1
raise Exception("Keyboard interrupt")
except:
nfailures = nfailures + 1
log.error(indent + "Installation from sub-manifest %s failed", submanifest)
continue
else:
log.debug("Skipping line for different achitecture: %s" % line)
continue
elif fields[0] == "@local":
srcpath = fields[1]
fullsrcpath = os.path.abspath(os.path.join(manifestDirPath, fields[1]))
destpath = fields[2]
else:
log.error(
indent
+ "Format error in manifest file. Lines with three fields must begin with @local or @arch"
)
nfailures = nfailures + 1
continue
elif len(fields) == 2:
# @include <manifestfile>
# Include another manifest file. Path is relative to current manifest file
#
# <srcpath> <destpath>
# Copy from srcpath in source tree to destpath in dest tree
if fields[0] == "@include":
submanifest = fields[1]
if not os.path.isabs(submanifest):
submanifest = os.path.abspath(os.path.join(manifestDirPath, submanifest))
log.info(indent + "Installing files from sub-manifest %s" % submanifest)
nsubmanifests = nsubmanifests + 1
try:
installFromManifest(
submanifest,
srcdir,
destdir,
level=level + 1,
overwrite=overwrite,
destdirExists=True,
allArchitectures=allArchitectures,
arch=myarch,
allowSymbolicLinks=allowSymbolicLinks,
)
except KeyboardInterrupt:
nfailures = nfailures + 1
raise Exception("Keyboard interrupt")
except:
nfailures = nfailures + 1
log.error(indent + "Installation from sub-manifest %s failed", submanifest)
continue
else:
srcpath = fields[0]
fullsrcpath = os.path.join(srcdir, srcpath)
destpath = fields[1]
elif len(fields) == 1:
# <path>
# Copy from source to dest, using same relative path
srcpath = fields[0]
destpath = srcpath
fullsrcpath = os.path.join(srcdir, srcpath)
else:
log.error(indent + "Format error in manifest file. ")
log.error(indent + "Bad line: '%s'" % line)
nfailures = nfailures + 1
continue
if not os.path.exists(fullsrcpath):
log.error(indent + "Source path %s does not exist" % fullsrcpath)
nfailures = nfailures + 1
continue
fulldestpath = os.path.join(destdir, destpath)
fulldestdir = os.path.dirname(fulldestpath)
if not os.path.exists(fulldestdir):
log.debug(indent + "Creating directory %s" % fulldestdir)
os.makedirs(fulldestdir, 0755)
if os.path.exists(fulldestpath):
if not overwrite:
log.error(indent + "File %s already exists and overwrite flag is not set" % (fulldestpath))
nfailures = nfailures + 1
else:
log.debug(indent + "File %s already exists -- overwriting" % (fulldestpath))
if overwrite and not os.path.exists(fulldestpath):
log.warning(indent + "File %s does not exist and overwrite flag is set (not fatal)" % (fulldestpath))
# Complain if the file is a symbolic link but ok if allowSymbolicLinks is specified
if os.path.islink(fullsrcpath):
if allowSymbolicLinks:
target = os.readlink(fullsrcpath)
if os.path.isabs(target):
fullsrcpath = target
else:
fullsrcpath = os.path.normpath(os.path.join(os.path.dirname(fullsrcpath), target))
log.warn(indent + "Copying target of symbolic link %s" % fullsrcpath)
else:
log.error(indent + "Skipping symbolic link %s" % fullsrcpath)
nfailures = nfailures + 1
continue
# copy the file
log.debug(indent + "Copying %s -> %s" % (srcpath, destpath))
nfiles = nfiles + 1
try:
shutil.copy2(fullsrcpath, fulldestpath)
except Exception, e:
nfailures = nfailures + 1
log.error(indent + "Unable to copy %s to %s. Error: %s", srcpath, destpath, e)
### while(newline != "")
if nfailures == 0:
if nsubmanifests == 0:
log.info(indent + "Done. %d files copied" % (nfiles))
else:
log.info(indent + "Done. %d files copied %s submanifests processed" % (nfiles, nsubmanifests))
else:
log.info(
indent + "Done. %d copy attempts; %d failures; %d submanifests" % (nfiles, nfailures, nsubmanifests)
)
raise Exception("install from manifest failed")
import os
import sys
import re
from pprint import pprint as pp
import operator
scriptDir = os.path.dirname(__file__)
externalDir = os.path.abspath(os.path.join(scriptDir, '../../external'))
pybuildDir = os.path.abspath(
os.path.join(scriptDir, '../../build_system/pybuild'))
langPyDir = os.path.abspath(os.path.join(scriptDir, '../py'))
# Get the architecture first
sys.path = [os.path.join(pybuildDir)] + sys.path
import arch
a = arch.getArch()
# Get yaml from external (it is not installed)
pythonVersion = '%d.%d' % sys.version_info[:2]
sys.path = [
os.path.join(externalDir, a, 'lib/python%s/site-packages' % pythonVersion)
] + sys.path
import yaml
# Restore sys.path
sys.path = sys.path[2:]
# The directory that contains the NuPIC 2 C++ object model
baseDir = os.path.abspath(os.path.join(scriptDir, '../../nta/engine'))
assert os.path.isdir(baseDir)
def buildit(rootdir, srcdir, builddir, installdir, svnpath, stamp, revision, datestamp, assertions, update, nprocs=1):
"""
Do a full checkout and build
"""
arch = getArch()
log.info("Building for architecture " + arch)
#
# If source dir already exists, we're doing an incremental checkout. If not, we're
# doing a full checkout.
#
incremental = True
if not os.path.exists(srcdir):
if not update:
log.error("No svn update requested, but source dir '%s' does not exist.", srcdir)
raise Exception()
else:
incremental = False
log.info("Source directory %s does not exist. Will do a full checkout", srcdir)
else:
if not os.path.isdir(srcdir):
log.error("Source directory %s is not a directory.", srcdir)
raise Exception()
log.debug("Creating release directory %s", installdir)
utils.createDir(installdir, True)
utils.changeDir(os.path.join(installdir, os.pardir))
buildfilename = os.path.join(installdir, ".buildinfo")
#
# Bring source tree up to date if requested
#
if update:
log.info("Bringing source tree up to date at svnpath %s revision %s", svnpath, revision)
svn.checkout(srcdir, repo=svnpath, revision=revision, incremental=incremental, keepLocalMods=False)
else:
log.warn("Skipping bringing up to date. Build will be tainted")
utils.touchFile(os.path.join(installdir, "TAINTED_BUILD"))
#
# Revision may not be a number. May be "HEAD" or "PREV" or something else, so
# retrieve it from svn
#
revisionnum = svn.getRevision(srcdir)
#
# place a temporary file in the install dir to indicate a build in progress
# the file is removed at the end of the build process
#
build_in_progress_filename = os.path.join(installdir,"UNFINISHED_BUILD." + arch)
utils.touchFile(build_in_progress_filename)
log.debug("Updating build description file " + buildfilename)
try:
f = open(buildfilename, mode = "a")
print >> f, "============================="
print >> f, "Timestamp: " + datestamp
print >> f, "Arch: " + arch
print >> f, "Buildhost: " + platform.node()
print >> f, "Svnpath: " + svnpath
print >> f, "Revision: " + str(revisionnum)
print >> f, "Rootdir: " + rootdir
print >> f, "Srcdir: " + srcdir
print >> f, "Stamp: " + stamp
print >> f, "Assertions:" + str(assertions)
print >> f, "Update: " + str(update)
print >> f, ""
f.close()
except Exception, e:
raise Exception("Error writing to build description file '%s'", buildfilename)
def buildCustomerRelease(releaseType, releaseSpecDir, releaseTarball, binaryTarball, prefix=None, nprocs=1,parentTmpDir=None):
"""From the given source release tarball, compile and verify that the
build product is correct.
@param releaseType -- one of the source release types
@param releaseSpecDir -- trunk/release directory containing manifest files for this release
@param releaseTarball -- tarball containing the release to be built.
@param binaryTarball -- tarball containing the binary release. This must be provided
when building the tools source release, which requires prebuilt libraries that
only exist in the binary release. These libraries are copied in to the source
release tree prior to building the source release.
Throws an exception if the build fails or the build product does not
exactly match the manifest file.
If prefix is not None, install the build product into the prefix directory
No return value"""
log.info("Building customer %s release", releaseType)
origDir = os.getcwd()
if not releaseType in sourceReleaseTypes:
raise Exception("buildCustomerRelease: unknown release type %s" % releaseType)
tarballBasename = os.path.basename(releaseTarball)
(dirName, ext) = os.path.splitext(tarballBasename)
manifest = os.path.abspath(os.path.join(releaseSpecDir,
"release",
releaseType+"_release",
"manifests",
"build_output.manifest"))
tempdir = utils.createTemporaryDirectory("build_"+releaseType, parentDir=parentTmpDir)
installdir = os.path.join(tempdir, "install")
utils.createDir(installdir)
builddir = os.path.join(tempdir, "build")
utils.createDir(builddir)
srcdir = utils.extractArchive(releaseTarball, tempdir)
# Use the precompiled runtime library for the full source build.
if releaseType in releasesRequiringSPNupic:
if binaryTarball is None:
raise Exception("No binary tarball provided when building tools release, which requires prebuilt libraries")
binarydir = utils.extractArchive(binaryTarball, tempdir)
log.info("Copying libruntime.a from binary release into source release")
if getArch() != "win32":
smartCopy(os.path.join(binarydir, "lib", "libruntime.a"),
os.path.join(srcdir, "external", getArch(), "lib", "libruntime.a"))
smartCopy(os.path.join(binarydir, "lib", "libnode.a"),
os.path.join(srcdir, "external", getArch(), "lib", "libnode.a"))
smartCopy(os.path.join(binarydir, "lib", "libipcserial.a"),
os.path.join(srcdir, "external", getArch(), "lib", "libipcserial.a"))
else:
smartCopy(os.path.join(binarydir, "lib", "release", "runtime.lib"),
os.path.join(srcdir, "external", getArch(), "lib", "runtime.lib"))
smartCopy(os.path.join(binarydir, "lib", "release", "node.lib"),
os.path.join(srcdir, "external", getArch(), "lib", "node.lib"))
smartCopy(os.path.join(binarydir, "lib", "release", "ipcserial.lib"),
os.path.join(srcdir, "external", getArch(), "lib", "ipcserial.lib"))
build.build(srcdir, builddir, installdir, assertions=False, customerRelease=True, nprocs=nprocs)
# To verify the build product, we install from manifest and then
# confirm that the installed tree has the same number of files as the original tree.
# An exception in the installFromManifest phase usually means there is a file in
# the manifest that was not built. Then if the number of files is the same, it usually
# means that a file was built that was not in the manifest.
tempInstallDir = os.path.join(tempdir, "temp_install")
installFromManifest(manifest, installdir, tempInstallDir,
level=0, overwrite=False, destdirExists=False)
installedFiles = []
for root,dirs,files in os.walk(tempInstallDir):
installedFiles += [os.path.join(root, f) for f in files]
# normalize the list of files
prefixLen = len(tempInstallDir)
installedFiles = [f[prefixLen:] for f in installedFiles]
installedFiles = sorted(installedFiles)
builtFiles = []
for root,dirs,files in os.walk(installdir):
builtFiles += [os.path.join(root, f) for f in files]
# normalize the list of files
prefixLen = len(installdir)
builtFiles = [f[prefixLen:] for f in builtFiles]
builtFiles = sorted(builtFiles)
if len(installedFiles) != len(builtFiles):
log.error("Manifest error when building %s release", releaseType)
log.error("Built files: %d Files in manifest: %d", len(builtFiles), len(installedFiles))
# we know that every file in installedFiles is also in builtFiles
# because installed dir was created from built dir
for f in installedFiles:
builtFiles.remove(f)
for f in builtFiles:
log.error("File '%s' installed but not in build_output.manifest" % f)
raise Exception("Error building %s release -- file(s) missing from build_output.manifest" % releaseType)
else:
log.info("Release %s built successfully. Build produces %d files", releaseType, len(builtFiles))
# copy all built files from directory where built into the the prefix dir, if specified
if prefix is not None:
log.info("Installing build products from '%s' release into binary release at '%s'",
releaseType, prefix)
installFromManifest(manifest, installdir, prefix,
level=0, overwrite=True, destdirExists=True)
# clean up
os.chdir(origDir)
utils.remove(tempdir)
"/opt/buildtools/bin",
"/opt/python%s/bin" % pythonVersion,
"/usr/local/bin",
"/usr/bin",
"/bin")
rootdir = os.path.expanduser("~/releases")
builddir = ""
srcdir = ""
svnpath = "trunk"
revision = "HEAD"
assertions = False;
update = True;
dotests = True;
dobuild = True;
dorelease = True;
arch = getArch()
datestamp = time.strftime("%Y-%m-%d-%H%M")
stamp = datestamp + "." + os.path.basename(svnpath)
options = [
"rootdir=",
"srcdir=",
"builddir=",
"svnpath=",
"stamp=",
"revision=",
"disable-test",
"enable-assertions",
"disable-update",
"disable-build",