Exemplo n.º 1
0
def installFromList(filename, srcdir, destdir, overwrite=True, link=False, optimizeCopy=False,developerOnly=False):
  """Takes a list of src/dest files and directories. Each line has a source,
  and a destination. Lines starting with "#" are comments.
  This method is use for the post-build step on both unix
  and windows.
  A line may be preceded by "@unix" or "@win32" to indicate that
  it should only be copied for that architecture
  """
  substitutions = dict(
    PythonVersion="%d.%d" % (sys.version_info[0], sys.version_info[1]),
    arch=getArch())

  text = open(filename).read()
  t = string.Template(text)
  text = t.substitute(substitutions)
  lines = text.split("\n")

  # strip out comments and newlines
  lines = [l.strip() for l in lines]
  lines = [l for l in lines if l != ""]
  lines = [l for l in lines if l[0] != "#"]

  for line in lines:
    elements = line.split()

    arch = elements[0]
    if len(elements) == 3:
      if arch == "@win32":
        if getArch() != "win32": continue
      elif arch == "@unix":
        if getArch() == "win32": continue
      elif arch == "@darwin86":
        if getArch() != "darwin86": continue
      elif arch == "@darwin64":
        if getArch() != "darwin64": continue
      elif arch == "@darwin":
        # both darwin86 and darwin64
        if getArch() not in ["darwin86", "darwin64"]: continue
      elif arch == "@linux64":
        if getArch() != "linux64": continue
      elif arch == "@linux32":
        if getArch() != "linux32": continue
      elif arch == "@linux32arm":
        if getArch() != "linux32arm": continue
      elif arch == "@linux32armv7":
        if getArch() != "linux32armv7": continue
      else:
        raise Exception("Unknown architecture type %s in file %s" % (arch, filename))
      src = elements[1]
      dest = elements[2]
    elif len(elements) == 2:
      src = elements[0]
      dest = elements[1]
    else:
      raise Exception("Line in unknown format: '%s'" % line)
    src = os.path.join(srcdir, src)
    dest = os.path.join(destdir, dest)
    smartCopy(src, dest, overwrite=overwrite, link=link, optimizeCopy=optimizeCopy, developerOnly=developerOnly)
Exemplo n.º 2
0
def installFromList(filename, srcdir, destdir, overwrite=True, link=False, optimizeCopy=False,developerOnly=False):
  """Takes a list of src/dest files and directories. Each line has a source,
  and a destination. Lines starting with "#" are comments.
  This method is use for the post-build step on both unix
  and windows.
  A line may be preceded by "@unix" or "@win32" to indicate that
  it should only be copied for that architecture
  """
  substitutions = dict(
    PythonVersion="%d.%d" % (sys.version_info[0], sys.version_info[1]),
    arch=getArch())

  text = open(filename).read()
  t = string.Template(text)
  text = t.substitute(substitutions)
  lines = text.split("\n")

  # strip out comments and newlines
  lines = [l.strip() for l in lines]
  lines = [l for l in lines if l != ""]
  lines = [l for l in lines if l[0] != "#"]

  for line in lines:
    elements = line.split()

    arch = elements[0]
    if len(elements) == 3:
      if arch == "@win32":
        if getArch() != "win32": continue
      elif arch == "@unix":
        if getArch() == "win32": continue
      elif arch == "@darwin86":
        if getArch() != "darwin86": continue
      elif arch == "@darwin64":
        if getArch() != "darwin64": continue
      elif arch == "@darwin":
        # both darwin86 and darwin64
        if getArch() not in ["darwin86", "darwin64"]: continue
      elif arch == "@linux64":
        if getArch() != "linux64": continue
      elif arch == "@linux32":
        if getArch() != "linux32": continue
      else:
        raise Exception("Unknown architecture type %s in file %s" % (arch, filename))
      src = elements[1]
      dest = elements[2]
    elif len(elements) == 2:
      src = elements[0]
      dest = elements[1]
    else:
      raise Exception("Line in unknown format: '%s'" % line)
    src = os.path.join(srcdir, src)
    dest = os.path.join(destdir, dest)
    smartCopy(src, dest, overwrite=overwrite, link=link, optimizeCopy=optimizeCopy, developerOnly=developerOnly)
Exemplo n.º 3
0
def createCustomerRelease(releaseType, releaseSpecDir, destDir, fromDir, releaseName, allArchitectures=True, allArchiveTypes=False, allowSymbolicLinks=False, parentTmpDir=None):
  """Create a customer release tarball in destdir from the
  directories engReleaseDir and releaseSpecDir
  releaseType is one of biaryReleaseTypes or sourceReleaseTypes
  fromDir is engineering build for the binary release, 
             source dir for a source release
  releaseSpecDir is directory containing manifests
  Returns a path to the tarball
  Throws an exception on error"""

  if not os.path.exists(destDir):
    utils.createDir(destDir)

  if releaseType in binaryReleaseTypes:
    isSourceRelease = False
  elif releaseType in sourceReleaseTypes:
    isSourceRelease = True
  else:
    raise Exception("Unknown release type '%s'" % releaseType)
    
  if not isSourceRelease:
    allArchitectures=False

  manifestFile = os.path.join(releaseSpecDir, releaseType + "_release", 
                              "manifests", releaseType + "_release.manifest")

  log.info("Creating customer release of type %s", releaseType)
  log.info("specDir: %s", releaseSpecDir)
  log.info("fromDir: %s", fromDir)
  tempdir = utils.createTemporaryDirectory("assemble_release", parentDir=parentTmpDir)
  try:
    releaseImageDir = os.path.join(tempdir, releaseName)
    installFromManifest(manifestFile, fromDir, releaseImageDir, 
                        level=0, overwrite=False, destdirExists=False, 
                        allArchitectures=allArchitectures, allowSymbolicLinks=allowSymbolicLinks)
    if isSourceRelease:
      # run autogen equivalent on source trees
      build.prepareSourceTree(releaseImageDir, customerRelease=True)

    # Need to create both zip and tgz source releases on the master build system
    if allArchiveTypes == True:
      # sanity checks
      if not isSourceRelease:
        raise Exception("Attempt to create both zip and tar file for a binary release. Probably an error")
      if getArch() == "win32":
        raise Exception("Attempt to create .tgz source release on a windows system. Would not include autogen.sh")
      zipFile = utils.createArchive(releaseImageDir, destDir, type="zip")
      tarFile = utils.createArchive(releaseImageDir, destDir, type="tar")
      return (tarFile, zipFile)
    else:
      release = utils.createArchive(releaseImageDir, destDir)
      return release
  finally:
    utils.remove(tempdir)
Exemplo n.º 4
0
    def __init__(self, addr):

        self.calls = {}
        self.rets = {}
        self.callDsts = {}
        self.funcAddrs = {}
        ah = getArch(ARCH)
        self.BIT = ah.getBit() // 4
        self.RET = ah.getRets()

        tracerFile = addr.split('/')[-1] + '.tracer'
        if (not self.load(tracerFile, addr)):
            self.analyze(addr)
            self.save(tracerFile, addr)
Exemplo n.º 5
0
    def __init__(self, p, funcName):
        self.p = p
        self.func = funcName
        self.sourceFolder = SOURCEFOLDER
        ah = getArch(ARCH)
        self.eip = ah.getEip()
        self.max_brkTime = 30  # If a breakpoint is triggered more than this number, it will be removed
        self.ed = '394743516231415926'  # A random number as end of output of a session

        print('loading %s' % ASMFILE)
        cur_time = time.clock()
        self.asm = asmAnalyser(ASMFILE)
        print('finished loading, cost %ss' % str(time.clock() - cur_time))
        if (not self.asm.funcExist(funcName)):
            print('ERROR! Function %s not exist!' % funcName)
            return
        p.stdin.write(bytes('target remote:%s\n' % GDBPORT, encoding='utf8'))
Exemplo n.º 6
0
def getReleaseName(stamp, releaseType, debug=False):
  arch = getArch()
  if debug:
    prefix = "nupic-debug"
  else:
    prefix = "nupic"

  if releaseType == "binary":
    releaseName = "%s-%s-%s" % (prefix, stamp, arch)
  elif releaseType == "vitamind":
    releaseName = "%s-vitamind-%s-%s" % (prefix, stamp, arch)
  elif releaseType == "npp":
    releaseName = "%s-npp-%s-%s" % (prefix, stamp, arch)
  elif releaseType in sourceReleaseTypes:
    releaseName = "%s-%s-%s" % (prefix, stamp, releaseType)
  else:
    raise Exception("getReleaseName: unknown release type '%s'" % releaseType)

  return releaseName
Exemplo n.º 7
0
import os
import sys
import re
from pprint import pprint as pp
import operator

scriptDir = os.path.dirname(__file__)
externalDir = os.path.abspath(os.path.join(scriptDir, '../../external'))
pybuildDir = os.path.abspath(os.path.join(scriptDir, '../../build_system/pybuild'))
langPyDir = os.path.abspath(os.path.join(scriptDir, '../py'))

# Get the architecture first
sys.path = [os.path.join(pybuildDir)] + sys.path
import arch
a = arch.getArch()

# Get yaml from external (it is not installed)
pythonVersion = '%d.%d' % sys.version_info[:2]
sys.path = [os.path.join(externalDir, a,
            'lib/python%s/site-packages' % pythonVersion)] + sys.path
import yaml

# Restore sys.path
sys.path = sys.path[2:]

# The directory that contains the NuPIC 2 C++ object model
baseDir = os.path.abspath(os.path.join(scriptDir, '../../nta/engine'))
assert os.path.isdir(baseDir)

def getBlock(lines):
Exemplo n.º 8
0
def installFromManifest(manifestFileName, srcdir, destdir, level, overwrite, destdirExists, allArchitectures=False, arch=None, allowSymbolicLinks=False):
  # use an additional two spaces of indentation for each submanifest level
  indent = ""
  for i in xrange(0, level):
    indent = indent + "    "

  nfailures = 0
  if arch is None:
    myarch = getArch()
  else:
    myarch = arch

  log.info(indent + "Installing from %s" % os.path.basename(manifestFileName))

  try:

    try:
      manifest = open(manifestFileName)
    except:
      log.error(indent + "Unable to open manifest file '%s'" % manifestFileName)
      raise

    if not os.path.isdir(srcdir):
      nfailures = nfailures + 1
      raise Exception(indent + 'Source directory "%s" does not exist or is not a directory' % srcdir)

    if os.path.exists(destdir) and not destdirExists:
      raise Exception(indent + 'Destination directory "%s" already exists. Delete before installing manifest.' % destdir)
  
    #
    # by default, files in the manifest file are specified by a pathname
    # relative to the source root. There are two exceptions
    # 1. pathnames to a manifest file (beginning with '@include') are relative to the path
    # of the directory containing the current manifest file
    # 2. pathnames specified with @local are relative to the directory containing the current 
    #    manifest file (typically "../files/<filename>").

    fullManifestPath = os.path.abspath(manifestFileName)
    manifestDirPath = os.path.dirname(fullManifestPath)

    if not destdirExists:
      os.makedirs(destdir, mode=0755)

    nfiles = 0
    nsubmanifests = 0

    # Go through the manifest file line by line. Primitive parsing is ok our
    # simple format, though it is a bit clunky. 
    lines = manifest.readlines()
    for line in lines:
      # ignore comments and blank lines
      line = line.strip()
      if line == "":
        continue
      if line[0] == "#":
        continue

      # @all@ expands to one file per architecture
      if line.find("@all@") != -1:
        if allArchitectures:
          arches = ["linux64", "linux32", "darwin86", "win32"]
        else:
          arches = [myarch]
        for a in arches:
          newline = line.replace("@all@", a)
          # Do the substitutions right here, rather than on the second
          # pass, because this is where we know the architecture type
          newline = doSubstitutions(newline, a)
          log.debug("Adding rule '%s' based on '%s'", newline, line)
          lines.append(newline)
        continue
      # @allunix@ expands to one file per unix architecture
      elif line.find("@allunix@") != -1:
        if allArchitectures:
          arches = ["linux64", "linux32", "darwin86"]
        elif myarch != "win32":
          arches = [myarch]
        else:
          arches = list()
        for a in arches:
          newline = line.replace("@allunix@", a)
          # Do the substitutions right here, rather than on the second
          # pass, because this is where we know the architecture type
          newline = doSubstitutions(newline, a)
          lines.append(newline)
          log.debug("Adding rule '%s' based on '%s'",newline, line)
        continue
      else:
        line = doSubstitutions(line, myarch)

      fields = line.split()

      if len(fields) == 3:
        # @arch <archname> <filename>
        # This is an architecture-specific filename. If allArchitectures is false, 
        # it is only copied if myarch matches. If allArchitectures is true, 
        # then the file is copied for that specific architecture. 
        # archname may be a specific architecture or may be "win32"
        # @include arch <manifestfile>
        # @local <srcpath> <destpath>
        if fields[0] == "@arch":
          if allArchitectures or \
             myarch == fields[1] or \
             (fields[1] == "unix" and myarch != "win32"):
            srcpath = fields[2]
            fullsrcpath = os.path.join(srcdir, srcpath)
            destpath = srcpath
          else:
            log.debug(indent + "Skipping line for different architecture: %s" % line)
            continue
        elif fields[0] == "@include":
          arch = fields[1]
          submanifest = fields[2]
          if allArchitectures or (arch  == "unix" and myarch != "win32") or arch == myarch:
            # @todo consolidate submanifest code with @include below?
            if not os.path.isabs(submanifest):
              submanifest = os.path.abspath(os.path.join(manifestDirPath, submanifest))
            log.info(indent + "Installing files from sub-manifest %s" % submanifest)
            nsubmanifests = nsubmanifests + 1
            try:
              installFromManifest(submanifest, srcdir, destdir, 
                                  level=level+1, overwrite=overwrite, 
                                  destdirExists=True, allArchitectures=allArchitectures, 
                                  arch=myarch, allowSymbolicLinks=allowSymbolicLinks)
            except KeyboardInterrupt:
              nfailures = nfailures + 1
              raise Exception("Keyboard interrupt")
            except:
              nfailures = nfailures + 1
              log.error(indent + "Installation from sub-manifest %s failed", submanifest)
              continue
          else:
            log.debug("Skipping line for different achitecture: %s" % line)
          continue
        elif fields[0] == "@local":
          srcpath = fields[1]
          fullsrcpath = os.path.abspath(os.path.join(manifestDirPath, fields[1]))
          destpath = fields[2]
        else:
          log.error(indent + 'Format error in manifest file. Lines with three fields must begin with @local or @arch')
          nfailures = nfailures+1
          continue

      elif len(fields) == 2:
        # @include <manifestfile>
        # Include another manifest file. Path is relative to current manifest file
        #
        # <srcpath> <destpath>
        # Copy from srcpath in source tree to destpath in dest tree
        if fields[0] == "@include":
          submanifest = fields[1]
          if not os.path.isabs(submanifest):
            submanifest = os.path.abspath(os.path.join(manifestDirPath, submanifest))
          log.info(indent + "Installing files from sub-manifest %s" % submanifest)
          nsubmanifests = nsubmanifests + 1
          try:
            installFromManifest(submanifest, srcdir, destdir, 
                                level=level+1, overwrite=overwrite, 
                                destdirExists=True, allArchitectures=allArchitectures,
                                arch=myarch,allowSymbolicLinks=allowSymbolicLinks)
          except KeyboardInterrupt:
            nfailures = nfailures + 1
            raise Exception("Keyboard interrupt")
          except:
            nfailures = nfailures + 1
            log.error(indent + "Installation from sub-manifest %s failed", submanifest)
          continue
        else:
          srcpath = fields[0]
          fullsrcpath = os.path.join(srcdir, srcpath)
          destpath = fields[1]
      elif len(fields) == 1:
        # <path>
        # Copy from source to dest, using same relative path
        srcpath = fields[0]
        destpath = srcpath
        fullsrcpath = os.path.join(srcdir, srcpath)
      else:
        log.error(indent + "Format error in manifest file. ")
        log.error(indent + "Bad line: '%s'" % line)
        nfailures = nfailures + 1
        continue

      if not os.path.exists(fullsrcpath):
        log.error(indent + "Source path %s does not exist" % fullsrcpath)
        nfailures = nfailures + 1
        continue

      fulldestpath = os.path.join(destdir, destpath)
      fulldestdir = os.path.dirname(fulldestpath)
      if not os.path.exists(fulldestdir):
        log.debug(indent + "Creating directory %s" % fulldestdir)
        os.makedirs(fulldestdir, 0755)

      if os.path.exists(fulldestpath):
        if not overwrite:
          log.error(indent + "File %s already exists and overwrite flag is not set" % (fulldestpath))
          nfailures = nfailures + 1
        else:
          log.debug(indent + "File %s already exists -- overwriting" % (fulldestpath))

      if overwrite and not os.path.exists(fulldestpath):
        log.warning(indent + "File %s does not exist and overwrite flag is set (not fatal)" % (fulldestpath))

      # Complain if the file is a symbolic link but ok if allowSymbolicLinks is specified
      if os.path.islink(fullsrcpath):
        if allowSymbolicLinks:
          target = os.readlink(fullsrcpath)
          if os.path.isabs(target):
            fullsrcpath = target
          else:
            fullsrcpath = os.path.normpath(os.path.join(os.path.dirname(fullsrcpath), target))
          log.warn(indent + "Copying target of symbolic link %s" % fullsrcpath)
        else:
          log.error(indent + "Skipping symbolic link %s" % fullsrcpath)
          nfailures = nfailures + 1
          continue

      # copy the file
      log.debug(indent + "Copying %s -> %s" % (srcpath, destpath))
      nfiles = nfiles + 1
      try:
        shutil.copy2(fullsrcpath, fulldestpath)
      except Exception, e:
        nfailures = nfailures + 1
        log.error(indent + "Unable to copy %s to %s. Error: %s", srcpath, destpath, e)
      
      ### while(newline != "")

    if nfailures == 0:
      if nsubmanifests == 0:
        log.info(indent + "Done. %d files copied" % (nfiles))
      else:
        log.info(indent + "Done. %d files copied %s submanifests processed" % (nfiles, nsubmanifests))
    else:
      log.info(indent + "Done. %d copy attempts; %d failures; %d submanifests" % (nfiles, nfailures, nsubmanifests))
      raise Exception("install from manifest failed")
Exemplo n.º 9
0
def installFromManifest(
    manifestFileName,
    srcdir,
    destdir,
    level,
    overwrite,
    destdirExists,
    allArchitectures=False,
    arch=None,
    allowSymbolicLinks=False,
):
    # use an additional two spaces of indentation for each submanifest level
    indent = ""
    for i in xrange(0, level):
        indent = indent + "    "

    nfailures = 0
    if arch is None:
        myarch = getArch()
    else:
        myarch = arch

    log.info(indent + "Installing from %s" % os.path.basename(manifestFileName))

    try:

        try:
            manifest = open(manifestFileName)
        except:
            log.error(indent + "Unable to open manifest file '%s'" % manifestFileName)
            raise

        if not os.path.isdir(srcdir):
            nfailures = nfailures + 1
            raise Exception(indent + 'Source directory "%s" does not exist or is not a directory' % srcdir)

        if os.path.exists(destdir) and not destdirExists:
            raise Exception(
                indent + 'Destination directory "%s" already exists. Delete before installing manifest.' % destdir
            )

        #
        # by default, files in the manifest file are specified by a pathname
        # relative to the source root. There are two exceptions
        # 1. pathnames to a manifest file (beginning with '@include') are relative to the path
        # of the directory containing the current manifest file
        # 2. pathnames specified with @local are relative to the directory containing the current
        #    manifest file (typically "../files/<filename>").

        fullManifestPath = os.path.abspath(manifestFileName)
        manifestDirPath = os.path.dirname(fullManifestPath)

        if not destdirExists:
            os.makedirs(destdir, mode=0755)

        nfiles = 0
        nsubmanifests = 0

        # Go through the manifest file line by line. Primitive parsing is ok our
        # simple format, though it is a bit clunky.
        lines = manifest.readlines()
        for line in lines:
            # ignore comments and blank lines
            line = line.strip()
            if line == "":
                continue
            if line[0] == "#":
                continue

            # @all@ expands to one file per architecture
            if line.find("@all@") != -1:
                if allArchitectures:
                    arches = ["linux64", "linux32", "darwin86", "win32"]
                else:
                    arches = [myarch]
                for a in arches:
                    newline = line.replace("@all@", a)
                    # Do the substitutions right here, rather than on the second
                    # pass, because this is where we know the architecture type
                    newline = doSubstitutions(newline, a)
                    log.debug("Adding rule '%s' based on '%s'", newline, line)
                    lines.append(newline)
                continue
            # @allunix@ expands to one file per unix architecture
            elif line.find("@allunix@") != -1:
                if allArchitectures:
                    arches = ["linux64", "linux32", "darwin86"]
                elif myarch != "win32":
                    arches = [myarch]
                else:
                    arches = list()
                for a in arches:
                    newline = line.replace("@allunix@", a)
                    # Do the substitutions right here, rather than on the second
                    # pass, because this is where we know the architecture type
                    newline = doSubstitutions(newline, a)
                    lines.append(newline)
                    log.debug("Adding rule '%s' based on '%s'", newline, line)
                continue
            else:
                line = doSubstitutions(line, myarch)

            fields = line.split()

            if len(fields) == 3:
                # @arch <archname> <filename>
                # This is an architecture-specific filename. If allArchitectures is false,
                # it is only copied if myarch matches. If allArchitectures is true,
                # then the file is copied for that specific architecture.
                # archname may be a specific architecture or may be "win32"
                # @include arch <manifestfile>
                # @local <srcpath> <destpath>
                if fields[0] == "@arch":
                    if allArchitectures or myarch == fields[1] or (fields[1] == "unix" and myarch != "win32"):
                        srcpath = fields[2]
                        fullsrcpath = os.path.join(srcdir, srcpath)
                        destpath = srcpath
                    else:
                        log.debug(indent + "Skipping line for different architecture: %s" % line)
                        continue
                elif fields[0] == "@include":
                    arch = fields[1]
                    submanifest = fields[2]
                    if allArchitectures or (arch == "unix" and myarch != "win32") or arch == myarch:
                        # @todo consolidate submanifest code with @include below?
                        if not os.path.isabs(submanifest):
                            submanifest = os.path.abspath(os.path.join(manifestDirPath, submanifest))
                        log.info(indent + "Installing files from sub-manifest %s" % submanifest)
                        nsubmanifests = nsubmanifests + 1
                        try:
                            installFromManifest(
                                submanifest,
                                srcdir,
                                destdir,
                                level=level + 1,
                                overwrite=overwrite,
                                destdirExists=True,
                                allArchitectures=allArchitectures,
                                arch=myarch,
                                allowSymbolicLinks=allowSymbolicLinks,
                            )
                        except KeyboardInterrupt:
                            nfailures = nfailures + 1
                            raise Exception("Keyboard interrupt")
                        except:
                            nfailures = nfailures + 1
                            log.error(indent + "Installation from sub-manifest %s failed", submanifest)
                            continue
                    else:
                        log.debug("Skipping line for different achitecture: %s" % line)
                    continue
                elif fields[0] == "@local":
                    srcpath = fields[1]
                    fullsrcpath = os.path.abspath(os.path.join(manifestDirPath, fields[1]))
                    destpath = fields[2]
                else:
                    log.error(
                        indent
                        + "Format error in manifest file. Lines with three fields must begin with @local or @arch"
                    )
                    nfailures = nfailures + 1
                    continue

            elif len(fields) == 2:
                # @include <manifestfile>
                # Include another manifest file. Path is relative to current manifest file
                #
                # <srcpath> <destpath>
                # Copy from srcpath in source tree to destpath in dest tree
                if fields[0] == "@include":
                    submanifest = fields[1]
                    if not os.path.isabs(submanifest):
                        submanifest = os.path.abspath(os.path.join(manifestDirPath, submanifest))
                    log.info(indent + "Installing files from sub-manifest %s" % submanifest)
                    nsubmanifests = nsubmanifests + 1
                    try:
                        installFromManifest(
                            submanifest,
                            srcdir,
                            destdir,
                            level=level + 1,
                            overwrite=overwrite,
                            destdirExists=True,
                            allArchitectures=allArchitectures,
                            arch=myarch,
                            allowSymbolicLinks=allowSymbolicLinks,
                        )
                    except KeyboardInterrupt:
                        nfailures = nfailures + 1
                        raise Exception("Keyboard interrupt")
                    except:
                        nfailures = nfailures + 1
                        log.error(indent + "Installation from sub-manifest %s failed", submanifest)
                    continue
                else:
                    srcpath = fields[0]
                    fullsrcpath = os.path.join(srcdir, srcpath)
                    destpath = fields[1]
            elif len(fields) == 1:
                # <path>
                # Copy from source to dest, using same relative path
                srcpath = fields[0]
                destpath = srcpath
                fullsrcpath = os.path.join(srcdir, srcpath)
            else:
                log.error(indent + "Format error in manifest file. ")
                log.error(indent + "Bad line: '%s'" % line)
                nfailures = nfailures + 1
                continue

            if not os.path.exists(fullsrcpath):
                log.error(indent + "Source path %s does not exist" % fullsrcpath)
                nfailures = nfailures + 1
                continue

            fulldestpath = os.path.join(destdir, destpath)
            fulldestdir = os.path.dirname(fulldestpath)
            if not os.path.exists(fulldestdir):
                log.debug(indent + "Creating directory %s" % fulldestdir)
                os.makedirs(fulldestdir, 0755)

            if os.path.exists(fulldestpath):
                if not overwrite:
                    log.error(indent + "File %s already exists and overwrite flag is not set" % (fulldestpath))
                    nfailures = nfailures + 1
                else:
                    log.debug(indent + "File %s already exists -- overwriting" % (fulldestpath))

            if overwrite and not os.path.exists(fulldestpath):
                log.warning(indent + "File %s does not exist and overwrite flag is set (not fatal)" % (fulldestpath))

            # Complain if the file is a symbolic link but ok if allowSymbolicLinks is specified
            if os.path.islink(fullsrcpath):
                if allowSymbolicLinks:
                    target = os.readlink(fullsrcpath)
                    if os.path.isabs(target):
                        fullsrcpath = target
                    else:
                        fullsrcpath = os.path.normpath(os.path.join(os.path.dirname(fullsrcpath), target))
                    log.warn(indent + "Copying target of symbolic link %s" % fullsrcpath)
                else:
                    log.error(indent + "Skipping symbolic link %s" % fullsrcpath)
                    nfailures = nfailures + 1
                    continue

            # copy the file
            log.debug(indent + "Copying %s -> %s" % (srcpath, destpath))
            nfiles = nfiles + 1
            try:
                shutil.copy2(fullsrcpath, fulldestpath)
            except Exception, e:
                nfailures = nfailures + 1
                log.error(indent + "Unable to copy %s to %s. Error: %s", srcpath, destpath, e)

            ### while(newline != "")

        if nfailures == 0:
            if nsubmanifests == 0:
                log.info(indent + "Done. %d files copied" % (nfiles))
            else:
                log.info(indent + "Done. %d files copied %s submanifests processed" % (nfiles, nsubmanifests))
        else:
            log.info(
                indent + "Done. %d copy attempts; %d failures; %d submanifests" % (nfiles, nfailures, nsubmanifests)
            )
            raise Exception("install from manifest failed")
Exemplo n.º 10
0
import os
import sys
import re
from pprint import pprint as pp
import operator

scriptDir = os.path.dirname(__file__)
externalDir = os.path.abspath(os.path.join(scriptDir, '../../external'))
pybuildDir = os.path.abspath(
    os.path.join(scriptDir, '../../build_system/pybuild'))
langPyDir = os.path.abspath(os.path.join(scriptDir, '../py'))

# Get the architecture first
sys.path = [os.path.join(pybuildDir)] + sys.path
import arch
a = arch.getArch()

# Get yaml from external (it is not installed)
pythonVersion = '%d.%d' % sys.version_info[:2]
sys.path = [
    os.path.join(externalDir, a, 'lib/python%s/site-packages' % pythonVersion)
] + sys.path
import yaml

# Restore sys.path
sys.path = sys.path[2:]

# The directory that contains the NuPIC 2 C++ object model
baseDir = os.path.abspath(os.path.join(scriptDir, '../../nta/engine'))
assert os.path.isdir(baseDir)
Exemplo n.º 11
0
def buildit(rootdir, srcdir, builddir, installdir, svnpath, stamp, revision, datestamp, assertions, update, nprocs=1):
  """
  Do a full checkout and build
  """

  arch = getArch()
  log.info("Building for architecture " + arch)

  #
  # If source dir already exists, we're doing an incremental checkout. If not, we're
  # doing a full checkout. 
  #
  incremental = True
  if not os.path.exists(srcdir):
    if not update:
      log.error("No svn update requested, but source dir '%s' does not exist.", srcdir)
      raise Exception()
    else:
      incremental = False
      log.info("Source directory %s does not exist. Will do a full checkout", srcdir)
  else:
    if not os.path.isdir(srcdir):
      log.error("Source directory %s is not a directory.", srcdir)
      raise Exception()


  log.debug("Creating release directory %s", installdir)
  utils.createDir(installdir, True)
  utils.changeDir(os.path.join(installdir, os.pardir))
  buildfilename = os.path.join(installdir, ".buildinfo")

  #
  # Bring source tree up to date if requested
  #
  if update:
    log.info("Bringing source tree up to date at svnpath %s revision %s", svnpath, revision)
    svn.checkout(srcdir, repo=svnpath, revision=revision, incremental=incremental, keepLocalMods=False)
  else:
    log.warn("Skipping bringing up to date. Build will be tainted")
    utils.touchFile(os.path.join(installdir, "TAINTED_BUILD"))

  #
  # Revision may not be a number. May be "HEAD" or "PREV" or something else, so
  # retrieve it from svn
  #
  revisionnum = svn.getRevision(srcdir)

  #
  # place a temporary file in the install dir to indicate a build in progress
  # the file is removed at the end of the build process
  #
  build_in_progress_filename = os.path.join(installdir,"UNFINISHED_BUILD." + arch)
  utils.touchFile(build_in_progress_filename)


  log.debug("Updating build description file " + buildfilename)
  try:
    f = open(buildfilename, mode = "a")
    print >> f, "============================="
    print >> f, "Timestamp: " + datestamp
    print >> f, "Arch:      " + arch
    print >> f, "Buildhost: " + platform.node()
    print >> f, "Svnpath:   " + svnpath
    print >> f, "Revision:  " + str(revisionnum)
    print >> f, "Rootdir:   " + rootdir
    print >> f, "Srcdir:    " + srcdir
    print >> f, "Stamp:     " + stamp
    print >> f, "Assertions:" + str(assertions)
    print >> f, "Update:    " + str(update)
    print >> f, ""
    f.close()
  except Exception, e:
    raise Exception("Error writing to build description file '%s'", buildfilename)
Exemplo n.º 12
0
def buildCustomerRelease(releaseType, releaseSpecDir, releaseTarball, binaryTarball, prefix=None, nprocs=1,parentTmpDir=None):
    """From the given source release tarball, compile and verify that the 
    build product is correct.
    @param releaseType -- one of the source release types
    @param releaseSpecDir -- trunk/release directory containing manifest files for this release
    @param releaseTarball -- tarball containing the release to be built. 
    @param binaryTarball -- tarball containing the binary release. This must be provided
        when building the tools source release, which requires prebuilt libraries that 
        only exist in the binary release. These libraries are copied in to the source 
        release tree prior to building the source release. 
    Throws an exception if the build fails or the build product does not 
    exactly match the manifest file.
    If prefix is not None, install the build product into the prefix directory
    No return value"""


    log.info("Building customer %s release", releaseType)
    

    origDir = os.getcwd()

    if not releaseType in sourceReleaseTypes:
        raise Exception("buildCustomerRelease: unknown release type %s" % releaseType)

    tarballBasename = os.path.basename(releaseTarball)
    (dirName, ext) = os.path.splitext(tarballBasename)

    manifest = os.path.abspath(os.path.join(releaseSpecDir, 
                                            "release",
                                            releaseType+"_release", 
                                            "manifests", 
                                            "build_output.manifest"))
    
    tempdir = utils.createTemporaryDirectory("build_"+releaseType, parentDir=parentTmpDir)

    installdir = os.path.join(tempdir, "install")
    utils.createDir(installdir)

    builddir = os.path.join(tempdir, "build")
    utils.createDir(builddir)

    srcdir = utils.extractArchive(releaseTarball, tempdir)

    # Use the precompiled runtime library for the full source build. 
    if releaseType in releasesRequiringSPNupic:
      if binaryTarball is None:
        raise Exception("No binary tarball provided when building tools release, which requires prebuilt libraries")
      binarydir = utils.extractArchive(binaryTarball, tempdir)
      log.info("Copying libruntime.a from binary release into source release")
      if getArch() != "win32":
        smartCopy(os.path.join(binarydir, "lib", "libruntime.a"), 
                  os.path.join(srcdir, "external", getArch(), "lib", "libruntime.a"))
        smartCopy(os.path.join(binarydir, "lib", "libnode.a"), 
                  os.path.join(srcdir, "external", getArch(), "lib", "libnode.a"))
        smartCopy(os.path.join(binarydir, "lib", "libipcserial.a"), 
                  os.path.join(srcdir, "external", getArch(), "lib", "libipcserial.a"))
      else:
        smartCopy(os.path.join(binarydir, "lib", "release", "runtime.lib"), 
                  os.path.join(srcdir, "external", getArch(), "lib", "runtime.lib"))
        smartCopy(os.path.join(binarydir, "lib", "release", "node.lib"), 
                  os.path.join(srcdir, "external", getArch(), "lib", "node.lib"))
        smartCopy(os.path.join(binarydir, "lib", "release", "ipcserial.lib"), 
                  os.path.join(srcdir, "external", getArch(), "lib", "ipcserial.lib"))


    build.build(srcdir, builddir, installdir, assertions=False, customerRelease=True, nprocs=nprocs)

    # To verify the build product, we install from manifest and then 
    # confirm that the installed tree has the same number of files as the original tree. 
    # An exception in the installFromManifest phase usually means there is a file in 
    # the manifest that was not built. Then if the number of files is the same, it usually
    # means that a file was built that was not in the manifest. 
    tempInstallDir = os.path.join(tempdir, "temp_install")
    installFromManifest(manifest, installdir, tempInstallDir, 
                        level=0, overwrite=False, destdirExists=False)
    
    installedFiles = []

    for root,dirs,files in os.walk(tempInstallDir):
      installedFiles += [os.path.join(root, f) for f in files]
    # normalize the list of files
    prefixLen = len(tempInstallDir)
    installedFiles = [f[prefixLen:] for f in installedFiles]
    installedFiles = sorted(installedFiles)
      
    builtFiles = []
    for root,dirs,files in os.walk(installdir):
      builtFiles += [os.path.join(root, f) for f in files]
    # normalize the list of files
    prefixLen = len(installdir)
    builtFiles = [f[prefixLen:] for f in builtFiles]
    builtFiles = sorted(builtFiles)

    if len(installedFiles) != len(builtFiles):
      log.error("Manifest error when building %s release", releaseType)
      log.error("Built files: %d  Files in manifest: %d", len(builtFiles), len(installedFiles))
      # we know that every file in installedFiles is also in builtFiles
      # because installed dir was created from built dir
      for f in installedFiles:
        builtFiles.remove(f)
      
      for f in builtFiles:
        log.error("File '%s' installed but not in build_output.manifest" % f)
      
      raise Exception("Error building %s release -- file(s) missing from build_output.manifest" % releaseType)
    else:
      log.info("Release %s built successfully. Build produces %d files", releaseType, len(builtFiles))

    # copy all built files from directory where built into the the prefix dir, if specified
    if prefix is not None:
      log.info("Installing build products from '%s' release into binary release at '%s'", 
               releaseType, prefix)
      installFromManifest(manifest, installdir, prefix, 
                          level=0, overwrite=True, destdirExists=True)
    

    # clean up
    os.chdir(origDir)
    utils.remove(tempdir)
Exemplo n.º 13
0
      "/opt/buildtools/bin",
      "/opt/python%s/bin" % pythonVersion,
      "/usr/local/bin",
      "/usr/bin",
      "/bin")
    rootdir = os.path.expanduser("~/releases")
    builddir = ""
    srcdir = ""
    svnpath = "trunk"
    revision = "HEAD"
    assertions = False;
    update = True;
    dotests = True;
    dobuild = True;
    dorelease = True;
    arch = getArch()

    datestamp = time.strftime("%Y-%m-%d-%H%M")
    stamp = datestamp + "." + os.path.basename(svnpath)

    options = [
        "rootdir=",
        "srcdir=",
        "builddir=",
        "svnpath=",
        "stamp=",
        "revision=",
        "disable-test",
        "enable-assertions",
        "disable-update",
        "disable-build",