def add_build(self, pkg, check_timestamp, force=False): ''' Add a package to the build list if it or its dependencies have changed. ''' debug ('add_build %s [%s] %s' % (pkg, pkg.deps, force)) if self.database[pkg.name + '-status'] == 'installed': return 0 if not force and pkg.installed(): path = python_path(pkg.name) if os.path.commonprefix([path,self.libdir]) != self.libdir: return 0 deps_needed = 0 for p in pkg.deps: deps_needed += self.add_build(p, check_timestamp, force) if check_timestamp and deps_needed == 0: f, t = dirwalk.DirWalk(pkg.psdir).find_newest() debug('%s: %s\t%s' % (pkg.psdir, f, time.strftime("%b %d %Y %H:%M %Z", time.localtime(t)))) if t < self.database[pkg.name] and self.database[pkg.name + '-status'] == pkg.status(): return 0 if not pkg in self.packages: if check_timestamp: if self.database[pkg.name + '-status'] == pkg.status(): verbose(1, '\t%s needs rebuilt' % pkg.name) elif self.database[pkg.name + '-status'] == '': verbose(1, '\t%s needs built' % pkg.name) else: verbose(1, "\t%s needs rebuilt. Status changed from '%s' to '%s.'" % (pkg.name, self.database[pkg.name + '-status'], pkg.status())) self.packages.append(pkg) if pkg.name == 'python': # FIXME: major hack self.add_build([p for p in self.all_packages if p.name == 'matplotlib'][0], check_timestamp, True) self.add_build([p for p in self.all_packages if p.name == 'scipy'][0], check_timestamp, True) self.add_build([p for p in self.all_packages if p.name == 'nose'][0], check_timestamp, True) self.add_build([p for p in self.all_packages if p.name == 'sympy'][0], check_timestamp, True) self.add_build([p for p in self.all_packages if p.name == 'numpy'][0], check_timestamp, True) self.add_build([p for p in self.all_packages if p.name == 'tcl'][0], check_timestamp, True) self.add_build([p for p in self.all_packages if p.name == 'tk'][0], check_timestamp, True) self.add_build([p for p in self.all_packages if p.name == 'h5py'][0], check_timestamp, True) self.add_build([p for p in self.all_packages if p.name == 'setuptools'][0], check_timestamp, True) return 1
def add_build(self, pkg, check_timestamp, force=False): ''' Add a package to the build list if it or its dependencies have changed. ''' debug('add_build %s [%s] %s' % (pkg, pkg.deps, force)) if self.database[pkg.name + '-status'] == 'installed': return 0 if not force and pkg.installed(): path = python_path(pkg.name) if os.path.commonprefix([path, self.libdir]) != self.libdir: return 0 deps_needed = 0 for p in pkg.deps: deps_needed += self.add_build(p, check_timestamp, force) if check_timestamp and deps_needed == 0: f, t = dirwalk.DirWalk(pkg.psdir).find_newest() debug('%s: %s\t%s' % (pkg.psdir, f, time.strftime("%b %d %Y %H:%M %Z", time.localtime(t)))) if t < self.database[pkg.name] and self.database[ pkg.name + '-status'] == pkg.status(): return 0 if not pkg in self.packages: if check_timestamp: if self.database[pkg.name + '-status'] == pkg.status(): verbose(1, '\t%s needs rebuilt' % pkg.name) elif self.database[pkg.name + '-status'] == '': verbose(1, '\t%s needs built' % pkg.name) else: verbose( 1, "\t%s needs rebuilt. Status changed from '%s' to '%s.'" % (pkg.name, self.database[pkg.name + '-status'], pkg.status())) self.packages.append(pkg) if pkg.name == 'python': # FIXME: major hack self.add_build([ p for p in self.all_packages if p.name == 'matplotlib' ][0], check_timestamp, True) self.add_build([ p for p in self.all_packages if p.name == 'scipy' ][0], check_timestamp, True) self.add_build([ p for p in self.all_packages if p.name == 'nose' ][0], check_timestamp, True) self.add_build([ p for p in self.all_packages if p.name == 'sympy' ][0], check_timestamp, True) self.add_build([ p for p in self.all_packages if p.name == 'numpy' ][0], check_timestamp, True) self.add_build([ p for p in self.all_packages if p.name == 'tcl' ][0], check_timestamp, True) self.add_build([ p for p in self.all_packages if p.name == 'tk' ][0], check_timestamp, True) self.add_build([ p for p in self.all_packages if p.name == 'h5py' ][0], check_timestamp, True) self.add_build([ p for p in self.all_packages if p.name == 'setuptools' ][0], check_timestamp, True) return 1
def __init__(self, cname, topdir, make_path): # create build directories cwd = os.getcwd() topdir = os.path.abspath(topdir) self.topdir = topdir self.blddir = os.path.join(os.getcwd(), "build-%s" % cname) self.logdir = os.path.join(self.blddir, "log") self.bindir = os.path.join(self.blddir, "bin") self.libdir = os.path.join(self.blddir, "lib") self.incdir = os.path.join(self.blddir, "include") for p in [ self.blddir, self.logdir, self.libdir, self.bindir, self.incdir ]: if not os.path.isdir(p): try: os.makedirs(p) except: fatal("error creating directory " + p) # load build database self.database = shelve.open(os.path.join(self.logdir, 'PACKAGES')) # import modules from 'packages' directory for path in (os.path.join(topdir, 'config', 'packages'), os.path.join(make_path, 'packages')): if os.path.isdir(path): debug("importing package modules from %s" % path) try: os.chdir(path) sys.path = [path] + sys.path except: fatal('Cannot read modules from %s' % path) for m in glob.glob('*.py'): m = m.split('.')[0] if m == '__init__': continue debug("importing '%s'" % m) try: loaded = eval(m) except NameError: loaded = False if not loaded: _temp = __import__(m, globals(), locals(), [m]) exec('global %s; %s=_temp.%s; self.%s=%s' % (m, m, m, m, m)) # create package class instances self.all_packages = [] path = os.path.join(topdir, 'config', 'packages') os.chdir(path) for line in open('SOURCES'): cls, sdir = line.split() acls = [] try: exec("acls = %s(self, '%s')" % (cls, sdir)) self.all_packages.append(acls) if cls == 'Scons': self.Scons = acls except NameError: Warning('Package %s in SOURCES, but no build script found.' % cls) # load PACKAGES database for pname in [p.name for p in self.all_packages]: try: tmp = self.database[pname] debug('database[%s] = %s' % (pname, tmp)) except KeyError: self.database[pname] = 0 try: tmp = self.database[pname + '-status'] debug('database[%s] = %s' % (pname + '-status', tmp)) except KeyError: self.database[pname + '-status'] = '' # cache package instances by name key self.pkglist = {} for p in self.all_packages: self.pkglist[p.name] = p # now build dependency list for all packages for p in self.all_packages: deps = [] p.deps = [] if hasattr(p, 'requires'): deps += p.requires deps = p.add_required(deps) for dep in deps: try: p.deps.append(self.pkglist[dep]) except: fatal('Dependency %s for %s not recognized' % (dep, p.name)) self.packages = [] self.build_pkg_list(True) if self.packages: self.reorder_pkgs() os.chdir(cwd)
def putfile(f, uri, username=None, password=None): """HTTP PUT the file f to uri, with optional auth data.""" host, port, path = parseuri(uri) redirect = set([301, 302, 307]) authenticate = set([401]) okay = set([200, 201, 204]) authorized = False authorization = None tries = 0 while True: # Attempt to HTTP PUT the data h = httplib.HTTPConnection(host, port) h.putrequest('PUT', path) h.putheader('User-Agent', 'put.py/1.0') h.putheader('Connection', 'keep-alive') h.putheader('Transfer-Encoding', 'chunked') h.putheader('Expect', '100-continue') h.putheader('Accept', '*/*') if authorization: h.putheader('Authorization', authorization) h.endheaders() # Chunked transfer encoding # Cf. 'All HTTP/1.1 applications MUST be able to receive and # decode the "chunked" transfer-coding' # - http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html while True: bytes = f.read(2048) if not bytes: break length = len(bytes) h.send('%X\r\n' % length) h.send(bytes + '\r\n') h.send('0\r\n\r\n') f.seek(0); resp = h.getresponse() status = resp.status # an int # Got a response, now decide how to act upon it if status in redirect: location = resp.getheader('Location') uri = urlparse.urljoin(uri, location) host, port, path = parseuri(uri) # We may have to authenticate again if authorization: authorization = None elif status in authenticate: # If we've done this already, break if authorization: # barf("Going around in authentication circles") barf("Authentication failed") if not (username and password): barf("Need a username and password to authenticate with") # Get the scheme: Basic or Digest? wwwauth = resp.msg['www-authenticate'] # We may need this again wauth = wwwauth.lstrip(' \t') # Hence use wauth not wwwauth here wauth = wwwauth.replace('\t', ' ') i = wauth.index(' ') scheme = wauth[:i].lower() if scheme in set(['basic', 'digest', 'Basic', 'Digest']): msg = "Performing %s Authentication..." % scheme.capitalize() else: barf("Unknown authentication scheme: %s" % scheme) if scheme == 'basic' or scheme == 'Basic': import base64 userpass = username + ':' + password userpass = base64.encodestring(userpass).strip() authorized, authorization = True, 'Basic ' + userpass elif scheme == 'digest': msg = "uses fragile, undocumented features in urllib2" build_utils.debug("Warning! Digest Auth %s" % msg) import urllib2 # See warning above passwd = type('Password', (object,), { 'find_user_password': lambda self, *args: (username, password), 'add_password': lambda self, *args: None })() req = type('Request', (object,), { 'get_full_url': lambda self: uri, 'has_data': lambda self: None, 'get_method': lambda self: 'PUT', 'get_selector': lambda self: path })() # Cf. urllib2.AbstractDigestAuthHandler.retry_http_digest_auth auth = urllib2.AbstractDigestAuthHandler(passwd) token, challenge = wwwauth.split(' ', 1) chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) userpass = auth.get_authorization(req, chal) authorized, authorization = True, 'Digest ' + userpass elif status in okay: if (username and password) and (not authorized): msg = "Warning! The supplied username and password went unused" print msg if build_utils.opt_debug: resultLine = "Success! Resource %s" statuses = {200: 'modified', 201: 'created', 204: 'modified'} build_utils.debug(resultLine % statuses[status]) statusLine = "Response-Status: %s %s" build_utils.debug(statusLine % (status, resp.reason)) body = resp.read(58) body = body.rstrip('\r\n') body = body.encode('string_escape') if len(body) >= 58: body = body[:57] + '[...]' bodyLine = 'Response-Body: "%s"' build_utils.debug(bodyLine % body) break # @@ raise PutError, do the catching in main? else: barf('Got "%s %s"' % (status, resp.reason)) tries += 1 if tries >= 50: barf("Too many redirects") return status, resp
def __init__(self, cname, topdir, make_path): # create build directories cwd = os.getcwd() topdir = os.path.abspath(topdir) self.topdir = topdir self.blddir = os.path.join(os.getcwd(), "build-%s" % cname) self.logdir = os.path.join(self.blddir, "log") self.bindir = os.path.join(self.blddir, "bin") self.libdir = os.path.join(self.blddir, "lib") self.incdir = os.path.join(self.blddir, "include") for p in [self.blddir, self.logdir, self.libdir, self.bindir, self.incdir]: if not os.path.isdir(p): try: os.makedirs(p) except: fatal("error creating directory " + p) # load build database self.database = shelve.open(os.path.join(self.logdir, 'PACKAGES')) # import modules from 'packages' directory for path in (os.path.join(topdir, 'config', 'packages'), os.path.join(make_path, 'packages')): if os.path.isdir(path): debug("importing package modules from %s" % path) try: os.chdir(path) sys.path = [path] + sys.path except: fatal('Cannot read modules from %s' % path) for m in glob.glob('*.py'): m = m.split('.')[0] if m == '__init__': continue debug("importing '%s'" % m) try: loaded = eval(m) except NameError: loaded = False if not loaded: _temp = __import__(m, globals(), locals(), [m]) exec('global %s; %s=_temp.%s; self.%s=%s' % (m, m, m, m, m)) # create package class instances self.all_packages = [] path = os.path.join(topdir, 'config', 'packages') os.chdir(path) for line in open('SOURCES'): cls, sdir = line.split() acls = [] try: exec("acls = %s(self, '%s')" % (cls, sdir)) self.all_packages.append(acls) if cls == 'Scons': self.Scons = acls except NameError: Warning('Package %s in SOURCES, but no build script found.' % cls) # load PACKAGES database for pname in [p.name for p in self.all_packages]: try: tmp = self.database[pname] debug('database[%s] = %s' % (pname, tmp)) except KeyError: self.database[pname] = 0 try: tmp = self.database[pname + '-status'] debug('database[%s] = %s' % (pname + '-status', tmp)) except KeyError: self.database[pname + '-status'] = '' # cache package instances by name key self.pkglist = {} for p in self.all_packages: self.pkglist[p.name] = p # now build dependency list for all packages for p in self.all_packages: deps = [] p.deps = [] if hasattr(p, 'requires'): deps += p.requires deps = p.add_required(deps) for dep in deps: try: p.deps.append(self.pkglist[dep]) except: fatal('Dependency %s for %s not recognized' % (dep, p.name)) self.packages = [] self.build_pkg_list(True) if self.packages: self.reorder_pkgs() os.chdir(cwd)