def add_build(self, pkg, check_timestamp, force=False):
''' Add a package to the build list if it or its dependencies have changed. '''
debug ('add_build %s [%s] %s' % (pkg, pkg.deps, force))
if self.database[pkg.name + '-status'] == 'installed':
return 0
if not force and pkg.installed():
path = python_path(pkg.name)
if os.path.commonprefix([path,self.libdir]) != self.libdir:
return 0
deps_needed = 0
for p in pkg.deps:
deps_needed += self.add_build(p, check_timestamp, force)
if check_timestamp and deps_needed == 0:
f, t = dirwalk.DirWalk(pkg.psdir).find_newest()
debug('%s: %s\t%s' % (pkg.psdir, f, time.strftime("%b %d %Y %H:%M %Z", time.localtime(t))))
if t < self.database[pkg.name] and self.database[pkg.name + '-status'] == pkg.status():
return 0
if not pkg in self.packages:
if check_timestamp:
if self.database[pkg.name + '-status'] == pkg.status():
verbose(1, '\t%s needs rebuilt' % pkg.name)
elif self.database[pkg.name + '-status'] == '':
verbose(1, '\t%s needs built' % pkg.name)
else:
verbose(1, "\t%s needs rebuilt. Status changed from '%s' to '%s.'"
% (pkg.name, self.database[pkg.name + '-status'], pkg.status()))
self.packages.append(pkg)
if pkg.name == 'python':
# FIXME: major hack
self.add_build([p for p in self.all_packages if p.name == 'matplotlib'][0], check_timestamp, True)
self.add_build([p for p in self.all_packages if p.name == 'scipy'][0], check_timestamp, True)
self.add_build([p for p in self.all_packages if p.name == 'nose'][0], check_timestamp, True)
self.add_build([p for p in self.all_packages if p.name == 'sympy'][0], check_timestamp, True)
self.add_build([p for p in self.all_packages if p.name == 'numpy'][0], check_timestamp, True)
self.add_build([p for p in self.all_packages if p.name == 'tcl'][0], check_timestamp, True)
self.add_build([p for p in self.all_packages if p.name == 'tk'][0], check_timestamp, True)
self.add_build([p for p in self.all_packages if p.name == 'h5py'][0], check_timestamp, True)
self.add_build([p for p in self.all_packages if p.name == 'setuptools'][0], check_timestamp, True)
return 1
def add_build(self, pkg, check_timestamp, force=False):
''' Add a package to the build list if it or its dependencies have changed. '''
debug('add_build %s [%s] %s' % (pkg, pkg.deps, force))
if self.database[pkg.name + '-status'] == 'installed':
return 0
if not force and pkg.installed():
path = python_path(pkg.name)
if os.path.commonprefix([path, self.libdir]) != self.libdir:
return 0
deps_needed = 0
for p in pkg.deps:
deps_needed += self.add_build(p, check_timestamp, force)
if check_timestamp and deps_needed == 0:
f, t = dirwalk.DirWalk(pkg.psdir).find_newest()
debug('%s: %s\t%s' %
(pkg.psdir, f,
time.strftime("%b %d %Y %H:%M %Z", time.localtime(t))))
if t < self.database[pkg.name] and self.database[
pkg.name + '-status'] == pkg.status():
return 0
if not pkg in self.packages:
if check_timestamp:
if self.database[pkg.name + '-status'] == pkg.status():
verbose(1, '\t%s needs rebuilt' % pkg.name)
elif self.database[pkg.name + '-status'] == '':
verbose(1, '\t%s needs built' % pkg.name)
else:
verbose(
1,
"\t%s needs rebuilt. Status changed from '%s' to '%s.'"
% (pkg.name, self.database[pkg.name + '-status'],
pkg.status()))
self.packages.append(pkg)
if pkg.name == 'python':
# FIXME: major hack
self.add_build([
p for p in self.all_packages if p.name == 'matplotlib'
][0], check_timestamp, True)
self.add_build([
p for p in self.all_packages if p.name == 'scipy'
][0], check_timestamp, True)
self.add_build([
p for p in self.all_packages if p.name == 'nose'
][0], check_timestamp, True)
self.add_build([
p for p in self.all_packages if p.name == 'sympy'
][0], check_timestamp, True)
self.add_build([
p for p in self.all_packages if p.name == 'numpy'
][0], check_timestamp, True)
self.add_build([
p for p in self.all_packages if p.name == 'tcl'
][0], check_timestamp, True)
self.add_build([
p for p in self.all_packages if p.name == 'tk'
][0], check_timestamp, True)
self.add_build([
p for p in self.all_packages if p.name == 'h5py'
][0], check_timestamp, True)
self.add_build([
p for p in self.all_packages if p.name == 'setuptools'
][0], check_timestamp, True)
return 1
def __init__(self, cname, topdir, make_path):
# create build directories
cwd = os.getcwd()
topdir = os.path.abspath(topdir)
self.topdir = topdir
self.blddir = os.path.join(os.getcwd(), "build-%s" % cname)
self.logdir = os.path.join(self.blddir, "log")
self.bindir = os.path.join(self.blddir, "bin")
self.libdir = os.path.join(self.blddir, "lib")
self.incdir = os.path.join(self.blddir, "include")
for p in [
self.blddir, self.logdir, self.libdir, self.bindir, self.incdir
]:
if not os.path.isdir(p):
try:
os.makedirs(p)
except:
fatal("error creating directory " + p)
# load build database
self.database = shelve.open(os.path.join(self.logdir, 'PACKAGES'))
# import modules from 'packages' directory
for path in (os.path.join(topdir, 'config', 'packages'),
os.path.join(make_path, 'packages')):
if os.path.isdir(path):
debug("importing package modules from %s" % path)
try:
os.chdir(path)
sys.path = [path] + sys.path
except:
fatal('Cannot read modules from %s' % path)
for m in glob.glob('*.py'):
m = m.split('.')[0]
if m == '__init__':
continue
debug("importing '%s'" % m)
try:
loaded = eval(m)
except NameError:
loaded = False
if not loaded:
_temp = __import__(m, globals(), locals(), [m])
exec('global %s; %s=_temp.%s; self.%s=%s' %
(m, m, m, m, m))
# create package class instances
self.all_packages = []
path = os.path.join(topdir, 'config', 'packages')
os.chdir(path)
for line in open('SOURCES'):
cls, sdir = line.split()
acls = []
try:
exec("acls = %s(self, '%s')" % (cls, sdir))
self.all_packages.append(acls)
if cls == 'Scons':
self.Scons = acls
except NameError:
Warning('Package %s in SOURCES, but no build script found.' %
cls)
# load PACKAGES database
for pname in [p.name for p in self.all_packages]:
try:
tmp = self.database[pname]
debug('database[%s] = %s' % (pname, tmp))
except KeyError:
self.database[pname] = 0
try:
tmp = self.database[pname + '-status']
debug('database[%s] = %s' % (pname + '-status', tmp))
except KeyError:
self.database[pname + '-status'] = ''
# cache package instances by name key
self.pkglist = {}
for p in self.all_packages:
self.pkglist[p.name] = p
# now build dependency list for all packages
for p in self.all_packages:
deps = []
p.deps = []
if hasattr(p, 'requires'): deps += p.requires
deps = p.add_required(deps)
for dep in deps:
try:
p.deps.append(self.pkglist[dep])
except:
fatal('Dependency %s for %s not recognized' %
(dep, p.name))
self.packages = []
self.build_pkg_list(True)
if self.packages:
self.reorder_pkgs()
os.chdir(cwd)
def putfile(f, uri, username=None, password=None):
"""HTTP PUT the file f to uri, with optional auth data."""
host, port, path = parseuri(uri)
redirect = set([301, 302, 307])
authenticate = set([401])
okay = set([200, 201, 204])
authorized = False
authorization = None
tries = 0
while True:
# Attempt to HTTP PUT the data
h = httplib.HTTPConnection(host, port)
h.putrequest('PUT', path)
h.putheader('User-Agent', 'put.py/1.0')
h.putheader('Connection', 'keep-alive')
h.putheader('Transfer-Encoding', 'chunked')
h.putheader('Expect', '100-continue')
h.putheader('Accept', '*/*')
if authorization:
h.putheader('Authorization', authorization)
h.endheaders()
# Chunked transfer encoding
# Cf. 'All HTTP/1.1 applications MUST be able to receive and
# decode the "chunked" transfer-coding'
# - http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html
while True:
bytes = f.read(2048)
if not bytes: break
length = len(bytes)
h.send('%X\r\n' % length)
h.send(bytes + '\r\n')
h.send('0\r\n\r\n')
f.seek(0);
resp = h.getresponse()
status = resp.status # an int
# Got a response, now decide how to act upon it
if status in redirect:
location = resp.getheader('Location')
uri = urlparse.urljoin(uri, location)
host, port, path = parseuri(uri)
# We may have to authenticate again
if authorization:
authorization = None
elif status in authenticate:
# If we've done this already, break
if authorization:
# barf("Going around in authentication circles")
barf("Authentication failed")
if not (username and password):
barf("Need a username and password to authenticate with")
# Get the scheme: Basic or Digest?
wwwauth = resp.msg['www-authenticate'] # We may need this again
wauth = wwwauth.lstrip(' \t') # Hence use wauth not wwwauth here
wauth = wwwauth.replace('\t', ' ')
i = wauth.index(' ')
scheme = wauth[:i].lower()
if scheme in set(['basic', 'digest', 'Basic', 'Digest']):
msg = "Performing %s Authentication..." % scheme.capitalize()
else:
barf("Unknown authentication scheme: %s" % scheme)
if scheme == 'basic' or scheme == 'Basic':
import base64
userpass = username + ':' + password
userpass = base64.encodestring(userpass).strip()
authorized, authorization = True, 'Basic ' + userpass
elif scheme == 'digest':
msg = "uses fragile, undocumented features in urllib2"
build_utils.debug("Warning! Digest Auth %s" % msg)
import urllib2 # See warning above
passwd = type('Password', (object,),
{
'find_user_password': lambda self, *args: (username, password),
'add_password': lambda self, *args: None
})()
req = type('Request', (object,),
{
'get_full_url': lambda self: uri,
'has_data': lambda self: None,
'get_method': lambda self: 'PUT',
'get_selector': lambda self: path
})()
# Cf. urllib2.AbstractDigestAuthHandler.retry_http_digest_auth
auth = urllib2.AbstractDigestAuthHandler(passwd)
token, challenge = wwwauth.split(' ', 1)
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
userpass = auth.get_authorization(req, chal)
authorized, authorization = True, 'Digest ' + userpass
elif status in okay:
if (username and password) and (not authorized):
msg = "Warning! The supplied username and password went unused"
print msg
if build_utils.opt_debug:
resultLine = "Success! Resource %s"
statuses = {200: 'modified', 201: 'created', 204: 'modified'}
build_utils.debug(resultLine % statuses[status])
statusLine = "Response-Status: %s %s"
build_utils.debug(statusLine % (status, resp.reason))
body = resp.read(58)
body = body.rstrip('\r\n')
body = body.encode('string_escape')
if len(body) >= 58:
body = body[:57] + '[...]'
bodyLine = 'Response-Body: "%s"'
build_utils.debug(bodyLine % body)
break
# @@ raise PutError, do the catching in main?
else:
barf('Got "%s %s"' % (status, resp.reason))
tries += 1
if tries >= 50:
barf("Too many redirects")
return status, resp
def __init__(self, cname, topdir, make_path):
# create build directories
cwd = os.getcwd()
topdir = os.path.abspath(topdir)
self.topdir = topdir
self.blddir = os.path.join(os.getcwd(), "build-%s" % cname)
self.logdir = os.path.join(self.blddir, "log")
self.bindir = os.path.join(self.blddir, "bin")
self.libdir = os.path.join(self.blddir, "lib")
self.incdir = os.path.join(self.blddir, "include")
for p in [self.blddir, self.logdir,
self.libdir, self.bindir, self.incdir]:
if not os.path.isdir(p):
try:
os.makedirs(p)
except:
fatal("error creating directory " + p)
# load build database
self.database = shelve.open(os.path.join(self.logdir, 'PACKAGES'))
# import modules from 'packages' directory
for path in (os.path.join(topdir, 'config', 'packages'),
os.path.join(make_path, 'packages')):
if os.path.isdir(path):
debug("importing package modules from %s" % path)
try:
os.chdir(path)
sys.path = [path] + sys.path
except:
fatal('Cannot read modules from %s' % path)
for m in glob.glob('*.py'):
m = m.split('.')[0]
if m == '__init__':
continue
debug("importing '%s'" % m)
try:
loaded = eval(m)
except NameError:
loaded = False
if not loaded:
_temp = __import__(m, globals(), locals(), [m])
exec('global %s; %s=_temp.%s; self.%s=%s' % (m, m, m, m, m))
# create package class instances
self.all_packages = []
path = os.path.join(topdir, 'config', 'packages')
os.chdir(path)
for line in open('SOURCES'):
cls, sdir = line.split()
acls = []
try:
exec("acls = %s(self, '%s')" % (cls, sdir))
self.all_packages.append(acls)
if cls == 'Scons':
self.Scons = acls
except NameError:
Warning('Package %s in SOURCES, but no build script found.' % cls)
# load PACKAGES database
for pname in [p.name for p in self.all_packages]:
try:
tmp = self.database[pname]
debug('database[%s] = %s' % (pname, tmp))
except KeyError:
self.database[pname] = 0
try:
tmp = self.database[pname + '-status']
debug('database[%s] = %s' % (pname + '-status', tmp))
except KeyError:
self.database[pname + '-status'] = ''
# cache package instances by name key
self.pkglist = {}
for p in self.all_packages:
self.pkglist[p.name] = p
# now build dependency list for all packages
for p in self.all_packages:
deps = []
p.deps = []
if hasattr(p, 'requires'): deps += p.requires
deps = p.add_required(deps)
for dep in deps:
try:
p.deps.append(self.pkglist[dep])
except:
fatal('Dependency %s for %s not recognized' % (dep, p.name))
self.packages = []
self.build_pkg_list(True)
if self.packages:
self.reorder_pkgs()
os.chdir(cwd)
