def _install_python_package(self, directory):
"""Install a python package."""
self._set_py_path()
cmd = '"%s" setup.py install %s %s %s' % (
self.executable,
'--install-purelib="%s"' % self.site_packages_path,
'--install-platlib="%s"' % self.site_packages_path,
'--prefix=%s' % self.prefix
)
# moving and restoring if problem :)
cwd = os.getcwd()
os.chdir(directory)
tmp = '%s.%s.old' % (int(time.time()*1000), self.site_packages_path)
if os.path.exists(self.site_packages_path):
if os.path.exists(tmp):
remove_path(tmp)
copy_tree(self.site_packages_path, tmp)
if not self.options.get('noinstall', None):
try:
if not os.path.exists(self.site_packages_path):
os.makedirs(self.site_packages_path)
self._system(cmd)
except Exception, e:
remove_path(tmp)
copy_tree(tmp, self.site_packages_path)
raise core.MinimergeError('PythonPackage Install failed:\n\t%s' % e)
def reinstall_pil(self):
self._sync(['dependencies', 'eggs'])
pys = []
for v in self.PYTHON_VERSIONS:
py = self._find_minibuild('python-%s' % v)
if self.is_installed(py):
pys.append(v)
pil = self._find_minibuild('pil-1.1.7')
update = self._update
upgrade = self._upgrade
packages = self._packages
nodeps = self._nodeps
for v in pys:
self.pyvers = {pil.name: [v]}
if self.is_installed(pil):
for p in (glob(self._prefix+'/eggs/cache/PIL-1.1.7*%s*'%v)+
glob(self._prefix+'/eggs/cache/Pillow-1.7.7*%s*'%v)) :
print "DELETING OLD PIL FOR PYTHON%s" % v
remove_path(p)
ps = self._compute_dependencies([pil.name])
pps = self.install_filter(ps)
reinstalled = [a.name for a in pps]
if not 'pil-1.1.7' in reinstalled:
reinstalled.append('pil-1.1.7')
self._packages=reinstalled
self.reinstall_packages(reinstalled,
force=True,
pyvers=self.pyvers)
self._update = update
self._upgrade = upgrade
self._packages = packages
self._nodeps = nodeps
def remove_egg_info(p):
for dirpath, dirnames, filenames in os.walk(p):
for filename in dirnames+filenames:
if 'egg-info' in filename:
remove_path(
os.path.join(dirpath, filename)
)
for directory in dirnames:
subdir = os.path.join(dirpath, directory)
remove_egg_info(subdir)
def testRemovePath(self):
"""testRemovePath."""
file = tempfile.mkstemp()
file = file[1]
open(file,'w').write('a')
self.assertTrue(os.path.isfile(file))
common.remove_path(file)
self.assertFalse(os.path.isfile(file))
a = tempfile.mkdtemp()
self.assertTrue(os.path.isdir(a))
common.remove_path(a)
self.assertFalse(os.path.isdir(a))
def move_lafiles(directory, logger=False):
if not logger:
logging.basicConfig()
logger = logging.getLogger('cmmi.remove_lafiles')
p = os.path.abspath(directory)
if os.path.exists(p):
for cwd, ds, fs in os.walk(p):
for f in fs:
fp = os.path.join(cwd, f)
if fp.endswith('.la'):
bfp = "%s.old" % fp
if os.path.exists(bfp):
try:
remove_path("%s.old" % fp)
except:
pass
try:
os.rename(fp, "%s.old" % fp)
except:
logger.error('Cant remove Libtool Archive: %s' % fp)
def move_lafiles(directory, logger=False):
if not logger:
logging.basicConfig()
logger = logging.getLogger('cmmi.remove_lafiles')
p = os.path.abspath(directory)
if os.path.exists(p):
for cwd, ds, fs in os.walk(p):
for f in fs:
fp = os.path.join(cwd, f)
if fp.endswith('.la'):
bfp = "%s.old" % fp
if os.path.exists(bfp):
try:
remove_path("%s.old" % fp)
except:
pass
try:
os.rename(fp, "%s.old" % fp)
except:
logger.error('Cant remove Libtool Archive: %s' % fp)
def _make_install(self, directory):
""""""
# moving and restoring if problem :)
self.go_inner_dir()
cwd = os.getcwd()
os.chdir(directory)
if self.makeinstalldir:
os.chdir(self.makeinstalldir)
tmp = '%s.old' % self.prefix
if len(self.install_options)>0:
self.make_options = self.install_options
self.make_options_before = self.make_options
self.make_options_after = ''
if len(self.make_install_append_options) > 0:
self.make_options_before = ''
self.make_options_after = self.make_options
self.make_options = ''
self.install_targets = ['%s %s %s' % (self.make_options_before,
t,
self.make_options_after)
for t in self.install_targets]
self.make_options = ''
if not self.noinstall:
if not os.path.exists(tmp):
os.makedirs(tmp)
if os.path.isdir(self.prefix):
copy_tree(self.prefix, tmp)
if not self.install_in_place:
os.chdir(cwd)
shutil.rmtree(self.prefix)
try:
if not os.path.exists(self.prefix):
os.makedirs(self.prefix)
self._call_hook('pending-make-install-hook')
self._make(directory, self.install_targets)
except Exception, e:
remove_path(self.prefix)
if os.path.exists(tmp):
shutil.move(tmp, self.prefix)
raise core.MinimergeError('Install failed:\n\t%s' % e)
def _make_install(self, directory):
""""""
# moving and restoring if problem :)
self.go_inner_dir()
cwd = os.getcwd()
os.chdir(directory)
if self.makeinstalldir:
os.chdir(self.makeinstalldir)
tmp = '%s.old' % self.prefix
if len(self.install_options)>0:
self.make_options = self.install_options
self.make_options_before = self.make_options
self.make_options_after = ''
if len(self.make_install_append_options) > 0:
self.make_options_before = ''
self.make_options_after = self.make_options
self.make_options = ''
self.install_targets = ['%s %s %s' % (self.make_options_before,
t,
self.make_options_after)
for t in self.install_targets]
self.make_options = ''
if not self.noinstall:
#if os.path.isdir(self.prefix):
# print "now going to copy from tmp to target"
# copy_tree(self.prefix, tmp)
if not self.install_in_place:
os.chdir(cwd)
shutil.rmtree(self.prefix)
try:
if not os.path.exists(self.prefix):
os.makedirs(self.prefix)
self._call_hook('pending-make-install-hook')
self._make(directory, self.install_targets)
except Exception, e:
remove_path(self.prefix)
if os.path.exists(tmp):
print "now move tmp target, dont know where to where"
shutil.move(tmp, self.prefix)
raise core.MinimergeError('Install failed:\n\t%s' % e)
def post(self, command, output_dir, vars):
# symlink conf, and logs directories inside $sys
print
print
dirs = [
os.path.join(vars['sys'], 'bin'),
os.path.join(vars['sys'], 'etc', 'init.d')
]
for directory in dirs:
for filep in os.listdir(directory):
p = os.path.join(directory, filep)
os.chmod(p, stat.S_IRGRP | stat.S_IXGRP | stat.S_IRWXU)
conf = os.path.join(vars['sys'], 'var', 'data', 'tomcat',
vars['tomcat_instance'], 'conf')
logs = os.path.join(vars['sys'], 'var', 'data', 'tomcat',
vars['tomcat_instance'], 'logs')
dconf = os.path.join(vars['sys'], 'etc', 'tomcat',
vars['tomcat_instance'])
dlogs = os.path.join(
vars['sys'],
'var',
'log',
'tomcat',
vars['tomcat_instance'],
)
for p in dlogs, dconf:
if os.path.exists(p):
remove_path(p)
os.symlink(conf, dconf)
os.symlink(logs, dlogs)
keysp = os.path.join(os.path.dirname(conf), 'keys')
if not os.path.exists(keysp):
os.makedirs(keysp)
ssl.generate_prefixed_ssl_bundle(vars, vars['tomcat_instance'])
README = '\n\n%s' % (
"Installation is now finished.\n"
"------------------------------\n"
"\n"
" * You can find an init script to lunach your tomcat instance in %s/etc/init.d/%s_%s.%s.\n"
" * Your CATALINA_BASE is installed in %s.\n"
" * wrappers for various tomcat script have been isntalled in %s/bin.\n"
" * %s\n"
" * logs are in %s/var/log/tomvat/cas\n"
" * symlinks for the configuration directory have been made into %s/etc.\n"
"\n" % (
vars['sys'],
vars['project'],
vars['project'],
vars['tomcat_instance'],
os.path.dirname(conf),
vars['sys'],
', '.join([
a for a in os.listdir(os.path.join(vars['sys'], 'bin'))
if 'tomcat_%s' % vars['tomcat_instance'] in a
]),
vars['sys'],
vars['sys'],
))
for app in ['ROOT', 'manager', 'host-manager']:
o = os.path.join(vars['catalina_home'], 'webapps', app)
d = os.path.join(vars['catalina_base'], 'webapps', app)
msg = " * Copying %s application from %s.\n" % (app, o)
README += msg
copy_tree(o, d)
if vars['inside_minitage']:
README += '\n%s' % (
"Please verify that tomcat-%s is in your minibuild's dependencies\n"
"---------------------------------------------------------------------\n"
"If not, please add it and run the following commands:\n"
" minimerge -v tomcat-%s\n"
" source %s/share/minitage/minitage.env\n"
" $MT/paster create -t minitage.instances.env %s\n"
"\n" % (version, version, vars['sys'], vars['project']))
readmep = os.path.join(vars['path'],
'README.tomcat.%s' % vars['tomcat_instance'])
print README
print "Those informations have been saved in %s" % readmep
open(readmep, 'w').write(README)
self.write_config(vars)
self._set_py_path()
cmd = '"%s" setup.py install %s %s %s' % (
self.executable,
'--install-purelib="%s"' % self.site_packages_path,
'--install-platlib="%s"' % self.site_packages_path,
'--prefix=%s' % self.prefix
)
# moving and restoring if problem :)
cwd = os.getcwd()
os.chdir(directory)
tmp = '%s.%s.old' % (int(time.time()*1000), self.site_packages_path)
if os.path.exists(self.site_packages_path):
if os.path.exists(tmp):
remove_path(tmp)
copy_tree(self.site_packages_path, tmp)
if not self.options.get('noinstall', None):
try:
if not os.path.exists(self.site_packages_path):
os.makedirs(self.site_packages_path)
self._system(cmd)
except Exception, e:
remove_path(tmp)
copy_tree(tmp, self.site_packages_path)
raise core.MinimergeError('PythonPackage Install failed:\n\t%s' % e)
if os.path.exists(tmp):
remove_path(tmp)
os.chdir(cwd)
def pre(self, command, output_dir, vars):
common.Template.pre(self, command, output_dir, vars)
vars['running_user'] = running_user
self.db_path = db_path = os.path.join(
vars['sys'], 'var', 'data', 'postgresql', vars['db_name']
)
conf = os.path.join(vars['sys'],
'var', 'data',
'postgresql',
vars['db_name'],
'postgresql.conf')
# registering where we initdb
# which database do we createdb
os.environ['PGDATA'] = db_path
os.environ['PGUSER'] = running_user
os.environ['PGDATABASE'] = vars['db_name']
os.environ['PGHOST'] = vars['db_host']
os.environ['PGPORT'] = vars['db_port']
# default charsets C avoiding regional problems :)
os.environ['LANG'] = os.environ['LC_ALL'] = 'C'
env_file = os.path.join(vars['sys'], 'share', 'minitage', 'minitage.env')
bash_init = '. %s' % (env_file,)
fic = open(env_file).read()
version = os.popen(
'bash -c "'
'%s;'
'initdb --version'
'"' % bash_init
).read()
if '8.2' in version:
vars['lc'] = 'redirect_stderr'
else:
vars['lc'] = 'logging_collector'
if not os.path.exists(conf):
if not os.path.exists(db_path):
os.makedirs(db_path)
pgre = re.compile('.*postgresql-([^\s]*)\s.*', re_flags)
m = pgre.match(fic)
# We are searching in the destination if we
# already have a postgresql installation.
# If we have, just register as already installed.
# and do not initdb.
# If no pgsql is installed, do initdb/createdb but
# remove files coming out by templates
# to be out of overwrite errors.
SANITIZER = re.compile('([;])', re_flags).sub
init_db = ';'.join(
[bash_init,
'initdb -E \'UTF-8\';'
'pg_ctl -w start -o "-k%s"' % self.db_path]
)
create_user = '******'.join(
[bash_init,
"echo CREATE USER %s"
" WITH ENCRYPTED PASSWORD \\'%s\\'|psql template1" % (
vars['db_user'],
SANITIZER(r'\\\1' ,vars['db_password']),
)]
)
create_db = ';'.join(
[bash_init,
'createdb -O %s %s ;' % (
vars['db_user'],
vars['db_name'],
)
]
)
grant = ';'.join(
[bash_init,
"echo GRANT ALL PRIVILEGES"
" ON DATABASE %s"
" to %s WITH GRANT OPTION|psql" % (
vars['db_name'],
vars['db_user'],
)]
)
server_stop = ';'.join(
[bash_init,
'pg_ctl stop']
)
for cmd in (init_db,
create_user,
create_db,
grant,
server_stop,
):
ret = os.system('bash -c "%s"' % cmd)
if ret>0:
print "\n\n%s" % (
'Error while initiliasing the database.\n'
'More likely Postgresql binaries were not found in your path. Do you have'
' built postgresql somewhere or launched minimerge to build it'
' after adding postgresql-x.x to your project minibuild?.'
)
sys.exit(1)
for f in ('pg_hba.conf',
'pg_ident.conf' ):
fp = os.path.join(db_path, f)
if os.path.isfile(fp):
remove_path(fp)
def webbuilder_process(context, request):
valid_actions = ['submit_cgwbDownload', 'submit_cgwbGenerate']
valid_actions = ['submit_cgwbDownload',]
errors = []
output_dir_prefix = None
params = dict(request.params)
actions = [action for action in request.params if re.match('^submit_', action)]
noecho = [params.pop(action) for action in actions]
configuration = params.pop('configuration', None)
download_path = None
if not configuration:
errors.append('Choose a configuration')
if not configuration in root.configurations:
errors.append('Invalid configurations')
if not errors:
for action in actions:
if action in valid_actions:
try:
paster = get_paster(configuration)
createcmd = [a
for a in pkg_resources.iter_entry_points(
'paste.global_paster_command',
'create')
][0].load()
noecho = [params.update({param: True})
for param in params if params[param] in [u'on', u'checkbox_enabled']]
project = params.pop('project', None)
if not project:
project = ''
project = project.strip()
if not project:
raise Exception('Project does not exist or is empty')
if action == 'submit_cgwbDownload':
output_dir_prefix = tempfile.mkdtemp()
else:
output_dir_prefix = os.path.join(get_generation_path(), project)
if os.path.exists(output_dir_prefix) and len(os.listdir(output_dir_prefix))>0:
if not get_settings().get('debug', '').lower() == 'true' :
raise Exception('Directory %s exists and is not empty' % output_dir_prefix)
# keep track of boolean options for descendant templates
boolean_consumed_options = {}
for template in paster.templates_data:
cparams = params.copy()
output_dir = output_dir_prefix
if template['self'].output:
output_dir = os.path.join(output_dir, template['self'].output)
top = ['-q',
'-t', template['name'],
'--no-interactive',
'-o', output_dir,
project
]
if template['aliases']:
for var, alias, default in template['aliases']:
# we have a value
if alias in cparams:
cparams[var.name] = cparams[alias]
#blank value, resetting
else:
if var.name in cparams:
del cparams[var.name]
# set to false unticked options
for g in template['groups']:
for myo in g['options']:
if myo[1] == 'boolean':
boolean_consumed_options[myo[0].name] = cparams.get(myo[0].name, False)
for option in boolean_consumed_options:
if not option in cparams:
cparams[option] = False
# cleanup variables
names = dict([(var.name,var) for var in template['template'].vars])
for iv in cparams.keys():
if not iv in names:
del cparams[iv]
continue
else:
if not cparams[iv]:
if not names[iv].default:
del cparams[iv]
continue
# be sure not to have unicode params there because paster will swallow them up
top.extend(
["%s=%s" % i for i in cparams.items()]
)
cmd = createcmd('create')
try:
ret = cmd.run(top)
except Exception, e:
remove_path(output_dir)
raise
# do post genration stuff
postprocess(paster, output_dir_prefix, project, params)
if action == 'submit_cgwbDownload':
qsparams = dict(request.POST)
qsparams.update(params)
qsparams.update(boolean_consumed_options)
qs = urllib.urlencode(qsparams)
ficp = os.path.join(output_dir_prefix, 'LINK_TO_REGENERATE.html')
burl = request.route_url('collect', configuration=request.POST.get('configuration'))
genurl = burl+'?%s' % urllib.urlencode(
dict(oldparams=zlib.compress(qs, 9).encode('base64')))
fic = open(ficp, 'w')
fic.write(
'<html><body>'
'<a href="%s">Click here to go to the generation service</a>'
'</body></html>' % genurl)
fic.close()
ts = '%s'%datetime.datetime.now()
ts = ts.replace(':', '-').replace(' ', '_')[:-4]
filename = '%s-%s.tar.gz' % (project, ts)
file_path = os.path.join(get_generation_path(), filename)
fic = tarfile.open(file_path, 'w:gz')
fic.add(output_dir_prefix, './')
fic.close()
download_path = os.path.basename(file_path)
resp = Response(
body = open(file_path).read(),
content_type = 'application/x-tar-gz',
headerlist = [
('Content-Disposition',
'attachment; filename="%s"' % os.path.basename(file_path),
),
('Content-Transfer-Encoding', 'binary'),
("Content-Length", os.path.getsize(file_path)),
],
request = request,
)
os.remove(file_path)
return resp
except NoSuchConfigurationError:
errors.append(
'%s/ The required configuration '
'does not exists : %s' % (
action, configuration)
)
except MinibuildNotFoundException, e:
#raise
errors.append(
'<div class="error">'
'<p>%s/ Error while reading paster variables:</p>'
'<p class="pythonerror">%r</p>'
'<br/><p>Are you inside a minitage ? </p>'
'<br/><ul><li> If no, please untick minitage support in the minitage '
'section (press your browser\'s back button)</p>'
'<br/><li> If yes, you must set the "MT" environment variable to '
'the root installation of minitage or use the minitage.instance.env template '
'and source the resulting environment file before launching cgwb<br/>'
'<p>You can generate one with:</p><br/>'
'<pre>\n'
'\t$minitage/bin/paster create -t minitage.instances.env cgwb'
'</pre>'
'<br/><p>And before launching cgwb, if you do not have done that previously in your running shell:</p><br/>'
'<pre>\n'
'\tsource $minitage/bfg/cgwb(-dev)/sys/share/minitage/minitage.env'
'</pre>'
'</li></ul><br/>'
'' % (action, e)
)
except Parser.ParseError, e:
errors.append(
'<div class="error">'
'<p>%s/ Error while reading paster variables:</p>'
'<p class="pythonerror">%r</p>'
'<p class="pythonerror"><pre>%s</pre></p>'
'</div>'% (action, e, e.report())
)
def post(self, command, output_dir, vars):
# symlink conf, and logs directories inside $sys
print
print
dirs = [os.path.join(vars['sys'], 'bin'),
os.path.join(vars['sys'], 'etc', 'init.d')]
for directory in dirs:
for filep in os.listdir(directory):
p = os.path.join(directory, filep)
os.chmod(p, stat.S_IRGRP|stat.S_IXGRP|stat.S_IRWXU)
conf = os.path.join(
vars['sys'],
'var', 'data',
'tomcat',
vars['tomcat_instance'],
'conf'
)
logs = os.path.join(
vars['sys'],
'var', 'data',
'tomcat',
vars['tomcat_instance'],
'logs'
)
dconf = os.path.join(
vars['sys'],
'etc',
'tomcat',
vars['tomcat_instance']
)
dlogs = os.path.join(
vars['sys'],
'var',
'log',
'tomcat',
vars['tomcat_instance'],
)
for p in dlogs, dconf:
if os.path.exists(p):
remove_path(p)
os.symlink(conf, dconf)
os.symlink(logs, dlogs)
keysp = os.path.join(
os.path.dirname(conf),
'keys'
)
if not os.path.exists(keysp):
os.makedirs(keysp)
ssl.generate_prefixed_ssl_bundle(vars, vars['tomcat_instance'])
README = '\n\n%s' % (
"Installation is now finished.\n"
"------------------------------\n"
"\n"
" * You can find an init script to lunach your tomcat instance in %s/etc/init.d/%s_%s.%s.\n"
" * Your CATALINA_BASE is installed in %s.\n"
" * wrappers for various tomcat script have been isntalled in %s/bin.\n"
" * %s\n"
" * logs are in %s/var/log/tomvat/cas\n"
" * symlinks for the configuration directory have been made into %s/etc.\n"
"\n" % (
vars['sys'],
vars['project'],
vars['project'],
vars['tomcat_instance'],
os.path.dirname(conf),
vars['sys'],
', '.join (
[a
for a in os.listdir(os.path.join(vars['sys'], 'bin'))
if 'tomcat_%s' % vars['tomcat_instance'] in a]
),
vars['sys'],
vars['sys'],
)
)
for app in ['ROOT', 'manager', 'host-manager']:
o = os.path.join(vars['catalina_home'], 'webapps', app)
d = os.path.join(vars['catalina_base'], 'webapps', app)
msg = " * Copying %s application from %s.\n" % (app, o)
README += msg
copy_tree(o, d)
if vars['inside_minitage']:
README += '\n%s' % (
"Please verify that tomcat-%s is in your minibuild's dependencies\n"
"---------------------------------------------------------------------\n"
"If not, please add it and run the following commands:\n"
" minimerge -v tomcat-%s\n"
" source %s/share/minitage/minitage.env\n"
" $MT/paster create -t minitage.instances.env %s\n"
"\n" % (version, version, vars['sys'], vars['project'])
)
readmep = os.path.join(vars['path'], 'README.tomcat.%s' % vars['tomcat_instance'])
print README
print "Those informations have been saved in %s" % readmep
open(readmep, 'w').write(README)
self.write_config(vars)
def install(self):
"""installs an egg
"""
self.cache = os.path.join(
self.buildout['buildout'].get(
'download-directory',
os.path.join(self.buildout['buildout']['directory'], 'downloads')
), 'minitage', 'fetcher'
)
if not os.path.exists(self.cache):
os.makedirs(self.cache)
directories = []
self.logger.info('Start checkouts')
for url, url_infos in self.urls.items():
dest = url_infos.get('directory')
if not dest:
dest = os.path.basename(dest)
if not dest.startswith('/'):
dest = os.path.join(
self.options['location'],
dest
)
cache_fname = os.path.basename(url)
cache_downloaded = os.path.join(self.cache, cache_fname)
downloaded = False
fname = ''
if os.path.isfile(cache_downloaded):
if test_md5(cache_downloaded, url_infos.get('revision', 1)):
downloaded = True
self.logger.info('%s is already downloaded' %
cache_downloaded)
fname = cache_downloaded
if not downloaded:
fname = self._download(url=url,
destination=dest,
cache=False)
if ('unpack' in self.options):
try:
# try to unpack
f = IUnpackerFactory()
u = f(fname)
tmpdest = tempfile.mkdtemp()
ftmpdest = tmpdest
if u:
if os.path.exists(dest):
fcontents = os.listdir(dest)
c = len(fcontents)
fbasename = os.path.basename(fname)
if c > 1:
for fcontent in fcontents:
if fbasename != fcontent:
remove_path(fcontent)
u.unpack(fname, tmpdest)
if not os.path.exists(self.cache):
os.makedirs(self.cache)
try:
os.rename(fname, cache_downloaded)
except:
if os.path.exists(cache_downloaded):
remove_path(cache_downloaded)
shutil.copy2(fname, cache_downloaded)
remove_path(fname)
c = os.listdir(tmpdest)
if len(c) == 1:
ftmpdest = os.path.join(tmpdest, c[0])
copy_tree(ftmpdest, dest)
shutil.rmtree(tmpdest)
except Exception, e:
message = 'Can\'t install file %s in its destination %s.'
self.logger.info('Completed dowbload of %s in %s' % (url, dest))
directories.append(fname)
