def repo_setup(repo, ref):
""" Clone repository """
puts(cyan(">>> Setting up repository %s with ref %s..." % (repo, ref)))
topsrcdir = repo_check(repo, check_path=False, workdir=False)
workdir = repo_check(repo, check_path=False, workdir=True)
gitdir = os.path.join(topsrcdir, '.git')
if not exists_local(env.CFG_SRCDIR):
res = confirm("Create repository root %s?" % env.CFG_SRCDIR)
if not res:
abort(red("Cannot continue") % env)
else:
sudo_local("mkdir -p %s" % env.CFG_SRCDIR, user=env.CFG_INVENIO_USER)
if not exists_local(gitdir) and exists_local(topsrcdir):
res = confirm("Remove %s (it does not seem to be a git repository)?" % topsrcdir)
if not res:
abort(red("Cannot continue") % env)
else:
sudo_local("rm -Rf %s" % topsrcdir, user=env.CFG_INVENIO_USER)
if not exists_local(gitdir):
git_clone(repo)
if not exists_local(workdir):
git_newworkdir(repo)
git_checkout(repo, ref)
repo_prepare(repo)
def venv_load():
"""
Load an archived virtualenv
The task will extract an archived virtual environment created with
:meth:`~venv_dump`. Normally this command is invoked
indirectly via the compound task :meth:`inveniofab.compound.load` which
takes care of loading the database after extracting the virtual
environment.
"""
puts(cyan(">>> Loading archived virtualenv..."))
ctx = {
'dirname': os.path.dirname(env.CFG_INVENIO_PREFIX),
'basename': os.path.basename(env.CFG_INVENIO_PREFIX),
}
archive_file = "%(dirname)s/%(basename)s.tar.gz" % ctx
if not exists_local(archive_file):
abort(red("Archived virtualenv does not exists - cannot continue") % env)
# Remove previous installation
if exists_local(env.CFG_INVENIO_PREFIX):
res = confirm("Remove installation in %(CFG_INVENIO_PREFIX)s ?" % env)
if not res:
abort(red("Cannot continue") % env)
else:
sudo_local("rm -Rf %(CFG_INVENIO_PREFIX)s" % env)
cmds = [
"cd %(dirname)s",
"tar -xvzf %(basename)s.tar.gz",
]
sudo_local(" && ".join(cmds) % ctx, user=env.CFG_INVENIO_USER)
def apache_conf():
""" Upload and update Apache configuration """
puts(cyan(">>> Configuring Apache..." % env))
conf_files = ['etc/apache/invenio-apache-vhost.conf', 'etc/apache/invenio-apache-vhost-ssl.conf']
conf_files = [(p, os.path.join(env.CFG_INVENIO_PREFIX, p)) for p in conf_files]
for local_file, remote_file in conf_files:
puts(">>> Writing %s ..." % remote_file)
try:
if not exists_local(os.path.dirname(remote_file)):
sudo_local("mkdir -p %s" % os.path.dirname(remote_file), user=env.CFG_INVENIO_USER)
write_template(remote_file, env, tpl_file=local_file, use_sudo=True)
except TemplateNotFound:
abort(red("Could not find template %s" % local_file))
apache_conf = env.get('CFG_APACHE_CONF', '/etc/httpd/conf/httpd.conf')
if confirm("Include created files in %s?" % apache_conf):
if exists_local(apache_conf, use_sudo=True):
lines = ["Include %s" % r for (l, r) in conf_files]
if is_local():
with settings(host_string="localhost"):
append(apache_conf, lines, use_sudo=True)
else:
append(apache_conf, lines, use_sudo=True)
else:
warn(red("File %s does not exists" % apache_conf))
sudo_local("%(CFG_INVENIO_APACHECTL)s configtest" % env)
def git_clone(repo):
topsrcdir = repo_check(repo, check_path=False, workdir=False)
try:
repo_url = dict(env.CFG_INVENIO_REPOS)[repo]['repository']
except KeyError:
abort(red("Repository URL for %s not defined" % repo))
basename = os.path.basename(topsrcdir)
parent = os.path.dirname(topsrcdir)
if exists_local(topsrcdir):
res = confirm("Remove existing source code in %s ?" % topsrcdir)
if not res:
abort(red("Cannot continue") % env)
else:
sudo_local("rm -Rf %s" % topsrcdir, user=env.CFG_INVENIO_USER)
else:
if not exists_local(parent):
sudo_local("mkdir -p %s" % parent, user=env.CFG_INVENIO_USER)
ctx = {
'basename': basename,
'parent': parent,
'topsrcdir': topsrcdir,
'url': repo_url,
}
sudo_local("cd %(parent)s; git clone %(url)s %(basename)s " % ctx, user=env.CFG_INVENIO_USER)
def venv_dump():
"""
Archive a virtualenv
The task will create an archive ``<virtualenv name>.tar.gz`` of the entire
virtual environment. If an existing archive already exists, the user will
be asked for confirmation to remove it. Normally this command is invoked
indirectly via the compound task :meth:`inveniofab.compound.dump` which
takes care of dumping the database prior to archiving the virtual
environment.
"""
puts(cyan(">>> Creating archive of virtualenv in %(CFG_INVENIO_PREFIX)s..." % env))
ctx = {
'dirname': os.path.dirname(env.CFG_INVENIO_PREFIX),
'basename': os.path.basename(env.CFG_INVENIO_PREFIX),
}
archive_file = "%(dirname)s/%(basename)s.tar.gz" % ctx
if exists_local(archive_file):
res = confirm("Existing archive already exists - remove?")
if not res:
abort(red("Cannot continue") % env)
else:
sudo_local("rm -Rf %s" % archive_file, user=env.CFG_INVENIO_USER)
cmds = [
"cd %(dirname)s",
"tar -cvzf %(basename)s.tar.gz %(basename)s",
]
sudo_local(" && ".join(cmds) % ctx, user=env.CFG_INVENIO_USER)
def mysql_dump(outputdir=None):
"""
Dump database to file
"""
with hide('commands'):
puts(cyan(">>> Dumping database ..." % env))
answers = {
'user': env.CFG_DATABASE_USER,
'password': env.CFG_DATABASE_PASS,
}
# Escape quote characters
answers['password'] = answers['password'].replace("'", "\'").replace('"', '\"').replace('$', '\\$')
if not outputdir:
outputdir = env.CFG_DATABASE_DUMPDIR
if not exists_local(outputdir):
abort(red("Output directory %s does not exists" % outputdir))
user_pw = '-u %(user)s --password=%(password)s' % answers if answers['password'] else '-u %(user)s' % answers
outfile = os.path.join(outputdir, "%s.sql" % env.CFG_DATABASE_NAME)
outfile_gz = "%s.gz" % outfile
for f in [outfile, outfile_gz]:
if exists_local(f):
res = confirm("Remove existing DB dump in %s ?" % f)
if not res:
abort(red("Cannot continue") % env)
else:
sudo_local("rm -Rf %s" % f)
ctx = {
'user_pw': user_pw,
'host': env.CFG_DATABASE_HOST,
'name': env.CFG_DATABASE_NAME,
'port': env.CFG_DATABASE_PORT,
'outfile_gz': outfile_gz,
'outfile': outfile,
}
# Run commands
sudo_local('mysqldump %(user_pw)s -h %(host)s -P %(port)s --skip-opt '
'--add-drop-table --add-locks --create-options --quick '
'--extended-insert --set-charset --disable-keys %(name)s '
'| gzip -c > %(outfile_gz)s' % ctx, user=env.CFG_INVENIO_USER)
def git_describe(repo):
topsrcdir = repo_check(repo, workdir=True)
version_gen = os.path.join(topsrcdir, 'git-version-gen')
if exists_local(version_gen):
output = sudo_local("cd %s; %s" % (topsrcdir, version_gen), capture=True, user=env.CFG_INVENIO_USER)
else:
output = sudo_local("cd %s; git describe --always --abbrev=4 HEAD" % topsrcdir, capture=True, user=env.CFG_INVENIO_USER)
return output
def default_configure_hook(ctx):
"""
Default way to configure a repo. Assumes repo has a configure script.
"""
if exists_local(os.path.join(ctx['topsrcdir'], "configure")):
with settings(warn_only=True):
sudo_local("cd %(topsrcdir)s && make -s clean" % ctx, user=env.CFG_INVENIO_USER)
sudo_local("cd %(topsrcdir)s && ./configure --prefix=%(CFG_INVENIO_PREFIX)s "
"--with-python=%(CFG_INVENIO_PREFIX)s/bin/python" % ctx, user=env.CFG_INVENIO_USER)
sudo_local("cd %(topsrcdir)s && make -s clean" % ctx, user=env.CFG_INVENIO_USER)
def venv_libxslt_install():
"""
Install libxslt into virtualenv
"""
ctx = {
'libxslt': 'libxslt-1.1.26',
'libxslt_ver': '1.1.26',
'libxslt_path': os.path.join(env.CFG_INVENIO_PREFIX, 'libxslt-1.1.26'),
}
ctx.update(env)
if exists_local(ctx['libxslt_path']):
sudo_local("rm -Rf %(libxslt_path)s" % ctx, user=env.CFG_INVENIO_USER)
if exists_local("%(libxslt_path)s.tar.gz" % ctx):
sudo_local("rm -Rf %(libxslt_path)s.tar.gz" % ctx, user=env.CFG_INVENIO_USER)
sudo_local("cd %(CFG_INVENIO_PREFIX)s; wget ftp://xmlsoft.org/libxml2/%(libxslt)s.tar.gz; tar -xzf %(libxslt)s.tar.gz" % ctx, user=env.CFG_INVENIO_USER)
sudo_local("cd %(libxslt_path)s; ./configure --prefix=%(CFG_INVENIO_PREFIX)s --with-python=%(CFG_INVENIO_PREFIX)s/bin/python; make; cd python; make install" % ctx, user=env.CFG_INVENIO_USER)
sudo_local("rm -Rf %(libxslt_path)s" % ctx, user=env.CFG_INVENIO_USER)
sudo_local("rm -Rf %(libxslt_path)s.tar.gz" % ctx, user=env.CFG_INVENIO_USER)
def repo_check(repo, check_path=True, workdir=False):
all_repos = [x[0] for x in env.CFG_INVENIO_REPOS]
if repo not in all_repos:
abort(red("%s is not a valid repository." % repo))
repo_path = os.path.join(env.CFG_SRCWORKDIR if workdir and env.WITH_WORKDIR
else env.CFG_SRCDIR, repo)
if check_path and not exists_local(repo_path):
abort(red("Repository does not exists %s" % repo_path))
return repo_path
def mysql_load(dumpfile=None, stored_answers=None):
"""
Load MySQL dump file
"""
with hide('commands'):
puts(cyan(">>> Loading database dump..."))
if not dumpfile:
dumpfile = os.path.join(env.CFG_DATABASE_DUMPDIR, "%s.sql.gz" % env.CFG_DATABASE_NAME)
if not exists_local(dumpfile):
abort("File %s does not exists." % dumpfile)
answers = prompt_and_check([
("MySQL admin user:"******"user"),
("MySQL admin password:"******"password")
], mysql_admin_check(env.CFG_DATABASE_HOST, env.CFG_DATABASE_PORT),
cache_key="mysql://%s:%s" % (env.CFG_DATABASE_HOST, env.CFG_DATABASE_PORT),
stored_answers=stored_answers)
# Escape quote characters
answers['password'] = answers['password'].replace("'", "\'").replace('"', '\"').replace('$', '\\$')
user_pw = '-u %(user)s --password=%(password)s' % answers if answers['password'] else '-u %(user)s' % answers
if dumpfile.endswith(".gz"):
dumpfile_stream = "gunzip -c %s" % dumpfile
else:
dumpfile_stream = "cat %s" % dumpfile
ctx = {
'user_pw': user_pw,
'host': env.CFG_DATABASE_HOST,
'name': env.CFG_DATABASE_NAME,
'user': env.CFG_DATABASE_USER,
'password': env.CFG_DATABASE_PASS,
'port': env.CFG_DATABASE_PORT,
'dumpfile': dumpfile,
'dumpfile_stream': dumpfile_stream,
}
# Escape quote characters
ctx['password'] = ctx['password'].replace("'", "\'").replace('"', '\"').replace('$', '\\$')
if confirm("This will erease all data in the existing database. Are you sure you want to load %(dumpfile)s?" % ctx):
mysql_dropdb(stored_answers=answers)
mysql_createdb(stored_answers=answers)
sudo_local('%(dumpfile_stream)s | mysql %(user_pw)s -h %(host)s -P %(port)s -f %(name)s' % ctx, user=env.CFG_INVENIO_USER)
return dumpfile
def _apache_conf(files, with_python=True):
puts(cyan(">>> Configuring Apache..." % env))
conf_files = files
conf_files = [(p, os.path.join(env.CFG_INVENIO_PREFIX, p)) for p in conf_files]
if with_python:
pyver = python_version()
ctx = {'PYVER': pyver}
else:
ctx = {}
ctx.update(env)
for local_file, remote_file in conf_files:
puts(">>> Writing %s ..." % remote_file)
try:
if not exists_local(os.path.dirname(remote_file)):
sudo_local("mkdir -p %s" % os.path.dirname(remote_file), user=env.CFG_INVENIO_USER)
write_template(remote_file, ctx, tpl_file=local_file, use_sudo=True)
except TemplateNotFound:
abort(red("Could not find template %s" % local_file))
apache_conf = env.get('CFG_APACHE_CONF', '/etc/httpd/conf/httpd.conf')
if confirm("Include created files in %s?" % apache_conf):
if exists_local(apache_conf, use_sudo=True):
lines = ["Include %s" % r for (l, r) in conf_files]
if is_local():
with settings(host_string="localhost"):
append(apache_conf, lines, use_sudo=True)
else:
append(apache_conf, lines, use_sudo=True)
else:
warn(red("File %s does not exists" % apache_conf))
sudo_local("%(CFG_INVENIO_APACHECTL)s configtest" % env)
def venv_drop():
""" Drop virtualenv environment """
# Checks
if 'CFG_INVENIO_PREFIX' not in env:
abort(red("CFG_INVENIO_PREFIX is not specified in env.") % env)
puts(cyan(">>> Dropping virtualenv in %(CFG_INVENIO_PREFIX)s..." % env))
# Remove previous installation
if exists_local(env.CFG_INVENIO_PREFIX):
res = confirm("Remove installation in %(CFG_INVENIO_PREFIX)s ?" % env)
if not res:
abort(red("Cannot continue") % env)
else:
sudo_local("sudo rm -Rf %(CFG_INVENIO_PREFIX)s" % env)
else:
puts(">>> Nothing to remove - %(CFG_INVENIO_PREFIX)s does not exists..." % env)
def repo_update(**kwargs):
""" Pull repository updates """
# Check repositories to update/setup
repo_refs = []
for repo, info in env.CFG_INVENIO_REPOS:
if repo in kwargs:
ref = kwargs[repo]
if not ref:
ref = None
repo_refs.append((repo, ref))
del kwargs[repo]
else:
try:
ref = info['ref']
except KeyError:
ref = None
repo_refs.append((repo, ref))
if kwargs:
for repo, ref in kwargs.items():
abort(red("Invalid repository %s" % repo))
# Run update
for repo, ref in repo_refs:
topsrcdir = repo_check(repo, check_path=False, workdir=True)
gitdir = os.path.join(topsrcdir, '.git')
if not exists_local(gitdir):
repo_setup(repo, ref)
else:
if ref:
puts(cyan(">>> Updating repository %s to ref %s..." % (repo, ref)))
git_fetch(repo)
git_checkout(repo, ref)
else:
commit_id = git_describe(repo)
puts(cyan(">>> No ref specified for repository %s (currently at HEAD, commit %s) ..." % (repo, commit_id)))
def venv_create():
"""
Create virtualenv environment
The virtualenv is created in ``env.CFG_INVENIO_PREFIX``, and will also
create ``lib/python/invenio/`` and symlink it the virtualenv's
site-packages, as well as ``var/tmp/ooffice-tmp-files`` (via sudo). If
``env.WITH_DEVSCRIPTS`` is ``True``, invenio-devscripts will be installed.
If ``env.WITH_WORKDIR`` is ``True`` git-new-workdir will be installed.
Lastly, it will append render the template ``activate-profile.tpl`` and
append it to ``bin/activate``. The script will setup common needed
environment variables that e.g. invenio-devscripts depend on.
If an existing environment already exists, the user will be asked for
confirmation to remove the directory (using sudo, due to the directory
``var/tmp/ooffice-tmp-files`` which is created using sudo).
"""
# Checks
if 'CFG_INVENIO_PREFIX' not in env:
abort(red("CFG_INVENIO_PREFIX is not specified in env.") % env)
puts(cyan(">>> Creating virtualenv in %(CFG_INVENIO_PREFIX)s..." % env))
# Remove previous installation
if exists_local(env.CFG_INVENIO_PREFIX):
res = confirm("Remove installation in %(CFG_INVENIO_PREFIX)s ?" % env)
if not res:
abort(red("Cannot continue") % env)
else:
sudo_local("rm -Rf %(CFG_INVENIO_PREFIX)s" % env)
# Create virtual environment
dirname = os.path.dirname(env.CFG_INVENIO_PREFIX)
basename = os.path.basename(env.CFG_INVENIO_PREFIX)
sudo_local("mkdir -p %s" % dirname, user=env.CFG_INVENIO_USER)
sudo_local("cd %s && virtualenv -p %s %s" % (dirname, env.PYTHON, basename), user=env.CFG_INVENIO_USER)
# Create needed symboic links
pyver = python_version()
sudo_local("mkdir -p %(CFG_INVENIO_PREFIX)s/lib/python/invenio" % env, user=env.CFG_INVENIO_USER)
sudo_local(("mkdir -p %(CFG_INVENIO_PREFIX)s/lib/python" + pyver + "/site-packages") % env, user=env.CFG_INVENIO_USER)
sudo_local(("ln -s %(CFG_INVENIO_PREFIX)s/lib/python/invenio %(CFG_INVENIO_PREFIX)s/lib/python" + pyver + "/site-packages/invenio") % env, user=env.CFG_INVENIO_USER)
# Write extras into the activate script
write_template(os.path.join(env.CFG_INVENIO_PREFIX, 'bin/activate'), env, tpl_file='activate-profile.tpl', append=True, mark="ACTIVATE_PROFILE", use_sudo=True)
# Install devscripts
if env.WITH_DEVSCRIPTS:
puts(">>> Installing invenio-devscripts...")
sudo_local("cd %(CFG_INVENIO_PREFIX)s && git clone https://github.com/tiborsimko/invenio-devscripts.git" % env, user=env.CFG_INVENIO_USER)
sudo_local("cd %(CFG_INVENIO_PREFIX)s && mv invenio-devscripts/* bin/" % env, user=env.CFG_INVENIO_USER)
if env.WITH_WORKDIR:
puts(">>> Installing git-new-workdir...")
sudo_local('wget -O %(CFG_INVENIO_PREFIX)s/bin/git-new-workdir "http://repo.or.cz/w/git.git/blob_plain/HEAD:/contrib/workdir/git-new-workdir"' % env, user=env.CFG_INVENIO_USER)
sudo_local("chmod +x %(CFG_INVENIO_PREFIX)s/bin/git-new-workdir" % env, user=env.CFG_INVENIO_USER)
# OpenOffice temporary directory
sudo_local("mkdir -p %(CFG_INVENIO_PREFIX)s/var/tmp/ooffice-tmp-files" % env, user=env.CFG_INVENIO_USER)
sudo_local("sudo chown -R nobody %(CFG_INVENIO_PREFIX)s/var/tmp/ooffice-tmp-files" % env)
sudo_local("sudo chmod -R 755 %(CFG_INVENIO_PREFIX)s/var/tmp/ooffice-tmp-files" % env)
def default_prepare_hook(ctx):
"""
Default way to prepare source code which uses autotools.
"""
if exists_local(os.path.join(ctx['topsrcdir'], "configure.ac")):
sudo_local("cd %(topsrcdir)s; aclocal && automake -a && autoconf -f" % ctx, user=env.CFG_INVENIO_USER)
