def repo_setup(repo, ref): """ Clone repository """ puts(cyan(">>> Setting up repository %s with ref %s..." % (repo, ref))) topsrcdir = repo_check(repo, check_path=False, workdir=False) workdir = repo_check(repo, check_path=False, workdir=True) gitdir = os.path.join(topsrcdir, '.git') if not exists_local(env.CFG_SRCDIR): res = confirm("Create repository root %s?" % env.CFG_SRCDIR) if not res: abort(red("Cannot continue") % env) else: sudo_local("mkdir -p %s" % env.CFG_SRCDIR, user=env.CFG_INVENIO_USER) if not exists_local(gitdir) and exists_local(topsrcdir): res = confirm("Remove %s (it does not seem to be a git repository)?" % topsrcdir) if not res: abort(red("Cannot continue") % env) else: sudo_local("rm -Rf %s" % topsrcdir, user=env.CFG_INVENIO_USER) if not exists_local(gitdir): git_clone(repo) if not exists_local(workdir): git_newworkdir(repo) git_checkout(repo, ref) repo_prepare(repo)
def venv_load(): """ Load an archived virtualenv The task will extract an archived virtual environment created with :meth:`~venv_dump`. Normally this command is invoked indirectly via the compound task :meth:`inveniofab.compound.load` which takes care of loading the database after extracting the virtual environment. """ puts(cyan(">>> Loading archived virtualenv...")) ctx = { 'dirname': os.path.dirname(env.CFG_INVENIO_PREFIX), 'basename': os.path.basename(env.CFG_INVENIO_PREFIX), } archive_file = "%(dirname)s/%(basename)s.tar.gz" % ctx if not exists_local(archive_file): abort(red("Archived virtualenv does not exists - cannot continue") % env) # Remove previous installation if exists_local(env.CFG_INVENIO_PREFIX): res = confirm("Remove installation in %(CFG_INVENIO_PREFIX)s ?" % env) if not res: abort(red("Cannot continue") % env) else: sudo_local("rm -Rf %(CFG_INVENIO_PREFIX)s" % env) cmds = [ "cd %(dirname)s", "tar -xvzf %(basename)s.tar.gz", ] sudo_local(" && ".join(cmds) % ctx, user=env.CFG_INVENIO_USER)
def apache_conf(): """ Upload and update Apache configuration """ puts(cyan(">>> Configuring Apache..." % env)) conf_files = ['etc/apache/invenio-apache-vhost.conf', 'etc/apache/invenio-apache-vhost-ssl.conf'] conf_files = [(p, os.path.join(env.CFG_INVENIO_PREFIX, p)) for p in conf_files] for local_file, remote_file in conf_files: puts(">>> Writing %s ..." % remote_file) try: if not exists_local(os.path.dirname(remote_file)): sudo_local("mkdir -p %s" % os.path.dirname(remote_file), user=env.CFG_INVENIO_USER) write_template(remote_file, env, tpl_file=local_file, use_sudo=True) except TemplateNotFound: abort(red("Could not find template %s" % local_file)) apache_conf = env.get('CFG_APACHE_CONF', '/etc/httpd/conf/httpd.conf') if confirm("Include created files in %s?" % apache_conf): if exists_local(apache_conf, use_sudo=True): lines = ["Include %s" % r for (l, r) in conf_files] if is_local(): with settings(host_string="localhost"): append(apache_conf, lines, use_sudo=True) else: append(apache_conf, lines, use_sudo=True) else: warn(red("File %s does not exists" % apache_conf)) sudo_local("%(CFG_INVENIO_APACHECTL)s configtest" % env)
def git_clone(repo): topsrcdir = repo_check(repo, check_path=False, workdir=False) try: repo_url = dict(env.CFG_INVENIO_REPOS)[repo]['repository'] except KeyError: abort(red("Repository URL for %s not defined" % repo)) basename = os.path.basename(topsrcdir) parent = os.path.dirname(topsrcdir) if exists_local(topsrcdir): res = confirm("Remove existing source code in %s ?" % topsrcdir) if not res: abort(red("Cannot continue") % env) else: sudo_local("rm -Rf %s" % topsrcdir, user=env.CFG_INVENIO_USER) else: if not exists_local(parent): sudo_local("mkdir -p %s" % parent, user=env.CFG_INVENIO_USER) ctx = { 'basename': basename, 'parent': parent, 'topsrcdir': topsrcdir, 'url': repo_url, } sudo_local("cd %(parent)s; git clone %(url)s %(basename)s " % ctx, user=env.CFG_INVENIO_USER)
def venv_dump(): """ Archive a virtualenv The task will create an archive ``<virtualenv name>.tar.gz`` of the entire virtual environment. If an existing archive already exists, the user will be asked for confirmation to remove it. Normally this command is invoked indirectly via the compound task :meth:`inveniofab.compound.dump` which takes care of dumping the database prior to archiving the virtual environment. """ puts(cyan(">>> Creating archive of virtualenv in %(CFG_INVENIO_PREFIX)s..." % env)) ctx = { 'dirname': os.path.dirname(env.CFG_INVENIO_PREFIX), 'basename': os.path.basename(env.CFG_INVENIO_PREFIX), } archive_file = "%(dirname)s/%(basename)s.tar.gz" % ctx if exists_local(archive_file): res = confirm("Existing archive already exists - remove?") if not res: abort(red("Cannot continue") % env) else: sudo_local("rm -Rf %s" % archive_file, user=env.CFG_INVENIO_USER) cmds = [ "cd %(dirname)s", "tar -cvzf %(basename)s.tar.gz %(basename)s", ] sudo_local(" && ".join(cmds) % ctx, user=env.CFG_INVENIO_USER)
def mysql_dump(outputdir=None): """ Dump database to file """ with hide('commands'): puts(cyan(">>> Dumping database ..." % env)) answers = { 'user': env.CFG_DATABASE_USER, 'password': env.CFG_DATABASE_PASS, } # Escape quote characters answers['password'] = answers['password'].replace("'", "\'").replace('"', '\"').replace('$', '\\$') if not outputdir: outputdir = env.CFG_DATABASE_DUMPDIR if not exists_local(outputdir): abort(red("Output directory %s does not exists" % outputdir)) user_pw = '-u %(user)s --password=%(password)s' % answers if answers['password'] else '-u %(user)s' % answers outfile = os.path.join(outputdir, "%s.sql" % env.CFG_DATABASE_NAME) outfile_gz = "%s.gz" % outfile for f in [outfile, outfile_gz]: if exists_local(f): res = confirm("Remove existing DB dump in %s ?" % f) if not res: abort(red("Cannot continue") % env) else: sudo_local("rm -Rf %s" % f) ctx = { 'user_pw': user_pw, 'host': env.CFG_DATABASE_HOST, 'name': env.CFG_DATABASE_NAME, 'port': env.CFG_DATABASE_PORT, 'outfile_gz': outfile_gz, 'outfile': outfile, } # Run commands sudo_local('mysqldump %(user_pw)s -h %(host)s -P %(port)s --skip-opt ' '--add-drop-table --add-locks --create-options --quick ' '--extended-insert --set-charset --disable-keys %(name)s ' '| gzip -c > %(outfile_gz)s' % ctx, user=env.CFG_INVENIO_USER)
def git_describe(repo): topsrcdir = repo_check(repo, workdir=True) version_gen = os.path.join(topsrcdir, 'git-version-gen') if exists_local(version_gen): output = sudo_local("cd %s; %s" % (topsrcdir, version_gen), capture=True, user=env.CFG_INVENIO_USER) else: output = sudo_local("cd %s; git describe --always --abbrev=4 HEAD" % topsrcdir, capture=True, user=env.CFG_INVENIO_USER) return output
def default_configure_hook(ctx): """ Default way to configure a repo. Assumes repo has a configure script. """ if exists_local(os.path.join(ctx['topsrcdir'], "configure")): with settings(warn_only=True): sudo_local("cd %(topsrcdir)s && make -s clean" % ctx, user=env.CFG_INVENIO_USER) sudo_local("cd %(topsrcdir)s && ./configure --prefix=%(CFG_INVENIO_PREFIX)s " "--with-python=%(CFG_INVENIO_PREFIX)s/bin/python" % ctx, user=env.CFG_INVENIO_USER) sudo_local("cd %(topsrcdir)s && make -s clean" % ctx, user=env.CFG_INVENIO_USER)
def venv_libxslt_install(): """ Install libxslt into virtualenv """ ctx = { 'libxslt': 'libxslt-1.1.26', 'libxslt_ver': '1.1.26', 'libxslt_path': os.path.join(env.CFG_INVENIO_PREFIX, 'libxslt-1.1.26'), } ctx.update(env) if exists_local(ctx['libxslt_path']): sudo_local("rm -Rf %(libxslt_path)s" % ctx, user=env.CFG_INVENIO_USER) if exists_local("%(libxslt_path)s.tar.gz" % ctx): sudo_local("rm -Rf %(libxslt_path)s.tar.gz" % ctx, user=env.CFG_INVENIO_USER) sudo_local("cd %(CFG_INVENIO_PREFIX)s; wget ftp://xmlsoft.org/libxml2/%(libxslt)s.tar.gz; tar -xzf %(libxslt)s.tar.gz" % ctx, user=env.CFG_INVENIO_USER) sudo_local("cd %(libxslt_path)s; ./configure --prefix=%(CFG_INVENIO_PREFIX)s --with-python=%(CFG_INVENIO_PREFIX)s/bin/python; make; cd python; make install" % ctx, user=env.CFG_INVENIO_USER) sudo_local("rm -Rf %(libxslt_path)s" % ctx, user=env.CFG_INVENIO_USER) sudo_local("rm -Rf %(libxslt_path)s.tar.gz" % ctx, user=env.CFG_INVENIO_USER)
def repo_check(repo, check_path=True, workdir=False): all_repos = [x[0] for x in env.CFG_INVENIO_REPOS] if repo not in all_repos: abort(red("%s is not a valid repository." % repo)) repo_path = os.path.join(env.CFG_SRCWORKDIR if workdir and env.WITH_WORKDIR else env.CFG_SRCDIR, repo) if check_path and not exists_local(repo_path): abort(red("Repository does not exists %s" % repo_path)) return repo_path
def mysql_load(dumpfile=None, stored_answers=None): """ Load MySQL dump file """ with hide('commands'): puts(cyan(">>> Loading database dump...")) if not dumpfile: dumpfile = os.path.join(env.CFG_DATABASE_DUMPDIR, "%s.sql.gz" % env.CFG_DATABASE_NAME) if not exists_local(dumpfile): abort("File %s does not exists." % dumpfile) answers = prompt_and_check([ ("MySQL admin user:"******"user"), ("MySQL admin password:"******"password") ], mysql_admin_check(env.CFG_DATABASE_HOST, env.CFG_DATABASE_PORT), cache_key="mysql://%s:%s" % (env.CFG_DATABASE_HOST, env.CFG_DATABASE_PORT), stored_answers=stored_answers) # Escape quote characters answers['password'] = answers['password'].replace("'", "\'").replace('"', '\"').replace('$', '\\$') user_pw = '-u %(user)s --password=%(password)s' % answers if answers['password'] else '-u %(user)s' % answers if dumpfile.endswith(".gz"): dumpfile_stream = "gunzip -c %s" % dumpfile else: dumpfile_stream = "cat %s" % dumpfile ctx = { 'user_pw': user_pw, 'host': env.CFG_DATABASE_HOST, 'name': env.CFG_DATABASE_NAME, 'user': env.CFG_DATABASE_USER, 'password': env.CFG_DATABASE_PASS, 'port': env.CFG_DATABASE_PORT, 'dumpfile': dumpfile, 'dumpfile_stream': dumpfile_stream, } # Escape quote characters ctx['password'] = ctx['password'].replace("'", "\'").replace('"', '\"').replace('$', '\\$') if confirm("This will erease all data in the existing database. Are you sure you want to load %(dumpfile)s?" % ctx): mysql_dropdb(stored_answers=answers) mysql_createdb(stored_answers=answers) sudo_local('%(dumpfile_stream)s | mysql %(user_pw)s -h %(host)s -P %(port)s -f %(name)s' % ctx, user=env.CFG_INVENIO_USER) return dumpfile
def _apache_conf(files, with_python=True): puts(cyan(">>> Configuring Apache..." % env)) conf_files = files conf_files = [(p, os.path.join(env.CFG_INVENIO_PREFIX, p)) for p in conf_files] if with_python: pyver = python_version() ctx = {'PYVER': pyver} else: ctx = {} ctx.update(env) for local_file, remote_file in conf_files: puts(">>> Writing %s ..." % remote_file) try: if not exists_local(os.path.dirname(remote_file)): sudo_local("mkdir -p %s" % os.path.dirname(remote_file), user=env.CFG_INVENIO_USER) write_template(remote_file, ctx, tpl_file=local_file, use_sudo=True) except TemplateNotFound: abort(red("Could not find template %s" % local_file)) apache_conf = env.get('CFG_APACHE_CONF', '/etc/httpd/conf/httpd.conf') if confirm("Include created files in %s?" % apache_conf): if exists_local(apache_conf, use_sudo=True): lines = ["Include %s" % r for (l, r) in conf_files] if is_local(): with settings(host_string="localhost"): append(apache_conf, lines, use_sudo=True) else: append(apache_conf, lines, use_sudo=True) else: warn(red("File %s does not exists" % apache_conf)) sudo_local("%(CFG_INVENIO_APACHECTL)s configtest" % env)
def venv_drop(): """ Drop virtualenv environment """ # Checks if 'CFG_INVENIO_PREFIX' not in env: abort(red("CFG_INVENIO_PREFIX is not specified in env.") % env) puts(cyan(">>> Dropping virtualenv in %(CFG_INVENIO_PREFIX)s..." % env)) # Remove previous installation if exists_local(env.CFG_INVENIO_PREFIX): res = confirm("Remove installation in %(CFG_INVENIO_PREFIX)s ?" % env) if not res: abort(red("Cannot continue") % env) else: sudo_local("sudo rm -Rf %(CFG_INVENIO_PREFIX)s" % env) else: puts(">>> Nothing to remove - %(CFG_INVENIO_PREFIX)s does not exists..." % env)
def repo_update(**kwargs): """ Pull repository updates """ # Check repositories to update/setup repo_refs = [] for repo, info in env.CFG_INVENIO_REPOS: if repo in kwargs: ref = kwargs[repo] if not ref: ref = None repo_refs.append((repo, ref)) del kwargs[repo] else: try: ref = info['ref'] except KeyError: ref = None repo_refs.append((repo, ref)) if kwargs: for repo, ref in kwargs.items(): abort(red("Invalid repository %s" % repo)) # Run update for repo, ref in repo_refs: topsrcdir = repo_check(repo, check_path=False, workdir=True) gitdir = os.path.join(topsrcdir, '.git') if not exists_local(gitdir): repo_setup(repo, ref) else: if ref: puts(cyan(">>> Updating repository %s to ref %s..." % (repo, ref))) git_fetch(repo) git_checkout(repo, ref) else: commit_id = git_describe(repo) puts(cyan(">>> No ref specified for repository %s (currently at HEAD, commit %s) ..." % (repo, commit_id)))
def venv_create(): """ Create virtualenv environment The virtualenv is created in ``env.CFG_INVENIO_PREFIX``, and will also create ``lib/python/invenio/`` and symlink it the virtualenv's site-packages, as well as ``var/tmp/ooffice-tmp-files`` (via sudo). If ``env.WITH_DEVSCRIPTS`` is ``True``, invenio-devscripts will be installed. If ``env.WITH_WORKDIR`` is ``True`` git-new-workdir will be installed. Lastly, it will append render the template ``activate-profile.tpl`` and append it to ``bin/activate``. The script will setup common needed environment variables that e.g. invenio-devscripts depend on. If an existing environment already exists, the user will be asked for confirmation to remove the directory (using sudo, due to the directory ``var/tmp/ooffice-tmp-files`` which is created using sudo). """ # Checks if 'CFG_INVENIO_PREFIX' not in env: abort(red("CFG_INVENIO_PREFIX is not specified in env.") % env) puts(cyan(">>> Creating virtualenv in %(CFG_INVENIO_PREFIX)s..." % env)) # Remove previous installation if exists_local(env.CFG_INVENIO_PREFIX): res = confirm("Remove installation in %(CFG_INVENIO_PREFIX)s ?" % env) if not res: abort(red("Cannot continue") % env) else: sudo_local("rm -Rf %(CFG_INVENIO_PREFIX)s" % env) # Create virtual environment dirname = os.path.dirname(env.CFG_INVENIO_PREFIX) basename = os.path.basename(env.CFG_INVENIO_PREFIX) sudo_local("mkdir -p %s" % dirname, user=env.CFG_INVENIO_USER) sudo_local("cd %s && virtualenv -p %s %s" % (dirname, env.PYTHON, basename), user=env.CFG_INVENIO_USER) # Create needed symboic links pyver = python_version() sudo_local("mkdir -p %(CFG_INVENIO_PREFIX)s/lib/python/invenio" % env, user=env.CFG_INVENIO_USER) sudo_local(("mkdir -p %(CFG_INVENIO_PREFIX)s/lib/python" + pyver + "/site-packages") % env, user=env.CFG_INVENIO_USER) sudo_local(("ln -s %(CFG_INVENIO_PREFIX)s/lib/python/invenio %(CFG_INVENIO_PREFIX)s/lib/python" + pyver + "/site-packages/invenio") % env, user=env.CFG_INVENIO_USER) # Write extras into the activate script write_template(os.path.join(env.CFG_INVENIO_PREFIX, 'bin/activate'), env, tpl_file='activate-profile.tpl', append=True, mark="ACTIVATE_PROFILE", use_sudo=True) # Install devscripts if env.WITH_DEVSCRIPTS: puts(">>> Installing invenio-devscripts...") sudo_local("cd %(CFG_INVENIO_PREFIX)s && git clone https://github.com/tiborsimko/invenio-devscripts.git" % env, user=env.CFG_INVENIO_USER) sudo_local("cd %(CFG_INVENIO_PREFIX)s && mv invenio-devscripts/* bin/" % env, user=env.CFG_INVENIO_USER) if env.WITH_WORKDIR: puts(">>> Installing git-new-workdir...") sudo_local('wget -O %(CFG_INVENIO_PREFIX)s/bin/git-new-workdir "http://repo.or.cz/w/git.git/blob_plain/HEAD:/contrib/workdir/git-new-workdir"' % env, user=env.CFG_INVENIO_USER) sudo_local("chmod +x %(CFG_INVENIO_PREFIX)s/bin/git-new-workdir" % env, user=env.CFG_INVENIO_USER) # OpenOffice temporary directory sudo_local("mkdir -p %(CFG_INVENIO_PREFIX)s/var/tmp/ooffice-tmp-files" % env, user=env.CFG_INVENIO_USER) sudo_local("sudo chown -R nobody %(CFG_INVENIO_PREFIX)s/var/tmp/ooffice-tmp-files" % env) sudo_local("sudo chmod -R 755 %(CFG_INVENIO_PREFIX)s/var/tmp/ooffice-tmp-files" % env)
def default_prepare_hook(ctx): """ Default way to prepare source code which uses autotools. """ if exists_local(os.path.join(ctx['topsrcdir'], "configure.ac")): sudo_local("cd %(topsrcdir)s; aclocal && automake -a && autoconf -f" % ctx, user=env.CFG_INVENIO_USER)