pavement.py

from __future__ import with_statement 
from paver import svn
from paver.easy import *
from paver.easy import options
from paver.path25 import pushd
import functools
import os
import sys
import time
from datetime import date
import socket
import ConfigParser
import pkg_resources
import subprocess
import shutil
from shutil import copy
import zipfile
import tarfile
import urllib

assert sys.version_info >= (2,6), \
       SystemError("GeoNode Build requires python 2.6 or better")


options(
    config=Bunch(
        ini=path('shared/build.ini'),
        package_dir = path('shared/package')
    ),
    minilib=Bunch(extra_files=['virtual', 'doctools', 'misctasks']),
    sphinx=Bunch(
        docroot='docs',
        builddir="_build",
        sourcedir="source"
    ),
    virtualenv=Bunch(
        packages_to_install=[
            'http://bitbucket.org/ianb/pip/get/2cb1db7b2baf.gz#egg=pip',
            'jstools',
            'virtualenv'
        ],
        dest_dir='./',
        install_paver=True,
        script_name='bootstrap.py',
        paver_command_line='post_bootstrap'
    ),
    deploy=Bunch(
#        pavement=path('shared/package/pavement.py'),
        req_file=path('shared/package/requirements.txt'),
        packages_to_install=['pip'],
        dest_dir='./',
#        install_paver=True,
#        paver_command_line='post_bootstrap'      
    ),
    host=Bunch(
    	bind='localhost'
    )
)

venv = os.environ.get('VIRTUAL_ENV')
bundle = path('shared/geonode.pybundle')
dl_cache = "--download-cache=./build"
dlname = 'geonode.bundle'
gs_data = "gs-data"
geoserver_target = path('src/geoserver-geonode-ext/target/geoserver.war')
geonetwork_target = path('webapps/geonetwork.war')
def geonode_client_target(): return options.deploy.out_dir / "geonode-client.zip"
geonode_client_target_war = path('webapps/geonode-client.war')

deploy_req_txt = """
# NOTE... this file is generated
-r %(venv)s/shared/requirements.txt
-e %(venv)s/src/GeoNodePy
""" % locals()

@task
def auto(options):
    cp = ConfigParser.ConfigParser()
    cp.read(options.config.ini)
    options.config.parser = cp

    # set a few vars from the config ns
    package_dir = options.deploy.out_dir = options.config.package_dir
    options.deploy.script_name=package_dir / 'bootstrap.py'

    # set windows dependent opts
    platform_options(options)

@task
def install_deps(options):
    """Installs all the python deps from a requirements file"""
    if bundle.exists():
        info('using to install python deps bundle')
        call_task('install_bundle')
    else:
        info('Installing from requirements file. '\
             'Use "paver bundle_deps" to create an install bundle')
        pip_install("-r shared/%s" % options.config.corelibs)
        pip_install("-r shared/%s" % options.config.devlibs)
        if options.config.platform == "win32":
            info("You will need to install 'PIL' and 'ReportLab' "\
                 "separately to do PDF generation")

@task
def bundle_deps(options):
    """
    Create a pybundle of all python dependencies.  If created, this
    will be the default for installing python deps.
    """
    pip_bundle("-r shared/requirements.txt %s" % bundle)


@task
@needs(['download_bundle'])
def install_bundle(options):
    """
    Installs a bundle of dependencies located at %s.
    """ % bundle
    
    info('install the bundle')
    pip_install(bundle)


@task
def download_bundle(options):
    """
    Downloads zipped bundle of python dependencies to %s. Does not overwrite.
    """ % bundle
    
    bpath = bundle.abspath()
    if not bundle.exists():
        with pushd('shared'):
            grab("http://dev.capra.opengeo.org/repo/%s.zip" % dlname, bpath)
    else:
        info("Skipping download. 'rm bundle  %s' if you need a fresh download. " % bundle)


@task
def install_25_deps(options):
    """Fetch python 2_5-specific dependencies (not maintained)"""
    pass

    
@task
@needs(['install_deps'])
def post_bootstrap(options):
    """installs the current package"""
    pip = path(options.config.bin) / "pip"
    sh('%s install -e %s' %(pip, path("src/GeoNodePy")))


#TODO Move svn urls out to a config file

def grab(src, dest):
    from urllib import urlretrieve
    urlretrieve(str(src), str(dest))

@task
def setup_gs_data(options):
    """Fetch a data directory to use with GeoServer for testing."""
    src_url = str(options.config.parser.get('geoserver', 'gs_data_url'))
    shared = path("./shared")
    if not shared.exists():
        shared.mkdir()

    dst_url = shared / "geonode-geoserver-data.zip"
    grab(src_url, dst_url)

    if getattr(options, 'clean', False): path(gs_data).rmtree()
    if not path(gs_data).exists(): unzip_file(dst_url, gs_data)


@task
@needs(['setup_gs_data'])
def setup_geoserver(options):
    """Prepare a testing instance of GeoServer."""
    with pushd('src/geoserver-geonode-ext'):
        sh("mvn clean install")

@task
def setup_geonetwork(options):
    """Fetch the geonetwork.war and intermap.war to use with GeoServer for testing."""
    war_zip_file = options.config.parser.get('geonetwork', 'geonetwork_zip')
    src_url = str(options.config.parser.get('geonetwork', 'geonetwork_war_url') +  war_zip_file)
    info("geonetwork url: %s" %src_url)
    # where to download the war files. If changed change also
    # src/geoserver-geonode-ext/jetty.xml accordingly

    webapps = path("./webapps")
    if not webapps.exists():
        webapps.mkdir()

    dst_url = webapps / war_zip_file
    dst_war = webapps / "geonetwork.war"
    deployed_url = webapps / "geonetwork"

    if getattr(options, 'clean', False):
        deployed_url.rmtree()

    if not dst_war.exists():
    	info("getting geoserver.war")
        grab(src_url, dst_url)
        zip_extractall(zipfile.ZipFile(dst_url), webapps)
    if not deployed_url.exists():
        zip_extractall(zipfile.ZipFile(dst_war), deployed_url)

    src_url = str(options.config.parser.get('geonetwork', 'intermap_war_url'))
    dst_url = webapps / "intermap.war"

    if not dst_url.exists():
        grab(src_url, dst_url)

@task
@needs([
    'setup_geoserver',
    'setup_geonetwork',
    'setup_geonode_client'
])
def setup_webapps(options):
    pass

@task
@needs([
    'install_deps',
    'setup_webapps',
    'generate_geoserver_token',
    'sync_django_db',
    'package_client'
])
def build(options):
    """Get dependencies and generally prepare a GeoNode development environment."""
    info("""GeoNode development environment successfully set up.\nIf you have not set up an administrative account, please do so now.\nUse "paver host" to start up the server.""")

@task
@needs([
    'install_deps',
    'setup_geonode_client',
    'sync_django_db',
    'package_client'
])
def fastbuild(options):
    """Get dependencies and generally prepare a GeoNode development environment."""
    info("""GeoNode development environment successfully set up minus GeoServer and Geonetwork.\nIf you have not set up an administrative account, please do so now.\nUse "paver host" to start up the server.""")


@task
def setup_geonode_client(options):
    """
    Fetch geonode-client
    """
    static = path("./src/GeoNodePy/geonode/media/static")
    if not static.exists():
        static.mkdir()

    sh("git submodule update --init")

    with pushd("src/geonode-client/"):
        sh("mvn clean compile")
    
    src_zip = "src/geonode-client/build/geonode-client.zip"
    zip_extractall(zipfile.ZipFile(src_zip), static)

@task
def sync_django_db(options):
    sh("django-admin.py syncdb --settings=geonode.settings --noinput")
    try:
        sh("django-admin.py syncdb --database=wmdata --settings=geonode.settings --noinput")
    except:
        info("******CREATION OF GAZETTEER TABLE FAILED - if you want the gazetteer enabled, \n \
unescape the 'DATABASES' AND 'DATABASE_ROUTERS' settings in your settings file \n \
and modify the default values if necessary")
    sh("django-admin.py migrate --settings=geonode.settings --noinput")

@task
def generate_geoserver_token(options):
    gs_token_file = 'geoserver_token'
    if not os.path.exists(gs_token_file):
        from random import choice
        import string
        chars = string.letters + string.digits + "-_!@#$*"
        token = ''
        for i in range(32):
            token += choice(chars)
        tf = open('geoserver_token', 'w')
        tf.write(token)
        tf.close()

@task
def package_dir(options):
    """
    Adds a packaging directory
    """
    if not options.deploy.out_dir.exists():
        options.config.package_dir.mkdir()


@task
@needs('package_dir', 'setup_geonode_client')
@cmdopts([
    ('use_war', 'w', 'Use a war to deploy geonode-client')
])
def package_client(options):
    """Package compressed client resources (JavaScript, CSS, images)."""

    if(hasattr(options, 'use_war')): 
    	geonode_client_target_war.copy(options.deploy.out_dir)
    else:
        # Extract static files to static_location
    	geonode_media_dir = path("./src/GeoNodePy/geonode/media")
        static_location = geonode_media_dir / "static"

        dst_zip = "src/geonode-client/build/geonode-client.zip"

        zip_extractall(zipfile.ZipFile(dst_zip), static_location)
        os.remove(dst_zip)

@task
@needs('package_dir', 'setup_geoserver')
def package_geoserver(options):
    """Package GeoServer WAR file with appropriate extensions."""
    geoserver_target.copy(options.deploy.out_dir)


@task
@needs('package_dir', 'setup_geonetwork')
def package_geonetwork(options):
    """Package GeoNetwork WAR file for deployment."""
    geonetwork_target.copy(options.deploy.out_dir)


@task
@needs('package_dir')
def package_webapp(options):
    """Package (Python, Django) web application and dependencies."""
    with pushd('src/GeoNodePy'):
        sh('python setup.py egg_info sdist')
        
    req_file = options.deploy.req_file
    req_file.write_text(deploy_req_txt)
    pip_bundle("-r %s %s/geonode-webapp.pybundle" % (req_file, options.deploy.out_dir))


@task
@needs(
    'build',
    'package_geoserver',
    'package_geonetwork',
    'package_webapp',
    'package_bootstrap'
)
def package_all(options):
    info('all is packaged, ready to deploy')


def create_version_name():
    # we'll use the geonodepy version as our "official" version number
    # for now
    slug = "GeoNode-%s-%s" % (
        pkg_resources.get_distribution('GeoNodePy').version,
        date.today().isoformat()
    )

    return slug

@task
def make_devkit(options):
    import virtualenv
    from urllib import urlretrieve

    (path("package") / "devkit" / "share").makedirs()
    pip_bundle("package/devkit/share/geonode-core.pybundle -r shared/devkit.requirements")
    script = virtualenv.create_bootstrap_script("""
import os, subprocess, zipfile

def after_install(options, home_dir):
    if sys.platform == 'win32':
        bin = 'Scripts'
    else:
        bin = 'bin'

    installer_base = os.path.abspath(os.path.dirname(__file__))

    def pip(*args):
        subprocess.call([os.path.join(home_dir, bin, "pip")] + list(args))

    pip("install", os.path.join(installer_base, "share", "geonode-core.pybundle"))
    setup_jetty(source=os.path.join(installer_base, "share"), dest=os.path.join(home_dir, "share"))

def setup_jetty(source, dest):
    jetty_zip = os.path.join(source, "jetty-distribution-7.0.2.v20100331.zip")
    jetty_dir = os.path.join(dest, "jetty-distribution-7.0.2.v20100331")

    zipfile.ZipFile(jetty_zip).extractall(dest)
    shutil.rmtree(os.path.join(jetty_dir, "contexts"))
    shutil.rmtree(os.path.join(jetty_dir, "webapps"))
    os.mkdir(os.path.join(jetty_dir, "contexts"))
    os.mkdir(os.path.join(jetty_dir, "webapps"))

    deployments = [
        ('geoserver', 'geoserver-geonode-dev.war'),
        ('geonetwork', 'geonetwork.war'),
        ('media', 'geonode-client.zip')
    ]

    for context, archive in deployments:
        src = os.path.join(source, archive)
        dst = os.path.join(jetty_dir, "webapps", context)
        zipfile.ZipFile(src).extractall(dst)
""")

    open((path("package")/"devkit"/"go-geonode.py"), 'w').write(script)
    urlretrieve(
        "http://download.eclipse.org/jetty/7.0.2.v20100331/dist/jetty-distribution-7.0.2.v20100331.zip",
        "package/devkit/share/jetty-distribution-7.0.2.v20100331.zip"
    )
    urlretrieve(
        "http://pypi.python.org/packages/source/p/pip/pip-0.7.1.tar.gz",
        "package/devkit/share/pip-0.7.1.tar.gz"
    )
    geoserver_target.copy("package/devkit/share")
    geonetwork_target.copy("package/devkit/share")
    geonode_client_target().copy("package/devkit/share")
        
@task
@cmdopts([
    ('name=', 'n', 'Release number or name'),
    ('no_svn', 'D', 'Do not append svn version number as part of name '),
    ('append_to=', 'a', 'append to release name'),
    ('skip_packaging', 'y', 'Do not call package_all when creating a release'),
])
def make_release(options):
    """
    Creates a tarball to use for building the system elsewhere
    (production, distribution, etc)
    """

    if not hasattr(options, 'skip_packaging'):
        call_task("package_all")
    if hasattr(options, 'name'):
        pkgname = options.name
    else:
        pkgname = create_version_name()
        if hasattr(options, 'append_to'):
            pkgname += options.append_to

    with pushd('shared'):
        out_pkg = path(pkgname)
        out_pkg.rmtree()
        path('./package').copytree(out_pkg)

        tar = tarfile.open("%s.tar.gz" % out_pkg, "w:gz")
        for file in out_pkg.walkfiles():
            tar.add(file)
        tar.add('README.release.rst', arcname=('%s/README.rst' % out_pkg))
        tar.close()

        out_pkg.rmtree()
        info("%s.tar.gz created" % out_pkg.abspath())
                            

def unzip_file(src, dest):
    zip = zipfile.ZipFile(src)
    if not path(dest).exists():
        path(dest).makedirs()
        
    for name in zip.namelist():
        if name.endswith("/"):
            (path(dest) / name).makedirs()
        else:
            parent, file = path(name).splitpath()
            parent = path(dest) / parent
            if parent and not parent.isdir():
                path(parent).makedirs()
            out = open(path(parent) / file, 'wb')
            out.write(zip.read(name))
            out.close()


@task
def checkup_spec(options):
    parser = options.config.parser
    svn.checkup(parser.get('doc', 'spec_url'), path('docs') / 'spec')


def pip(*args):
    try:
        pkg_resources.require('pip>=0.6')
    except :
        error("**ATTENTION**: Update your 'pip' to at least 0.6")
        raise

    full_path_pip = options.config.bin / 'pip'

    sh("%(env)s %(cmd)s %(args)s" % {
        "env": options.config.pip_flags, 
        "cmd": full_path_pip,
        "args": " ".join(args)
    })

pip_install = functools.partial(pip, 'install', dl_cache)
pip_bundle = functools.partial(pip, 'bundle', dl_cache)


@task
@needs('package_dir')
def package_bootstrap(options):
    """Create a bootstrap script for deployment"""

    try:
        from paver.virtual import bootstrap
        options.virtualenv = options.deploy
        call_task("paver.virtual.bootstrap")
    except ImportError, e:
        info("VirtualEnv must be installed to enable 'paver bootstrap'. If you " + 
             "need this command, run: pip install virtualenv")


@task
def install_sphinx_conditionally(options):
    """if no sphinx, install it"""
    try:
        import sphinx
    except ImportError:
        sh("%s install sphinx" % (options.config.bin / 'pip'))

        # have to reload doctools so it will realize sphinx is now
        # available
        sys.modules['paver.doctools'] = reload(sys.modules['paver.doctools'])


@task
@needs('package_client')
@cmdopts([
    ('bind=', 'b', 'IP address to bind to. Default is localhost.')
])
def host(options):
    jettylog = open("jetty.log", "w")
    djangolog = open("django.log", "w")
    with pushd("src/geoserver-geonode-ext"):
        os.environ["MAVEN_OPTS"] = " ".join([
            "-XX:CompileCommand=exclude,net/sf/saxon/event/ReceivingContentHandler.startElement",
            "-Djetty.host=" + options.host.bind,
            "-Xmx512M",
            "-XX:MaxPermSize=128m"
        ])
        mvn = subprocess.Popen(
            ["mvn", "jetty:run"],
            stdout=jettylog,
            stderr=jettylog
        )
    django = subprocess.Popen([
            "paster", 
            "serve",
            "--reload",
	        "shared/dev-paste.ini"
        ],  
        stdout=djangolog,
        stderr=djangolog
    )

    def jetty_is_up():
        try:
            urllib.urlopen("http://" + options.host.bind + ":8001/geoserver/web/")
            return True
        except Exception, e:
            return False
    
    def django_is_up():
        try:
            urllib.urlopen("http://" + options.host.bind + ":8000")
            return True
        except Exception, e:
            return False

    socket.setdefaulttimeout(1)

    info("Django is starting up, please wait...")
    while not django_is_up():
        time.sleep(2)

    info("Logging servlet output to jetty.log and django output to django.log...")
    info("Jetty is starting up, please wait...")
    while not jetty_is_up():
        time.sleep(2)

    try:
        sh("django-admin.py updatelayers --settings=geonode.settings")
        
        info("Development GeoNode is running at http://" + options.host.bind + ":8000/")
        info("The GeoNode is an unstoppable machine")
        info("Press CTRL-C to shut down")
        django.wait()
        info("Django process terminated, see log for details.")
    finally:
        info("Shutting down...")
        try:
            django.terminate()
        except: 
            pass
        try:
            mvn.terminate()
        except:
            pass

        django.wait()
        mvn.wait()
        sys.exit()


@task
def test(options):
    sh("django-admin.py test --settings=geonode.settings")


def platform_options(options):
    "Platform specific options"
    options.config.platform = sys.platform
    
    # defaults:
    pip_flags = ""
    scripts = "bin"
    corelibs = "requirements.txt"
    devlibs = "dev-requirements.txt"

    if sys.platform == "win32":
        scripts = "Scripts"
    elif sys.platform == "darwin":
        pip_flags = "ARCHFLAGS='-arch i386'"
        
    options.config.bin = path(scripts)
    options.config.corelibs = corelibs
    options.config.devlibs = devlibs
    options.config.pip_flags = pip_flags

# include patched versions of zipfile code
# to extract zipfile dirs in python 2.6.1 and below...

def zip_extractall(zf, path=None, members=None, pwd=None):
    if sys.version_info >= (2, 6, 2):
        zf.extractall(path=path, members=members, pwd=pwd)
    else:
        _zip_extractall(zf, path=path, members=members, pwd=pwd)

def _zip_extractall(zf, path=None, members=None, pwd=None):
    """Extract all members from the archive to the current working
       directory. `path' specifies a different directory to extract to.
       `members' is optional and must be a subset of the list returned
       by namelist().
    """
    if members is None:
        members = zf.namelist()

    for zipinfo in members:
        _zip_extract(zf, zipinfo, path, pwd)

def _zip_extract(zf, member, path=None, pwd=None):
    """Extract a member from the archive to the current working directory,
       using its full name. Its file information is extracted as accurately
       as possible. `member' may be a filename or a ZipInfo object. You can
       specify a different directory using `path'.
    """
    if not isinstance(member, zipfile.ZipInfo):
        member = zf.getinfo(member)

    if path is None:
        path = os.getcwd()
    
    return _zip_extract_member(zf, member, path, pwd)


def _zip_extract_member(zf, member, targetpath, pwd):
    """Extract the ZipInfo object 'member' to a physical
       file on the path targetpath.
    """
    # build the destination pathname, replacing
    # forward slashes to platform specific separators.
    # Strip trailing path separator, unless it represents the root.
    if (targetpath[-1:] in (os.path.sep, os.path.altsep)
        and len(os.path.splitdrive(targetpath)[1]) > 1):
        targetpath = targetpath[:-1]

    # don't include leading "/" from file name if present
    if member.filename[0] == '/':
        targetpath = os.path.join(targetpath, member.filename[1:])
    else:
        targetpath = os.path.join(targetpath, member.filename)

    targetpath = os.path.normpath(targetpath)

    # Create all upper directories if necessary.
    upperdirs = os.path.dirname(targetpath)
    if upperdirs and not os.path.exists(upperdirs):
        os.makedirs(upperdirs)

    if member.filename[-1] == '/':
        if not os.path.isdir(targetpath):
            os.mkdir(targetpath)
        return targetpath

    source = zf.open(member, pwd=pwd)
    target = file(targetpath, "wb")
    shutil.copyfileobj(source, target)
    source.close()
    target.close()

    return targetpath