def run_tests(options):
"""
Executes the entire test suite.
"""
if options.get('coverage'):
prefix = 'coverage run --branch --source=geonode \
--omit="*/management/*,*/__init__*,*/views*,*/signals*,*/tasks*,*/test*,*/wsgi*,*/middleware*,*/search_indexes*,\
*/migrations*,*/context_processors*,geonode/qgis_server/*,geonode/upload/*,geonode/monitoring/*"'
else:
prefix = 'python'
local = options.get('local', 'false') # travis uses default to false
if not integration_tests and not integration_csw_tests and not integration_bdd_tests:
call_task('test', options={'prefix': prefix})
else:
if integration_tests:
call_task('test_integration')
# only start if using Geoserver backend
_backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND'])
if _backend == 'geonode.geoserver' and 'geonode.geoserver' in INSTALLED_APPS:
call_task('test_integration',
options={'name': 'geonode.upload.tests.integration'})
elif integration_csw_tests:
call_task('test_integration', options={'name': 'geonode.tests.csw'})
if integration_bdd_tests:
call_task('test_bdd', options={'local': local})
sh('flake8 geonode')
def deb(options):
"""
Creates debian packages.
Example uses:
paver deb
paver deb -k 12345
paver deb -k 12345 -p geonode/testing
"""
key = options.get("key", None)
ppa = options.get("ppa", None)
version, simple_version = versions()
info("Creating package for GeoNode version %s" % version)
with pushd("package"):
# Get rid of any uncommitted changes to debian/changelog
info("Getting rid of any uncommitted changes in debian/changelog")
sh("git checkout debian/changelog")
# Workaround for git-dch bug
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580
# path('.git').makedirs()
# Link the parent git repository folder in to the current one
# needed by git-dch
sh("ln -s ../.git .git")
# Install requirements
# sh('sudo apt-get -y install debhelper devscripts git-buildpackage')
sh(
(
"git-dch --spawn-editor=snapshot --git-author --new-version=%s"
" --id-length=6 --ignore-branch --release" % (simple_version)
)
)
deb_changelog = path("debian") / "changelog"
for line in fileinput.input([deb_changelog], inplace=True):
print line.replace("urgency=low", "urgency=high"),
## Revert workaround for git-dhc bug
sh("rm -rf .git")
if key is None and ppa is None:
# A local installable package
sh("debuild -uc -us -A")
elif key is None and ppa is not None:
# A sources package, signed by daemon
sh("debuild -S")
elif key is not None and ppa is None:
# A signed installable package
sh("debuild -k%s -A" % key)
elif key is not None and ppa is not None:
# A signed, source package
sh("debuild -k%s -S" % key)
if ppa is not None:
sh("dput ppa:%s geonode_%s_source.changes" % (ppa, simple_version))
def start_django():
"""
Start the GeoNode Django application
"""
bind = options.get("bind", "")
foreground = "" if options.get("foreground", False) else "&"
sh("python manage.py runserver %s %s" % (bind, foreground))
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
download_dir = path("downloaded")
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path("geoserver")
geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL)
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
grab(options.get("geoserver", GEOSERVER_URL), geoserver_bin, "geoserver binary")
grab(options.get("jetty", JETTY_RUNNER_URL), jetty_runner, "jetty runner")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir / "geoserver"
if not webapp_dir:
webapp_dir.makedirs()
print "extracting geoserver"
z = zipfile.ZipFile(geoserver_bin, "r")
z.extractall(webapp_dir)
_install_data_dir()
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
download_dir = path('downloaded')
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path('geoserver')
geoserver_bin = download_dir / os.path.basename(dev_config['GEOSERVER_URL'])
jetty_runner = download_dir / os.path.basename(dev_config['JETTY_RUNNER_URL'])
grab(options.get('geoserver', dev_config['GEOSERVER_URL']), geoserver_bin, "geoserver binary")
grab(options.get('jetty', dev_config['JETTY_RUNNER_URL']), jetty_runner, "jetty runner")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir / 'geoserver'
if not webapp_dir:
webapp_dir.makedirs()
print 'extracting geoserver'
z = zipfile.ZipFile(geoserver_bin, "r")
z.extractall(webapp_dir)
_install_data_dir()
def manifest():
"""Guarantee the existence of a basic MANIFEST.in.
manifest doc: http://docs.python.org/distutils/sourcedist.html#manifest
`options.paved.dist.manifest.include`: set of files (or globs) to include with the `include` directive.
`options.paved.dist.manifest.recursive_include`: set of files (or globs) to include with the `recursive-include` directive.
`options.paved.dist.manifest.prune`: set of files (or globs) to exclude with the `prune` directive.
`options.paved.dist.manifest.include_sphinx_docroot`: True -> sphinx docroot is added as `graft`
`options.paved.dist.manifest.include_sphinx_docroot`: True -> sphinx builddir is added as `prune`
"""
prune = options.paved.dist.manifest.prune
graft = set()
if options.paved.dist.manifest.include_sphinx_docroot:
docroot = options.get("docroot", "docs")
graft.update([docroot])
if options.paved.dist.manifest.exclude_sphinx_builddir:
builddir = docroot + "/" + options.get("builddir", ".build")
prune.update([builddir])
with open(options.paved.cwd / "MANIFEST.in", "w") as fo:
for item in graft:
fo.write("graft %s\n" % item)
for item in options.paved.dist.manifest.include:
fo.write("include %s\n" % item)
for item in options.paved.dist.manifest.recursive_include:
fo.write("recursive-include %s\n" % item)
for item in prune:
fo.write("prune %s\n" % item)
def start_django():
"""
Start the GeoNode Django application
"""
bind = options.get('bind', '')
foreground = '' if options.get('foreground', False) else '&'
sh('python manage.py runserver %s %s' % (bind, foreground))
def start_django():
"""
Start the GeoNode Django application
"""
settings = options.get('settings', '')
if settings:
settings = 'DJANGO_SETTINGS_MODULE=%s' % settings
bind = options.get('bind', '0.0.0.0:8000')
foreground = '' if options.get('foreground', False) else '&'
sh('%s python -W ignore manage.py runserver %s %s' % (settings, bind, foreground))
if ASYNC_SIGNALS:
celery_queues = [
"default",
"geonode",
"cleanup",
"update",
"email",
# Those queues are directly managed by messages.consumer
# "broadcast",
# "email.events",
# "all.geoserver",
# "geoserver.events",
# "geoserver.data",
# "geoserver.catalog",
# "notifications.events",
# "geonode.layer.viewer"
]
sh('%s celery -A geonode worker -Q %s -B -E -l INFO %s' % (settings, ",".join(celery_queues),foreground))
sh('%s python -W ignore manage.py runmessaging %s' % (settings, foreground))
def run_tests(options):
"""
Executes the entire test suite.
"""
if options.get('coverage'):
prefix = 'coverage run --branch --source=geonode --omit="*/management/*,geonode/contrib/*,*/test*,*/wsgi*,*/middleware*"'
else:
prefix = 'python'
local = options.get('local', 'false') # travis uses default to false
if not integration_tests:
sh('%s manage.py test geonode.tests.smoke %s %s' % (prefix, _keepdb, _parallel))
call_task('test', options={'prefix': prefix})
else:
call_task('test_integration')
call_task('test_integration', options={'name': 'geonode.tests.csw'})
# only start if using Geoserver backend
if 'geonode.geoserver' in INSTALLED_APPS and OGC_SERVER['default']['BACKEND'] == 'geonode.geoserver':
call_task('test_integration',
options={'name': 'geonode.upload.tests.integration',
'settings': 'geonode.upload.tests.test_settings'})
call_task('test_bdd', options={'local': local})
sh('flake8 geonode')
def sloccount():
'''Print "Source Lines of Code" and export to file.
Export is hudson_ plugin_ compatible: sloccount.sc
requirements:
- sloccount_ should be installed.
- tee and pipes are used
options.paved.pycheck.sloccount.param
.. _sloccount: http://www.dwheeler.com/sloccount/
.. _hudson: http://hudson-ci.org/
.. _plugin: http://wiki.hudson-ci.org/display/HUDSON/SLOCCount+Plugin
'''
# filter out subpackages
setup = options.get('setup')
packages = options.get('packages') if setup else None
if packages:
dirs = [x for x in packages if '.' not in x]
else:
dirs = ['.']
# sloccount has strange behaviour with directories,
# can cause exception in hudson sloccount plugin.
# Better to call it with file list
ls=[]
for d in dirs:
ls += list(path(d).walkfiles())
#ls=list(set(ls))
files=' '.join(ls)
param=options.paved.pycheck.sloccount.param
sh('sloccount {param} {files} | tee sloccount.sc'.format(param=param, files=files))
def deb(options):
"""
Creates debian packages.
Example uses:
paver deb
paver deb -k 12345
paver deb -k 12345 -p geonode/testing
"""
key = options.get('key', None)
ppa = options.get('ppa', None)
version, simple_version = versions()
info('Creating package for GeoNode version %s' % version)
# Get rid of any uncommitted changes to debian/changelog
info('Getting rid of any uncommitted changes in debian/changelog')
sh('git checkout package/debian/changelog')
# Workaround for git-dch bug
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580
sh('ln -s %s %s' % (os.path.realpath('.git'), os.path.realpath('package')))
with pushd('package'):
# Install requirements
# sh('sudo apt-get -y install debhelper devscripts git-buildpackage')
sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s'
' --id-length=6 --ignore-branch --release' % (simple_version)))
#In case you publish from Ubuntu Xenial (git-dch is removed from upstream)
# use the following line instead:
#sh(('gbp dch --spawn-editor=snapshot --git-author --new-version=%s'
# ' --id-length=6 --ignore-branch --release' % (simple_version)))
deb_changelog = path('debian') / 'changelog'
for line in fileinput.input([deb_changelog], inplace=True):
print line.replace("urgency=medium", "urgency=high"),
# Revert workaround for git-dhc bug
sh('rm -rf .git')
if key is None and ppa is None:
# A local installable package
sh('debuild -uc -us -A')
elif key is None and ppa is not None:
# A sources package, signed by daemon
sh('debuild -S')
elif key is not None and ppa is None:
# A signed installable package
sh('debuild -k%s -A' % key)
elif key is not None and ppa is not None:
# A signed, source package
sh('debuild -k%s -S' % key)
if ppa is not None:
sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))
def start_messaging():
"""
Start the GeoNode messaging server
"""
settings = options.get('settings', '')
if settings:
settings = 'DJANGO_SETTINGS_MODULE=%s' % settings
foreground = '' if options.get('foreground', False) else '&'
sh('%s python -W ignore manage.py runmessaging %s' % (settings, foreground))
def create(options):
"""create database objects and superuser account"""
args = ''
username = options.get('username', None)
password = options.get('password', None)
email = options.get('email', None)
if all([username, password, email]):
args = '%s %s %s' % (username, password, email)
sh('python GeoHealthCheck/models.py create %s' % args)
def deploy():
"""deploy install"""
venv = options.get('venv', None)
if venv is None:
raise ValueError('path to virtual environment required')
venv = os.path.abspath(venv)
loc = [
'import sys',
'sys.path.insert(0, "%s/app")' % venv,
'activate = "%s/bin/activate_this.py"' % venv,
'execfile(activate_this, {__file__: activate})',
'from app import APP as application'
]
with open(options.app.build / 'vocab.wsgi', 'w') as fileobj:
fileobj.write('\n'.join(loc))
loc = [
'WSGIDaemonProcess vocab user=user1 group=group1 threads=5',
'WSGIScriptAlias /vocab %s' % options.app.home,
'<Directory %s>' % path(venv),
' WSGIProcessGroup vocab',
' WSGIApplicationGroup %{GLOBAL}',
' Order deny,allow',
' Allow from all',
'</Directory>']
with open(options.app.build / 'osgeo.vocab.conf', 'w') as fileobj:
fileobj.write('\n'.join(loc))
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
fast = options.get('fast', False)
download_dir = path('downloaded')
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path('geoserver')
geoserver_bin = path('geoserver_ext/target/geoserver.war')
if not geoserver_bin.exists():
geoserver_bin = download_dir / os.path.basename(ROGUE_GEOSERVER_URL)
grab(ROGUE_GEOSERVER_URL, geoserver_bin, "geoserver binary")
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner")
data_dir = download_dir / os.path.basename(DATA_DIR_URL)
grab(DATA_DIR_URL, data_dir, "data dir")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir
if not webapp_dir:
webapp_dir.makedirs()
print 'extracting geoserver'
z = zipfile.ZipFile(geoserver_bin, "r")
z.extractall(webapp_dir)
# Set the geonode auth config to dev port 8000
sh("perl -pi.back -e 's/localhost/localhost:8000/g;' geoserver/data/security/auth/geonodeAuthProvider/config.xml")
def start_django():
"""
Start the GeoNode Django application
"""
bind = options.get('bind', '')
#Se comento la linea original para correr la app en una IP especifica
sh('python manage.py runserver %s &' % bind)
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
fast = options.get('fast', False)
download_dir = path('downloaded')
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path('geoserver')
geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL)
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
data_dir = download_dir / os.path.basename(DATA_DIR_URL)
grab(GEOSERVER_URL, geoserver_bin, "geoserver binary")
grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner")
grab(DATA_DIR_URL, data_dir, "data dir")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir / 'geoserver'
if not webapp_dir:
webapp_dir.makedirs()
print 'extracting geoserver'
with zipfile.ZipFile(geoserver_bin, "r") as z:
z.extractall(webapp_dir)
_install_data_dir()
def upload():
"""upload package zipfile to server"""
user = options.get("user", False)
if not user:
raise ValueError("OSGeo userid required")
password = getpass.getpass("Enter your password: "******"":
raise ValueError("password required")
call_task("package")
zipf = get_package_filename()
url = "http://%s:%s@%s:%d/%s" % (user, password, options.upload.host, options.upload.port, options.upload.endpoint)
info("Uploading to http://%s/%s" % (options.upload.host, options.upload.endpoint))
server = xmlrpclib.ServerProxy(url, verbose=False)
try:
with open(zipf) as zfile:
plugin_id, version_id = server.plugin.upload(xmlrpclib.Binary(zfile.read()))
info("Plugin ID: %s", plugin_id)
info("Version ID: %s", version_id)
except xmlrpclib.Fault as err:
error("ERROR: fault error")
error("Fault code: %d", err.faultCode)
error("Fault string: %s", err.faultString)
except xmlrpclib.ProtocolError as err:
error("Error: Protocol error")
error("%s : %s", err.errcode, err.errmsg)
if err.errcode == 403:
error("Invalid name and password")
def upload():
"""upload package zipfile to server"""
user = options.get('user', False)
if not user:
raise ValueError('OSGeo userid required')
password = getpass.getpass('Enter your password: '******'':
raise ValueError('password required')
call_task('package')
zipf = get_package_filename()
url = 'http://%s:%s@%s:%d/%s' % (user, password, options.upload.host,
options.upload.port,
options.upload.endpoint)
info('Uploading to http://%s/%s' % (options.upload.host,
options.upload.endpoint))
server = xmlrpclib.ServerProxy(url, verbose=False)
try:
with open(zipf) as zfile:
plugin_id, version_id = \
server.plugin.upload(xmlrpclib.Binary(zfile.read()))
info('Plugin ID: %s', plugin_id)
info('Version ID: %s', version_id)
except xmlrpclib.Fault, err:
error('ERROR: fault error')
error('Fault code: %d', err.faultCode)
error('Fault string: %s', err.faultString)
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
_reset()
# Start GeoServer
call_task('start_geoserver')
info("GeoNode is now available, running the tests now.")
name = options.get('name', 'geonode.tests.integration')
success = False
try:
if name == 'geonode.tests.csw':
call_task('sync')
call_task('start')
sh('sleep 30')
call_task('setup_data')
sh(('python manage.py test %s'
' --noinput --liveserver=0.0.0.0:8000' % name))
except BuildFailure as e:
info('Tests failed! %s' % str(e))
else:
success = True
finally:
# don't use call task here - it won't run since it already has
stop()
_reset()
if not success:
sys.exit(1)
def _adjust_options():
"""
Set default sphinx and gh_pages options.
"""
if options.get('_github_tools_options_adjusted') is None:
options.setdefault('sphinx', Bunch())
options.setdefault('gh_pages', Bunch())
options.sphinx.docroot = docroot \
= path( options.sphinx.get('docroot', 'docs'))
options.sphinx._buildir = buildir = \
docroot / options.sphinx.get('builddir', 'build')
options.sphinx._sourcedir = \
docroot / options.sphinx.get('sourcedir', 'source')
options.sphinx._doctrees = buildir / "doctrees"
options.sphinx._htmldir = htmldir = \
buildir / 'html'
gh_pages_root = options.gh_pages.get('root', None)
if gh_pages_root is None:
options.gh_pages.root = htmldir
else:
options.gh_pages.root = path(gh_pages_root)
gh_pages_htmlroot = options.gh_pages.get('htmlroot', None)
if gh_pages_htmlroot is None:
options.gh_pages.htmlroot = options.gh_pages.root
else:
options.gh_pages.htmlroot = path(gh_pages_htmlroot)
options.gh_pages.setdefault('remote_name', 'origin')
options.gh_pages.setdefault('master_branch', 'master')
options._github_tools_options_adjusted = True
def generate_csw_connections_file():
"""generate a CSW connections file from a flat file of CSW URLs"""
filename = options.get("filename", False)
if not filename:
raise ValueError("path to file of CSW URLs required")
conns = etree.Element("qgsCSWConnections")
conns.attrib["version"] = "1.0"
with open(filename) as connsfh:
for line in connsfh:
url = line.strip()
if not url: # blank line
continue
try:
csw = CatalogueServiceWeb(url)
title = unicode(csw.identification.title)
etree.SubElement(conns, "csw", name=title, url=url)
except Exception as err:
error("ERROR on CSW %s: %s", url, err)
with open("%s.xml" % filename, "w") as connsxmlfh:
connsxmlfh.write(etree.tostring(conns, encoding="utf-8"))
def start_geoserver(options):
"""
Start GeoServer with GeoNode extensions
"""
from geonode.settings import OGC_SERVER
GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION']
url = "http://localhost:8080/geoserver/"
if GEOSERVER_BASE_URL != url:
print 'your GEOSERVER_BASE_URL does not match %s' % url
sys.exit(1)
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
data_dir = path('geoserver/data').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/misc/jetty-runner.xml').abspath()
# @todo - we should not have set workdir to the datadir but a bug in geoserver
# prevents geonode security from initializing correctly otherwise
with pushd(data_dir):
javapath = "java"
loggernullpath = "/dev/null"
try:
sh(('java -version'))
except:
if not options.get('java_path', None):
print "Paver cannot find java in the Windows Environment. Please provide the --java_path flag with your full path to java.exe e.g. --java_path=C:/path/to/java/bin/java.exe"
sys.exit(1)
# if there are spaces
javapath = 'START /B "" "' + options['java_path'] + '"'
# cmd log file needs to exist in windows
# using folder from .gitignore
open("../../downloaded/null.txt", 'w+').close()
loggernullpath = "../../downloaded/null.txt"
sh((
'%(javapath)s -Xmx512m -XX:MaxPermSize=256m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
# workaround for JAI sealed jar issue and jetty classloader
' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()
))
info('Starting GeoServer on %s' % url)
# wait for GeoServer to start
started = waitfor(url)
info('The logs are available at %s' % log_file)
if not started:
# If applications did not start in time we will give the user a chance
# to inspect them and stop them manually.
info(('GeoServer never started properly or timed out.'
'It may still be running in the background.'))
sys.exit(1)
def generate_csw_connections_file():
"""generate a CSW connections file from a flat file of CSW URLs"""
filename = options.get('filename', False)
if not filename:
raise ValueError('path to file of CSW URLs required')
conns = etree.Element('qgsCSWConnections')
conns.attrib['version'] = '1.0'
with open(filename) as connsfh:
for line in connsfh:
url = line.strip()
if not url: # blank line
continue
try:
csw = CatalogueServiceWeb(url)
title = unicode(csw.identification.title)
conn = etree.SubElement(conns, 'csw', name=title, url=url)
except Exception as err:
error('ERROR on CSW %s: %s', url, err)
with open('%s.xml' % filename, 'w') as connsxmlfh:
connsxmlfh.write(etree.tostring(conns, encoding='utf-8'))
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
# only start if using Geoserver backend
_backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND'])
if _backend == 'geonode.qgis_server' or 'geonode.geoserver' not in INSTALLED_APPS:
return
download_dir = path('downloaded')
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path('geoserver')
geoserver_bin = download_dir / \
os.path.basename(dev_config['GEOSERVER_URL'])
jetty_runner = download_dir / \
os.path.basename(dev_config['JETTY_RUNNER_URL'])
if _django_11 and (integration_tests or integration_csw_tests or integration_bdd_tests):
"""Will make use of the docker container for the Integration Tests"""
pass
else:
grab(
options.get(
'geoserver',
dev_config['GEOSERVER_URL']),
geoserver_bin,
"geoserver binary")
grab(
options.get(
'jetty',
dev_config['JETTY_RUNNER_URL']),
jetty_runner,
"jetty runner")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir / 'geoserver'
if not webapp_dir:
webapp_dir.makedirs()
print 'extracting geoserver'
z = zipfile.ZipFile(geoserver_bin, "r")
z.extractall(webapp_dir)
_install_data_dir()
def test(options):
"""
Run GeoNode's Unit Test Suite
"""
prefix = options.get('prefix', 'python')
sh("%s manage.py test %s.tests --noinput -v 2" % (prefix,
'.tests '.join(GEONODE_APPS)))
def updategeoip(options):
"""
Update geoip db
"""
settings = options.get('settings', '')
if settings:
settings = 'DJANGO_SETTINGS_MODULE=%s' % settings
sh("%s python -W ignore manage.py updategeoip -o" % settings)
def setup_data():
"""
Import sample data (from gisdata package) into GeoNode
"""
import gisdata
ctype = options.get('type', None)
data_dir = gisdata.GOOD_DATA
if ctype in ['vector', 'raster', 'time']:
data_dir = os.path.join(gisdata.GOOD_DATA, ctype)
settings = options.get('settings', '')
if settings:
settings = 'DJANGO_SETTINGS_MODULE=%s' % settings
sh("%s python -W ignore manage.py importlayers %s -v2" % (settings, data_dir))
def add_language_catalogue(options):
"""adds new language profile"""
lang = options.get('lang', None)
if lang is None:
raise RuntimeError('missing lang argument')
sh('pybabel init -i %s -d %s -l %s' % (
options.base.pot, options.base.translations, lang))
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
# only start if using Geoserver backend
if 'geonode.geoserver' not in INSTALLED_APPS or OGC_SERVER['default']['BACKEND'] == 'geonode.qgis_server':
return
download_dir = path('downloaded')
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path('geoserver')
geoserver_bin = download_dir / \
os.path.basename(dev_config['GEOSERVER_URL'])
jetty_runner = download_dir / \
os.path.basename(dev_config['JETTY_RUNNER_URL'])
grab(
options.get(
'geoserver',
dev_config['GEOSERVER_URL']),
geoserver_bin,
"geoserver binary")
grab(
options.get(
'jetty',
dev_config['JETTY_RUNNER_URL']),
jetty_runner,
"jetty runner")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir / 'geoserver'
if not webapp_dir:
webapp_dir.makedirs()
print 'extracting geoserver'
z = zipfile.ZipFile(geoserver_bin, "r")
z.extractall(webapp_dir)
_install_data_dir()
def test(options):
"""
Run GeoNode's Unit Test Suite
"""
if on_travis:
if core_tests:
_apps = tuple(GEONODE_CORE_APPS)
if internal_apps_tests:
_apps = tuple(GEONODE_INTERNAL_APPS)
else:
_apps = tuple(GEONODE_APPS)
sh("%s manage.py test %s.tests --noinput %s %s" %
(options.get('prefix'), '.tests '.join(_apps), _keepdb, _parallel))
def sync(options):
"""
Run the migrate and migrate management commands to create and migrate a DB
"""
settings = options.get('settings', '')
if settings:
settings = 'DJANGO_SETTINGS_MODULE=%s' % settings
sh("%s python -W ignore manage.py makemigrations --noinput" % settings)
sh("%s python -W ignore manage.py migrate --noinput" % settings)
sh("%s python -W ignore manage.py loaddata sample_admin.json" % settings)
sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/default_oauth_apps.json" % settings)
sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/initial_data.json" % settings)
sh("%s python -W ignore manage.py set_all_layers_alternate" % settings)
def setup_data():
"""
Import sample data (from gisdata package) into GeoNode
"""
import gisdata
ctype = options.get('type', None)
data_dir = gisdata.GOOD_DATA
if ctype in ['vector', 'raster', 'time']:
data_dir = os.path.join(gisdata.GOOD_DATA, ctype)
sh("python manage.py importlayers %s -v2" % data_dir)
def run_tests(options):
"""
Executes the entire test suite.
"""
if options.get('coverage'):
prefix = 'coverage run --branch --source=geonode \
--omit="*/__init__*,*/test*,*/wsgi*,*/version*,*/migrations*,\
*/search_indexes*,*/management/*,*/context_processors*,*/qgis_server/*"'
else:
prefix = 'python'
local = options.get('local', 'false') # travis uses default to false
if not integration_tests and not integration_csw_tests and not integration_bdd_tests:
call_task('test', options={'prefix': prefix})
else:
if integration_tests:
call_task('test_integration', options={'prefix': prefix})
# only start if using Geoserver backend
_backend = os.environ.get('BACKEND',
OGC_SERVER['default']['BACKEND'])
if _backend == 'geonode.geoserver' and 'geonode.geoserver' in INSTALLED_APPS:
call_task(
'test_integration',
options={'name': 'geonode.geoserver.integration.tests'})
elif integration_csw_tests:
call_task('test_integration',
options={
'prefix': prefix,
'name': 'geonode.tests.csw'
})
if integration_bdd_tests:
call_task('test_bdd', options={'local': local})
sh('flake8 geonode')
def stop_qgis_server():
"""
Stop QGIS Server Backend.
"""
# only start if using QGIS Server backend
if 'geonode.qgis_server' not in INSTALLED_APPS or OGC_SERVER['default'][
'BACKEND'] == 'geonode.geoserver':
return
port = options.get('qgis_server_port', '9000')
sh('docker-compose -f docker-compose-qgis-server.yml down',
env={
'GEONODE_PROJECT_PATH': os.getcwd(),
'QGIS_SERVER_PORT': port
})
def sloccount():
'''Print "Source Lines of Code" and export to file.
Export is hudson_ plugin_ compatible: sloccount.sc
requirements:
- sloccount_ should be installed.
- tee and pipes are used
options.paved.pycheck.sloccount.param
.. _sloccount: http://www.dwheeler.com/sloccount/
.. _hudson: http://hudson-ci.org/
.. _plugin: http://wiki.hudson-ci.org/display/HUDSON/SLOCCount+Plugin
'''
# filter out subpackages
setup = options.get('setup')
packages = options.get('packages') if setup else None
if packages:
dirs = [x for x in packages if '.' not in x]
else:
dirs = ['.']
# sloccount has strange behaviour with directories,
# can cause exception in hudson sloccount plugin.
# Better to call it with file list
ls = []
for d in dirs:
ls += list(path(d).walkfiles())
# ls=list(set(ls))
files = ' '.join(ls)
param = options.paved.pycheck.sloccount.param
sh('sloccount {param} {files} | tee sloccount.sc'.format(param=param,
files=files))
def run_tests(options):
"""
Executes the entire test suite.
"""
if options.get('coverage'):
prefix = 'coverage run --branch --source=geonode --omit="*/management/*,geonode/contrib/*,*/test*,*/wsgi*,*/middleware*"'
else:
prefix = 'python'
local = options.get('local', 'false') # travis uses default to false
sh('%s manage.py test geonode.tests.smoke' % prefix)
call_task('test', options={'prefix': prefix})
call_task('test_integration')
call_task('test_integration', options={'name': 'geonode.tests.csw'})
# only start if using Geoserver backend
if 'geonode.geoserver' in INSTALLED_APPS:
call_task('test_integration',
options={
'name': 'geonode.upload.tests.integration',
'settings': 'geonode.upload.tests.test_settings'
})
call_task('test_bdd', options={'local': local})
sh('flake8 geonode')
def manifest():
"""Guarantee the existence of a basic MANIFEST.in.
manifest doc: http://docs.python.org/distutils/sourcedist.html#manifest
`options.paved.dist.manifest.include`: set of files (or globs) to include with the `include` directive.
`options.paved.dist.manifest.recursive_include`: set of files (or globs) to include with the `recursive-include` directive.
`options.paved.dist.manifest.prune`: set of files (or globs) to exclude with the `prune` directive.
`options.paved.dist.manifest.include_sphinx_docroot`: True -> sphinx docroot is added as `graft`
`options.paved.dist.manifest.include_sphinx_docroot`: True -> sphinx builddir is added as `prune`
"""
prune = options.paved.dist.manifest.prune
graft = set()
if options.paved.dist.manifest.include_sphinx_docroot:
docroot = options.get('docroot', 'docs')
graft.update([docroot])
if options.paved.dist.manifest.exclude_sphinx_builddir:
builddir = docroot + '/' + options.get("builddir", ".build")
prune.update([builddir])
with open(options.paved.cwd / 'MANIFEST.in', 'w') as fo:
for item in graft:
fo.write('graft %s\n' % item)
for item in options.paved.dist.manifest.include:
fo.write('include %s\n' % item)
for item in options.paved.dist.manifest.recursive_include:
fo.write('recursive-include %s\n' % item)
for item in prune:
fo.write('prune %s\n' % item)
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
from geonode.settings import INSTALLED_APPS
# only start if using Geoserver backend
if 'geonode.geoserver' not in INSTALLED_APPS:
return
download_dir = path('downloaded')
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path('geoserver')
geoserver_bin = download_dir / \
os.path.basename(dev_config['GEOSERVER_URL'])
jetty_runner = download_dir / \
os.path.basename(dev_config['JETTY_RUNNER_URL'])
grab(options.get('geoserver', dev_config['GEOSERVER_URL']), geoserver_bin,
"geoserver binary")
grab(options.get('jetty', dev_config['JETTY_RUNNER_URL']), jetty_runner,
"jetty runner")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir / 'geoserver'
if not webapp_dir:
webapp_dir.makedirs()
print 'extracting geoserver'
z = zipfile.ZipFile(geoserver_bin, "r")
z.extractall(webapp_dir)
_install_data_dir()
def start_qgis_server():
"""Start QGIS Server instance with GeoNode related plugins."""
# only start if using QGIS Serrver backend
if 'geonode.qgis_server' not in INSTALLED_APPS or OGC_SERVER['default'][
'BACKEND'] == 'geonode.geoserver':
return
info('Starting up QGIS Server...')
port = options.get('qgis_server_port', '9000')
sh('docker-compose -f docker-compose-qgis-server.yml up -d qgis-server',
env={
'GEONODE_PROJECT_PATH': os.getcwd(),
'QGIS_SERVER_PORT': port
})
info('QGIS Server is up.')
def stop_qgis_server():
"""
Stop QGIS Server Backend.
"""
from geonode.settings import INSTALLED_APPS
# only start if using QGIS Server backend
if 'geonode.qgis_server' not in INSTALLED_APPS:
return
port = options.get('qgis_server_port', '9000')
sh('docker-compose -f docker-compose-qgis-server.yml down',
env={
'GEONODE_PROJECT_PATH': os.getcwd(),
'QGIS_SERVER_PORT': port
})
def sync(options):
"""
Run the migrate and migrate management commands to create and migrate a DB
"""
# for app in dev_config['MIGRATE_APPS']:
# try:
# sh("python manage.py migrate {app} --noinput".format(app=app))
# except:
# pass
settings = options.get('settings', '')
if settings:
settings = 'DJANGO_SETTINGS_MODULE=%s' % settings
sh("%s python manage.py makemigrations --noinput" % settings)
sh("%s python manage.py migrate --noinput" % settings)
sh("%s python manage.py loaddata fixtures/sample_admin.json" % settings)
sh("%s python manage.py loaddata fixtures/default_oauth_apps.json" % settings)
sh("%s python manage.py loaddata fixtures/initial_data.json" % settings)
def test_bdd(options):
"""
Run GeoNode's BDD Test Suite
"""
local = str2bool(options.get('local', 'false'))
if local:
call_task('reset_hard')
else:
call_task('reset')
call_task('setup')
call_task('sync')
sh('sleep 30')
info("GeoNode is now available, running the bdd tests now.")
sh('py.test')
if local:
call_task('reset_hard')
def singletest(options):
"""
Run GeoNode's Unit Test Suite
"""
GEONODE_APPS = [
'geonode.maps.tests:MapsTest.test_map_remove',
'geonode.maps.tests:MapsTest.test_rating_map_remove',
'geonode.social.tests:SimpleTest.test_layer_activity',
'geonode.documents.tests:DocumentsTest.test_ajax_document_permissions',
'geonode.documents.tests:DocumentsTest.test_create_document_url',
'geonode.documents.tests:DocumentsTest.test_create_document_with_no_rel',
'geonode.documents.tests:DocumentsTest.test_create_document_with_rel',
]
apps_to_test = ' '.join(GEONODE_APPS)
prefix = options.get('prefix', 'python')
sh("%s manage.py test %s --noinput --failfast" % (prefix, apps_to_test))
def test(options):
"""
Run GeoNode's Unit Test Suite
"""
if on_travis:
if core_tests:
_apps = GEONODE_CORE_APPS
if internal_apps_tests:
_apps = GEONODE_INTERNAL_APPS
else:
_apps = GEONODE_APPS
_apps_to_test = []
for _app in _apps:
if _app and len(_app) > 0 and 'geonode' in _app:
_apps_to_test.append(_app)
sh("%s manage.py test geonode.tests.smoke %s.tests --noinput %s %s" %
(options.get('prefix'), '.tests '.join(_apps_to_test), _keepdb,
_parallel))
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
_reset()
# Start GeoServer
call_task('start_geoserver')
info("GeoNode is now available, running the tests now.")
name = options.get('name', 'geonode.tests.integration')
success = False
try:
if name == 'geonode.tests.csw':
call_task('start')
sh('sleep 30')
call_task('setup_data')
sh(('python manage.py test %s'
' --noinput --liveserver=localhost:8000' % name))
except BuildFailure, e:
info('Tests failed! %s' % str(e))
def test_bdd():
"""
Run GeoNode's BDD Test Suite
"""
call_task('stop_geoserver')
sh('sleep 30')
local = str2bool(options.get('local', 'false'))
if local:
call_task('reset_hard')
call_task('setup')
call_task('sync')
# Start GeoServer
call_task('start_geoserver')
sh('sleep 30')
info("GeoNode is now available, running the bdd tests now.")
sh('py.test')
if local:
call_task('stop_geoserver')
call_task('reset_hard')
def upload():
"""upload package zipfile to server"""
user = options.get('user', False)
if not user:
raise ValueError('OSGeo userid required')
password = getpass.getpass('Enter your password: '******'':
raise ValueError('password required')
call_task('package')
zipf = get_package_filename()
url = 'http://%s:%s@%s:%d/%s' % (user, password, options.upload.host,
options.upload.port,
options.upload.endpoint)
info('Uploading to http://%s/%s' %
(options.upload.host, options.upload.endpoint))
server = xmlrpc.client.ServerProxy(url, verbose=False)
try:
with open(zipf) as zfile:
plugin_id, version_id = \
server.plugin.upload(xmlrpc.client.Binary(zfile.read()))
info('Plugin ID: %s', plugin_id)
info('Version ID: %s', version_id)
except xmlrpc.client.Fault as err:
error('ERROR: fault error')
error('Fault code: %d', err.faultCode)
error('Fault string: %s', err.faultString)
except xmlrpc.client.ProtocolError as err:
error('Error: Protocol error')
error("%s : %s", err.errcode, err.errmsg)
if err.errcode == 403:
error('Invalid name and password')
def generate_csw_connections_file():
"""generate a CSW connections file from a flat file of CSW URLs"""
filename = options.get('filename', False)
if not filename:
raise ValueError('path to file of CSW URLs required')
conns = etree.Element('qgsCSWConnections')
conns.attrib['version'] = '1.0'
with open(filename) as connsfh:
for line in connsfh:
url = line.strip()
if not url: # blank line
continue
try:
csw = CatalogueServiceWeb(url)
title = unicode(csw.identification.title)
conn = etree.SubElement(conns, 'csw', name=title, url=url)
except Exception, err:
error('ERROR on CSW %s: %s', url, err)
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
fast = options.get('fast', False)
download_dir = path('downloaded')
download_dir.makedirs()
geoserver_dir = path('geoserver')
geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL)
geoserver_ext = download_dir / os.path.basename(GEONODE_GEOSERVER_EXT_URL)
if not geoserver_bin.exists():
print "Downloading geoserver binary distribution"
grab(GEOSERVER_URL, geoserver_bin)
elif not zipfile.is_zipfile(geoserver_bin):
print "Downloading geoserver binary distribution (corrupt file)"
grab(GEOSERVER_URL, geoserver_bin)
if not geoserver_ext.exists():
print "Downloading geonode geoserver extensions"
grab(GEONODE_GEOSERVER_EXT_URL, geoserver_ext)
elif not zipfile.is_zipfile(geoserver_ext):
print "Re-downloading geonode geoserver extensions (corrupt file)"
grab(GEONODE_GEOSERVER_EXT_URL, geoserver_ext)
if not geoserver_dir.exists():
with zipfile.ZipFile(geoserver_bin, "r") as z:
z.extractall(download_dir)
#FIXME(Ariel): Avoid hardcoding.
g = download_dir / "geoserver-2.2.3"
libs_dir = g / "webapps/geoserver/WEB-INF/lib"
with zipfile.ZipFile(geoserver_ext, "r") as z:
z.extractall(libs_dir)
# Move geoserver out of downloaded
geoserver_dir.remove()
shutil.move(g, geoserver_dir)
def deb(options):
"""
Creates debian packages.
Example uses:
paver deb
paver deb -k 12345
paver deb -k 12345 -p geonode/testing
"""
key = options.get('key', None)
ppa = options.get('ppa', None)
version, simple_version = versions()
info('Creating package for GeoNode version %s' % version)
# Get rid of any uncommitted changes to debian/changelog
info('Getting rid of any uncommitted changes in debian/changelog')
sh('git checkout package/debian/changelog')
# Workaround for git-dch bug
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580
sh('rm -rf %s/.git' % (os.path.realpath('package')))
sh('ln -s %s %s' % (os.path.realpath('.git'), os.path.realpath('package')))
with pushd('package'):
# Install requirements
# sh('sudo apt-get -y install debhelper devscripts git-buildpackage')
# sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s'
# ' --id-length=6 --ignore-branch --release' % (simple_version)))
# In case you publish from Ubuntu Xenial (git-dch is removed from upstream)
# use the following line instead:
# sh(('gbp dch --spawn-editor=snapshot --git-author --new-version=%s'
# ' --id-length=6 --ignore-branch --release' % (simple_version)))
distribution = "xenial"
sh(('gbp dch --distribution=%s --force-distribution --spawn-editor=snapshot --git-author --new-version=%s'
' --id-length=6 --ignore-branch --release' %
(distribution, simple_version)))
deb_changelog = path('debian') / 'changelog'
for idx, line in enumerate(
fileinput.input([deb_changelog], inplace=True)):
if idx == 0:
print "geonode (%s) %s; urgency=high" % (simple_version,
distribution),
else:
print line.replace("urgency=medium", "urgency=high"),
# Revert workaround for git-dhc bug
sh('rm -rf .git')
if key is None and ppa is None:
# A local installable package
sh('debuild -uc -us -A')
elif key is None and ppa is not None:
# A sources package, signed by daemon
sh('debuild -S')
elif key is not None and ppa is None:
# A signed installable package
sh('debuild -k%s -A' % key)
elif key is not None and ppa is not None:
# A signed, source package
sh('debuild -k%s -S' % key)
if ppa is not None:
sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
if 'geonode.qgis_server' not in INSTALLED_APPS or OGC_SERVER['default'][
'BACKEND'] == 'geonode.geoserver':
call_task('stop_geoserver')
_reset()
# Start GeoServer
call_task('start_geoserver')
else:
call_task('stop_qgis_server')
_reset()
# Start QGis Server
call_task('start_qgis_server')
sh('sleep 30')
name = options.get('name', 'geonode.tests.integration')
settings = options.get('settings', '')
if not settings and name == 'geonode.upload.tests.integration':
settings = 'geonode.upload.tests.test_settings'
success = False
try:
if name == 'geonode.tests.csw':
call_task('sync', options={'settings': settings})
call_task('start', options={'settings': settings})
sh('sleep 30')
call_task('setup_data', options={'settings': settings})
settings = 'DJANGO_SETTINGS_MODULE=%s' % settings if settings else ''
if name == 'geonode.upload.tests.integration':
sh("%s python -W ignore manage.py makemigrations --noinput" %
settings)
sh("%s python -W ignore manage.py migrate --noinput" % settings)
sh("%s python -W ignore manage.py loaddata sample_admin.json" %
settings)
sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/default_oauth_apps.json"
% settings)
sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/initial_data.json"
% settings)
call_task('start_geoserver')
bind = options.get('bind', '0.0.0.0:8000')
foreground = '' if options.get('foreground', False) else '&'
sh('%s python -W ignore manage.py runmessaging %s' %
(settings, foreground))
sh('%s python -W ignore manage.py runserver %s %s' %
(settings, bind, foreground))
sh('sleep 30')
settings = 'REUSE_DB=1 %s' % settings
sh(('%s python manage.py test %s'
' --noinput --liveserver=0.0.0.0:8000' % (settings, name)))
except BuildFailure as e:
info('Tests failed! %s' % str(e))
else:
success = True
finally:
# don't use call task here - it won't run since it already has
stop()
call_task('stop_geoserver')
_reset()
if not success:
sys.exit(1)
def start_geoserver(options):
"""
Start GeoServer with GeoNode extensions
"""
from geonode.settings import OGC_SERVER
GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION']
url = "http://localhost:8080/geoserver/"
if GEOSERVER_BASE_URL != url:
print 'your GEOSERVER_BASE_URL does not match %s' % url
sys.exit(1)
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
data_dir = path('geoserver/data').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/jetty-runner.xml').abspath()
# @todo - we should not have set workdir to the datadir but a bug
# in geoserver
# prevents geonode security from initializing correctly otherwise
with pushd(data_dir):
javapath = "java"
loggernullpath = os.devnull
# checking if our loggernullpath exists and if not, reset it to
# something manageable
if loggernullpath == "nul":
try:
open("../../downloaded/null.txt", 'w+').close()
except IOError as e:
print "Chances are that you have Geoserver currently running.\
Youcan either stop all servers with paver stop or start only\
the django application with paver start_django."
sys.exit(1)
loggernullpath = "../../downloaded/null.txt"
try:
sh(('java -version'))
except BaseException:
print "Java was not found in your path. \
Trying some other options: "
javapath_opt = None
if os.environ.get('JAVA_HOME', None):
print "Using the JAVA_HOME environment variable"
javapath_opt = os.path.join(
os.path.abspath(os.environ['JAVA_HOME']), "bin",
"java.exe")
elif options.get('java_path'):
javapath_opt = options.get('java_path')
else:
print "Paver cannot find java in the Windows Environment. \
Please provide the --java_path flag with your full path to \
java.exe e.g. --java_path=C:/path/to/java/bin/java.exe"
sys.exit(1)
# if there are spaces
javapath = 'START /B "" "' + javapath_opt + '"'
sh((
'%(javapath)s -Xmx512m -XX:MaxPermSize=256m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
# workaround for JAI sealed jar issue and jetty classloader
' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()))
info('Starting GeoServer on %s' % url)
# wait for GeoServer to start
started = waitfor(url)
info('The logs are available at %s' % log_file)
if not started:
# If applications did not start in time we will give the user a chance
# to inspect them and stop them manually.
info(('GeoServer never started properly or timed out.'
'It may still be running in the background.'))
sys.exit(1)
def start_django():
"""
Start the GeoNode Django application
"""
bind = options.get('bind', '')
sh('python manage.py runserver %s &' % bind)
def test(options):
"""
Run GeoNode's Unit Test Suite
"""
sh("%s manage.py test %s.tests --noinput" %
(options.get('prefix'), '.tests '.join(GEONODE_APPS)))
def start_geoserver(options):
"""
Start GeoServer with GeoNode extensions
"""
# only start if using Geoserver backend
if 'geonode.geoserver' not in INSTALLED_APPS or OGC_SERVER['default'][
'BACKEND'] == 'geonode.qgis_server':
return
GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION']
url = GEOSERVER_BASE_URL
if urlparse(GEOSERVER_BASE_URL).hostname != 'localhost':
print "Warning: OGC_SERVER['default']['LOCATION'] hostname is not equal to 'localhost'"
if not GEOSERVER_BASE_URL.endswith('/'):
print "Error: OGC_SERVER['default']['LOCATION'] does not end with a '/'"
sys.exit(1)
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / \
os.path.basename(dev_config['JETTY_RUNNER_URL'])
data_dir = path('geoserver/data').abspath()
geofence_dir = path('geoserver/data/geofence').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/misc/jetty-runner.xml').abspath()
jetty_port = urlparse(GEOSERVER_BASE_URL).port
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket_free = True
try:
s.bind(("127.0.0.1", jetty_port))
except socket.error as e:
socket_free = False
if e.errno == 98:
info('Port %s is already in use' % jetty_port)
else:
info(
'Something else raised the socket.error exception while checking port %s'
% jetty_port)
print(e)
finally:
s.close()
if socket_free:
# @todo - we should not have set workdir to the datadir but a bug in geoserver
# prevents geonode security from initializing correctly otherwise
with pushd(data_dir):
javapath = "java"
loggernullpath = os.devnull
# checking if our loggernullpath exists and if not, reset it to
# something manageable
if loggernullpath == "nul":
try:
open("../../downloaded/null.txt", 'w+').close()
except IOError as e:
print "Chances are that you have Geoserver currently running. You \
can either stop all servers with paver stop or start only \
the django application with paver start_django."
sys.exit(1)
loggernullpath = "../../downloaded/null.txt"
try:
sh(('java -version'))
except BaseException:
print "Java was not found in your path. Trying some other options: "
javapath_opt = None
if os.environ.get('JAVA_HOME', None):
print "Using the JAVA_HOME environment variable"
javapath_opt = os.path.join(
os.path.abspath(os.environ['JAVA_HOME']), "bin",
"java.exe")
elif options.get('java_path'):
javapath_opt = options.get('java_path')
else:
print "Paver cannot find java in the Windows Environment. \
Please provide the --java_path flag with your full path to \
java.exe e.g. --java_path=C:/path/to/java/bin/java.exe"
sys.exit(1)
# if there are spaces
javapath = 'START /B "" "' + javapath_opt + '"'
sh((
'%(javapath)s -Xms512m -Xmx2048m -server -XX:+UseConcMarkSweepGC -XX:MaxPermSize=512m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
' -Dgeofence.dir=%(geofence_dir)s'
# ' -Dgeofence-ovr=geofence-datasource-ovr.properties'
# workaround for JAI sealed jar issue and jetty classloader
# ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --port %(jetty_port)i'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()))
info('Starting GeoServer on %s' % url)
# wait for GeoServer to start
started = waitfor(url)
info('The logs are available at %s' % log_file)
if not started:
# If applications did not start in time we will give the user a chance
# to inspect them and stop them manually.
info(('GeoServer never started properly or timed out.'
'It may still be running in the background.'))
sys.exit(1)
def start_geoserver(options):
"""
Start GeoServer with GeoNode extensions
"""
from geonode.settings import OGC_SERVER
GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION']
url = GEOSERVER_BASE_URL
if urlparse(GEOSERVER_BASE_URL).hostname != 'localhost':
print "Warning: OGC_SERVER['default']['LOCATION'] hostname is not equal to 'localhost'"
if not GEOSERVER_BASE_URL.endswith('/'):
print "Error: OGC_SERVER['default']['LOCATION'] does not end with a '/'"
sys.exit(1)
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / os.path.basename(
dev_config['JETTY_RUNNER_URL'])
data_dir = path('geoserver/data').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/misc/jetty-runner.xml').abspath()
jetty_port = urlparse(GEOSERVER_BASE_URL).port
# @todo - we should not have set workdir to the datadir but a bug in geoserver
# prevents geonode security from initializing correctly otherwise
with pushd(data_dir):
javapath = "java"
loggernullpath = os.devnull
# checking if our loggernullpath exists and if not, reset it to something manageable
if loggernullpath == "nul":
try:
open("../../downloaded/null.txt", 'w+').close()
except IOError, e:
print "Chances are that you have Geoserver currently running. You \
can either stop all servers with paver stop or start only \
the django application with paver start_django."
sys.exit(1)
loggernullpath = "../../downloaded/null.txt"
try:
sh(('java -version'))
except:
print "Java was not found in your path. Trying some other options: "
javapath_opt = None
if os.environ.get('JAVA_HOME', None):
print "Using the JAVA_HOME environment variable"
javapath_opt = os.path.join(
os.path.abspath(os.environ['JAVA_HOME']), "bin",
"java.exe")
elif options.get('java_path'):
javapath_opt = options.get('java_path')
else:
print "Paver cannot find java in the Windows Environment. \
Please provide the --java_path flag with your full path to \
java.exe e.g. --java_path=C:/path/to/java/bin/java.exe"
sys.exit(1)
# if there are spaces
javapath = 'START /B "" "' + javapath_opt + '"'
sh((
'%(javapath)s -Xmx512m -XX:MaxPermSize=256m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
# workaround for JAI sealed jar issue and jetty classloader
' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --port %(jetty_port)i'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()))
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
_backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND'])
if _backend == 'geonode.geoserver' or 'geonode.qgis_server' not in INSTALLED_APPS:
call_task('stop_geoserver')
_reset()
else:
call_task('stop_qgis_server')
_reset()
name = options.get('name', 'geonode.tests.integration')
settings = options.get('settings', '')
if name == 'geonode.geoserver.integration.tests':
if _django_11:
sh("cp geonode/upload/tests/test_settings.py geonode/")
settings = 'geonode.test_settings'
else:
settings = 'geonode.upload.tests.test_settings'
success = False
try:
call_task('setup', options={'settings': settings})
if name == 'geonode.tests.csw':
call_task('sync', options={'settings': settings})
call_task('start', options={'settings': settings})
call_task('setup_data', options={'settings': settings})
if name == 'geonode.geoserver.integration.tests':
sh("DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py makemigrations --noinput"
% settings)
sh("DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py migrate --noinput"
% settings)
sh("DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py loaddata sample_admin.json"
% settings)
sh("DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py loaddata geonode/base/fixtures/default_oauth_apps.json"
% settings)
sh("DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py loaddata geonode/base/fixtures/initial_data.json"
% settings)
call_task('start_geoserver')
bind = options.get('bind', '0.0.0.0:8000')
foreground = '' if options.get('foreground', False) else '&'
sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runmessaging %s'
% (settings, foreground))
sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runserver %s %s'
% (settings, bind, foreground))
sh('sleep 30')
settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=%s' % settings
live_server_option = '--liveserver=localhost:8000'
if _django_11:
live_server_option = ''
info("Running the tests now...")
if name == 'geonode.geoserver.integration.tests':
sh(('%s python -W ignore manage.py test %s'
' %s --noinput %s' %
(settings, 'geonode.upload.tests.integration', _keepdb,
live_server_option)))
sh(('%s python -W ignore manage.py test %s'
' %s --noinput %s' %
(settings, 'geonode.monitoring.tests.integration', _keepdb,
live_server_option)))
else:
sh(('%s python -W ignore manage.py test %s'
' %s --noinput %s' %
(settings, name, _keepdb, live_server_option)))
except BuildFailure as e:
info('Tests failed! %s' % str(e))
else:
success = True
finally:
# don't use call task here - it won't run since it already has
stop(options)
_reset()
if not success:
sys.exit(1)
def start_messaging():
"""
Start the GeoNode messaging server
"""
foreground = '' if options.get('foreground', False) else '&'
sh('python manage.py runmessaging %s' % foreground)
def deb(options):
"""
Creates debian packages.
Example uses:
paver deb
paver deb -k 12345
paver deb -k 12345 -p geonode/testing
"""
key = options.get('key', None)
ppa = options.get('ppa', None)
import geonode
from geonode.version import get_git_changeset
raw_version = geonode.__version__
version = geonode.get_version()
timestamp = get_git_changeset()
major, minor, revision, stage, edition = raw_version
branch = 'dev'
if stage == 'alpha' and edition == 0:
tail = '%s%s' % (branch, timestamp)
else:
tail = '%s%s' % (stage, edition)
simple_version = '%s.%s.%s+%s' % (major, minor, revision, tail)
info('Creating package for GeoNode version %s' % version)
with pushd('package'):
# Get rid of any uncommitted changes to debian/changelog
info('Getting rid of any uncommitted changes in debian/changelog')
sh('git checkout debian/changelog')
# Workaround for git-dch bug
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580
path('.git').makedirs()
# Install requirements
#sh('sudo apt-get -y install debhelper devscripts git-buildpackage')
sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s'
' --id-length=6 --ignore-branch --release' % (
simple_version)))
## Revert workaround for git-dhc bug
path('.git').rmtree()
if key is None and ppa is None:
# A local installable package
sh('debuild -uc -us -A')
elif key is None and ppa is not None:
# A sources package, signed by daemon
sh('debuild -S')
elif key is not None and ppa is None:
# A signed installable package
sh('debuild -k%s -A' % key)
elif key is not None and ppa is not None:
# A signed, source package
sh('debuild -k%s -S' % key)
if ppa is not None:
sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))
