def localpi():
"""Make local package index (used by tox)."""
download_dir = path(options.develop.download_dir)
if not download_dir.exists():
call_task("download_depends")
info("MAKE LOCAL PACKAGE-INDEX: %s/" % download_dir)
sh("dir2pi %s" % download_dir)
def setup(options):
"""Get dependencies and prepare a GeoNode development environment."""
sh('pip install -e .')
info(('GeoNode development environment successfully set up.'
'If you have not set up an administrative account,'
' please do so now. Use "paver start" to start up the server.'))
def package_tar_gz(options):
"""Package a .tar.gz distribution"""
import tarfile
package_name = options.get('package_name', None)
if package_name is None:
raise Exception('Package name required')
filename = path('%s/%s.tar.gz' % (STAGE_DIR, package_name))
if filename.exists():
info('Package %s already exists' % filename)
return
with pushd(STAGE_DIR):
stage_path = '%s/%s' % (STAGE_DIR, package_name)
if not path(stage_path).exists():
raise Exception('Directory %s does not exist' % stage_path)
tar = tarfile.open(filename, 'w:gz')
tar.add(package_name)
tar.close()
def start():
"""
Start GeoNode (Django, GeoServer & Client)
"""
call_task('start_messaging')
info("GeoNode is now available.")
def tarball_unpack(fpath, dest, overwrite=False):
"""
Dumbly unpack tarballs and zips
fpath --
filepath of tarball
dest --
folder to upack into
@return
name of folder created by unpacking
"""
dest = path(dest)
filename = fpath.split("/")[-1]
newfile = dest / filename
old = set(os.listdir(dest))
dest_folder = dest / filename.split(".tar.")[0]
if not dest_folder.exists() and overwrite:
shutil.copyfile(str(fpath), str(newfile))
with pushd(dest):
catcmd = "zcat"
if filename.endswith("bz2") or filename.endswith("bz"):
catcmd = "bzcat"
cat = subprocess.Popen([catcmd, filename], stdout=subprocess.PIPE)
untar = subprocess.Popen(["tar", "-xf", "-"], stdin=cat.stdout, stdout=subprocess.PIPE)
info("Unpacking %s" %filename)
untar.communicate()
os.remove(newfile)
return dest_folder
def stop():
"""
Stop GeoNode
"""
info("Stopping GeoNode ...")
stop_django()
stop_geoserver()
def stop_geoserver():
"""
Stop GeoServer
"""
# we use docker-compose for integration tests
if integration_tests:
return
# only start if using Geoserver backend
if 'geonode.geoserver' not in INSTALLED_APPS or OGC_SERVER['default']['BACKEND'] == 'geonode.qgis_server':
return
kill('java', 'geoserver')
# Kill process.
try:
# proc = subprocess.Popen("ps -ef | grep -i -e '[j]ava\|geoserver' |
# awk '{print $2}'",
proc = subprocess.Popen(
"ps -ef | grep -i -e 'geoserver' | awk '{print $2}'",
shell=True,
stdout=subprocess.PIPE)
for pid in proc.stdout:
info('Stopping geoserver (process number %s)' % int(pid))
os.kill(int(pid), signal.SIGKILL)
os.kill(int(pid), 9)
sh('sleep 30')
# Check if the process that we killed is alive.
try:
os.kill(int(pid), 0)
# raise Exception("""wasn't able to kill the process\nHINT:use
# signal.SIGKILL or signal.SIGABORT""")
except OSError as ex:
continue
except Exception as e:
info(e)
def get_yuicomp():
current_dir = path.getcwd()
lib_dir = current_dir / "lib"
if not lib_dir.exists():
lib_dir.mkdir()
yui_dir = lib_dir / ("yuicompressor-%s" % options.compressor_version)
jst_conf = current_dir / DEFAULT_CFG
if getattr(options, "overwrite", False) and yui_dir.exists():
sh("rm -rf %s" %yui_dir)
if not yui_dir.exists():
zip_file = lib_dir / options.zip_name
lib_dir.chdir()
if not zip_file.exists():
info("Downloading %s", options.url)
sh("curl -O %s" % options.url)
sh("unzip %s" % options.zip_name)
set_yui_version(jst_conf, options.compressor_version, yui_dir)
current_dir.chdir()
return True
info("yui compressor already downloaded")
if getattr(options, 'set_as_default', None):
set_yui_version(jst_conf, options.compressor_version, yui_dir)
def develop(options):
"""Set up each of the profile service parts into the python environment.
"""
global BASKET
# Figure out and set up the runtime information:
#
if options.develop.basket:
# Force '-i'
BASKET = "-i %s" % options.develop.basket.replace("-i", "")
# Use the python process or virtualenv python we maybe running under:
python = sys.executable
easy.info("Using python: <%s>" % python)
for dev_pkg in options.DEV_PKGS_IN_DEP_ORDER:
dev_pkg.chdir()
easy.info("-- Changing to %s --" % dev_pkg)
# Then switch in pkg dir to perform the set up:
easy.info("-- Setting up %s in development mode --" % dev_pkg)
# Set up the the package in development mode:
easy.info("Setting up '{0}': (BASKET:'{1}') ".format(
dev_pkg,
BASKET
))
stdout = easy.sh("{python} setup.py develop {BASKET} ".format(
python=python,
BASKET=BASKET
),
capture=True,
)
easy.info(stdout)
def add_pg_user():
if sys.platform == 'darwin':
info("osx user add not implemented yet")
# http://osxdaily.com/2007/10/29/how-to-add-a-user-from-the-os-x-command-line-works-with-leopard/
## Create a new entry in the local (/) domain under the category /users.
## dscl / -create /Users/toddharris
## Create and set the shell property to bash.
## dscl / -create /Users/toddharris UserShell /bin/bash
## Create and set the user's full name.
## dscl / -create /Users/toddharris RealName "Dr. Todd Harris"
## Create and set the users ID.
## dscl / -create /Users/toddharris UniqueID 503
## Create and set the users group ID property.
## dscl / -create /Users/toddharris PrimaryGroupID 1000
## Create and set the user home directory.
## dscl / -create /Users/toddharris NFSHomeDirectory /Local/Users/toddharris
## Set the password.
## dscl / -passwd /Users/toddharris PASSWORD
else:
info("*nix user add not implemented yet")
return pwd.getpwnam('postgres')[2]
def gh_register():
"""Create a repository at GitHub and push it your local repository."""
_adjust_options()
repo = _get_repo(os.getcwd())
project_name = options.setup.name
project_description = options.setup.get('description','')
remote_name = options.gh_pages.remote_name
master_branch = options.gh_pages.master_branch
credentials = Credentials.get_credentials(repo)
if not credentials:
sys.exit('Your github name and token git config are not set.'
'Check http://github.com/blog/170-token-authentication' % os.getcwd())
project = dry(
"Create a repository called %s at %s's GitHub account..." % (
project_name, credentials.user),
repo.register,
project_name,
credentials,
description=project_description,
is_public=True,
remote_name=remote_name,
master_branch=master_branch)
if project is not None:
info('Opening your project home pages:%s', project.url.http)
webbrowser.open(project.url.http)
def deb(options):
"""
Creates debian packages.
Example uses:
paver deb
paver deb -k 12345
paver deb -k 12345 -p geonode/testing
"""
key = options.get("key", None)
ppa = options.get("ppa", None)
version, simple_version = versions()
info("Creating package for GeoNode version %s" % version)
with pushd("package"):
# Get rid of any uncommitted changes to debian/changelog
info("Getting rid of any uncommitted changes in debian/changelog")
sh("git checkout debian/changelog")
# Workaround for git-dch bug
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580
# path('.git').makedirs()
# Link the parent git repository folder in to the current one
# needed by git-dch
sh("ln -s ../.git .git")
# Install requirements
# sh('sudo apt-get -y install debhelper devscripts git-buildpackage')
sh(
(
"git-dch --spawn-editor=snapshot --git-author --new-version=%s"
" --id-length=6 --ignore-branch --release" % (simple_version)
)
)
deb_changelog = path("debian") / "changelog"
for line in fileinput.input([deb_changelog], inplace=True):
print line.replace("urgency=low", "urgency=high"),
## Revert workaround for git-dhc bug
sh("rm -rf .git")
if key is None and ppa is None:
# A local installable package
sh("debuild -uc -us -A")
elif key is None and ppa is not None:
# A sources package, signed by daemon
sh("debuild -S")
elif key is not None and ppa is None:
# A signed installable package
sh("debuild -k%s -A" % key)
elif key is not None and ppa is not None:
# A signed, source package
sh("debuild -k%s -S" % key)
if ppa is not None:
sh("dput ppa:%s geonode_%s_source.changes" % (ppa, simple_version))
def package_tar_gz(options):
"""Package a .tar.gz distribution"""
import tarfile
package_name = options.get("package_name", None)
if package_name is None:
raise Exception("Package name required")
filename = path("%s/%s.tar.gz" % (STAGE_DIR, package_name))
if filename.exists():
info("Package %s already exists" % filename)
return
with pushd(STAGE_DIR):
stage_path = "%s/%s" % (STAGE_DIR, package_name)
if not path(stage_path).exists():
raise Exception("Directory %s does not exist" % stage_path)
tar = tarfile.open(filename, "w:gz")
tar.add(package_name)
tar.close()
def gh_pages_clean():
"""Clean your documentation.
Update the submodule (every changes not committed and pushed will be lost),
pull any changes and remove any file in options.gh_pages.docroot.
"""
_adjust_options()
remote_name = options.gh_pages.remote_name
repo = _get_repo(os.getcwd())
module = repo.submodules.get(options.gh_pages.root, None)
if module is None:
info("You have not yet created the gh-pages submodule.")
return
module.update()
try:
dry('Checkout the gh-pages branch', module.git.checkout, 'gh-pages')
except GitCommandError:
dry('Checkout the gh-pages remote branch',
module.git.checkout, '-t', '%s/gh-pages' % remote_name)
else:
dry('Fetch any changes on the gh-pages remote branch',
module.git.pull, remote_name, 'gh-pages')
for dir_entry in options.gh_pages.htmlroot.listdir():
if dir_entry.isdir():
if dir_entry.basename() != '.git':
dry("Remove %s" % dir_entry, dir_entry.rmtree)
else:
dry('Remove %s' % dir_entry, dir_entry.unlink)
def after_bootstrap():
info("under construction")
call_task("dir_layout")
call_task("install_c_base")
call_task("install_postgis")
call_task("write_supervisor_config")
call_task("save_cfg")
def setup_testdata():
"""Create test databases and load test data"""
test_database_parameters = {
# suite: has_testdata
'apiso': True,
'cite': True,
'harvesting': False,
'manager': False
}
# remove CITE database so we can build fresh
for suite in test_database_parameters.keys():
dbfile = 'tests/suites/%s/data/records.db' % suite
if os.path.isfile(dbfile):
os.remove(dbfile)
for database, has_testdata in test_database_parameters.iteritems():
info('Setting up test database %s' % database)
cfg = path('tests/suites/%s/default.cfg' % database)
sh('pycsw-admin.py -c setup_db -f %s' % cfg)
if has_testdata:
datapath = path('tests/suites/%s/data' % database)
info('Loading test data from %s' % datapath)
sh('pycsw-admin.py -c load_records -f %s -p %s' % (cfg, datapath))
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
_reset()
# Start GeoServer
call_task('start_geoserver')
info("GeoNode is now available, running the tests now.")
name = options.get('name', 'geonode.tests.integration')
success = False
try:
if name == 'geonode.tests.csw':
call_task('sync')
call_task('start')
sh('sleep 30')
call_task('setup_data')
sh(('python manage.py test %s'
' --noinput --liveserver=0.0.0.0:8000' % name))
except BuildFailure as e:
info('Tests failed! %s' % str(e))
else:
success = True
finally:
# don't use call task here - it won't run since it already has
stop()
_reset()
if not success:
sys.exit(1)
def bundle():
"""Bundle required Python packages as ZIP archive."""
packages = options.bundle.packages
builddir = path("build")/"bundle"
builddir.makedirs()
errors = 0
for package in packages:
try:
info("bundle %s ..." % package)
module = __import__(package)
pkgfile = path(module.__file__)
basedir = pkgfile.dirname()
destdir = builddir/package
if destdir.exists():
destdir.rmtree()
basedir.copytree(destdir)
except Exception as e:
info("FAILED %s: %s" % (package, e))
errors += 1
archive = options.bundle.get("archive", "python-bundle.zip")
stored_files = make_zip_archive(archive, builddir)
# -- SUMMARY:
message1 = "bundle {pkgno} packages into {archive} "
message1 += "(with {errorno} errors, files: {fileno})"
message = message1.format(archive=archive, pkgno=len(packages),
errorno=errors, fileno=stored_files)
output = info
if errors:
message = "FAILED: {message}".format(message=message)
output = error
output(message)
def build(self):
"""
Invoke the r.js command. It checks the output of the r.js process to
write the list of files in ``cache_file``.
If the r.js process returns with anything else than 0, it raises a
``RuntimeError``.
"""
info('Writing %s', self.out)
command = self.get_command(
baseUrl=self.source,
mainConfigFile=self.config_js,
out=self.out,
**self.params
)
debug('Running: %s', ' '.join(command))
rjs_process = subprocess.Popen(
command,
stdout=subprocess.PIPE,
)
try:
files = self.parse_output(rjs_process.stdout)
finally:
rc = rjs_process.wait()
if rc != 0:
raise RuntimeError('r.js returned with {}'.format(rc))
# The config file is not added by requirejs as the list of files
files.append(self.config_js)
self.cache.write(files)
def package(options):
"""
Creates a tarball to use for building the system elsewhere
"""
import tarfile
import geonode
version = geonode.get_version()
# Use GeoNode's version for the package name.
pkgname = 'GeoNode-%s-all' % version
# Create the output directory.
out_pkg = path(pkgname)
out_pkg_tar = path("%s.tar.gz" % pkgname)
# Create a distribution in zip format for the geonode python package.
dist_dir = path('dist')
dist_dir.rmtree()
sh('python setup.py sdist --formats=zip')
with pushd('package'):
# Delete old tar files in that directory
for f in glob.glob('GeoNode*.tar.gz'):
old_package = path(f)
if old_package != out_pkg_tar:
old_package.remove()
if out_pkg_tar.exists():
info('There is already a package for version %s' % version)
return
# Clean anything that is in the oupout package tree.
out_pkg.rmtree()
out_pkg.makedirs()
support_folder = path('support')
install_file = path('install.sh')
# And copy the default files from the package folder.
justcopy(support_folder, out_pkg / 'support')
justcopy(install_file, out_pkg)
geonode_dist = path('..') / 'dist' / 'GeoNode-%s.zip' % version
justcopy(geonode_dist, out_pkg)
# Create a tar file with all files in the output package folder.
tar = tarfile.open(out_pkg_tar, "w:gz")
for file in out_pkg.walkfiles():
tar.add(file)
# Add the README with the license and important links to documentation.
tar.add('README', arcname=('%s/README.rst' % out_pkg))
tar.close()
# Remove all the files in the temporary output package directory.
out_pkg.rmtree()
# Report the info about the new package.
info("%s created" % out_pkg_tar.abspath())
def ensure_path_with_pmethods(path_class):
"""
Adds "..._p()" methods to path class.
Earlier versions did not have those.
EXAMPLE:
# -- file:pavement.py
from paver.easy import *
sys.path.insert(0, ".")
from paver_ext import paver_patch
paver_patch.ensure_path_with_pmethods(path)
:since: paver.path >= 1.2
"""
for pmethod_name in _PATH_PMETHOD_NAMES:
assert pmethod_name.endswith("_p")
method_name = pmethod_name[:-2]
pfunc = getattr(path_class, pmethod_name, None)
if not pfunc:
# -- PATCH PATH-CLASS: Set path_class.pmethod = method
# NOTE: Old method had sanity check that is now provided by pmethod.
info("PATCH: path.%s = %s" % (pmethod_name, method_name))
func = getattr(path_class, method_name)
setattr(path_class, pmethod_name, func)
def minify():
'''
minify javascript source with the jsmin module
'''
if jsmin is None:
easy.info('Jsmin not installed! cannot minify code')
return None
options.assets.js_files = map(lambda x: ((x[0],jsmin(x[1]))),options.assets.js_files)
def coffee():
'''
compile coffeescript files into javascript
'''
if compile_coffee is None:
easy.info('coffee-script not installed! cannot compile coffescript')
return None
options.assets.js_files = map(lambda x: ((x[0],compile_coffee(x[1],True))),options.assets.js_files)
def stop():
"""
Stop GeoNode
"""
# windows needs to stop the geoserver first b/c we can't tell which python
# is running, so we kill everything
info("Stopping GeoNode ...")
stop_django()
def ng_annotate():
'''
run code through ng-annotate compiler
'''
if ng is None:
easy.info('ng-annotate and pyng-annotate required for this task')
return None
options.assets.js_files = map(lambda x: ((x[0],ng(x[1]))),options.assets.js_files)
def supervisordconf():
ctx = DeployContext()
includes = []
daemons_conf_dir = ROOT.joinpath('etc/supervisord/daemons/')
daemons_conf_dir.makedirs_p()
for daemon in daemons:
daemon_conf = daemons_conf_dir.joinpath(daemon.name + '.conf')
includes.append(daemon_conf)
info('Writing daemon %s', daemon)
write_config(daemon_conf, {
'program:{}'.format(daemon.name): {
'command': daemon.command,
'environment': ','.join('{}="{}"'.format(key, val) for key, val in daemon.environ.items())
},
})
groups_conf_dir = ROOT.joinpath('etc/supervisord/groups/')
groups_conf_dir.makedirs_p()
for group in daemons.groups():
group_conf = groups_conf_dir.joinpath(group.name + '.conf')
includes.append(group_conf)
info('Writing group %s', group)
write_config(group_conf, {
'group:{}'.format(group.name): {
'programs': ','.join(d.name for d in group),
}
})
supervisorctl_conf = ROOT.joinpath('etc/supervisord/supervisorctl.conf')
write_config(supervisorctl_conf, {
'supervisorctl': {
'serverurl': 'unix://' + SOCKET,
},
})
supervisord_conf = ROOT.joinpath(defaults.SUPERVISORDCONF)
write_config(supervisord_conf, {
'rpcinterface:supervisor': {
'supervisor.rpcinterface_factory': 'supervisor.rpcinterface:make_main_rpcinterface',
},
'unix_http_server': {
'file': SOCKET,
'chown': '{UID}:{GID}'.format(**ctx),
},
'supervisord': {
'logfile': LOGS.joinpath('supervisord.log'),
'pidfile': PIDFILE,
},
'include': {
'files': ' '.join(includes),
}
})
def deb(options):
"""
Creates debian packages.
Example uses:
paver deb
paver deb -k 12345
paver deb -k 12345 -p geonode/testing
"""
key = options.get('key', None)
ppa = options.get('ppa', None)
version, simple_version = versions()
info('Creating package for GeoNode version %s' % version)
# Get rid of any uncommitted changes to debian/changelog
info('Getting rid of any uncommitted changes in debian/changelog')
sh('git checkout package/debian/changelog')
# Workaround for git-dch bug
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580
sh('ln -s %s %s' % (os.path.realpath('.git'), os.path.realpath('package')))
with pushd('package'):
# Install requirements
# sh('sudo apt-get -y install debhelper devscripts git-buildpackage')
sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s'
' --id-length=6 --ignore-branch --release' % (simple_version)))
#In case you publish from Ubuntu Xenial (git-dch is removed from upstream)
# use the following line instead:
#sh(('gbp dch --spawn-editor=snapshot --git-author --new-version=%s'
# ' --id-length=6 --ignore-branch --release' % (simple_version)))
deb_changelog = path('debian') / 'changelog'
for line in fileinput.input([deb_changelog], inplace=True):
print line.replace("urgency=medium", "urgency=high"),
# Revert workaround for git-dhc bug
sh('rm -rf .git')
if key is None and ppa is None:
# A local installable package
sh('debuild -uc -us -A')
elif key is None and ppa is not None:
# A sources package, signed by daemon
sh('debuild -S')
elif key is not None and ppa is None:
# A signed installable package
sh('debuild -k%s -A' % key)
elif key is not None and ppa is not None:
# A signed, source package
sh('debuild -k%s -S' % key)
if ppa is not None:
sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))
def localpi():
"""Make local python package index from download_dir contents."""
download_dir = path(options.pip.download_dir)
if not download_dir.exists():
call_task("download_depends")
require_script(MAKE_LOCALPI_SCRIPT)
info("MAKE LOCAL PACKAGE-INDEX: %s/" % download_dir)
sh("%s %s" % (MAKE_LOCALPI_SCRIPT, download_dir))
def watch(self):
"""Watches the sass source directories and launches compilation as needed"""
debug('Building/watching with %s', self._sass)
try:
watcher = Watcher(self._sass)
info('Starting watch %s', self._sass)
watcher()
except KeyboardInterrupt:
pass
def _run(self, command):
if self.env:
env = dict(os.environ)
env.update(self.env)
else:
env = os.environ
info('Running %s', ' '.join(command))
return subprocess.Popen(command, env=env)
def start():
"""
Start GeoNode (Django, GeoServer & Client)
"""
call_task('start_messaging')
info("GeoNode is now available.")
def install_yarn_components():
info("Installing Yarn/Bower components...")
sh('yarn --ignore-engines --ignore-scripts --prod')
sh('node -e "try { require(\'fs\').symlinkSync(require(\'path\').resolve(\'node_modules/@bower_components\'), '
'\'g3w-admin/core/static/bower_components\', \'junction\') } catch (e) { }"')
info("Yarn/Bower components installed.")
def start_geoserver(options):
"""
Start GeoServer with GeoNode extensions
"""
from geonode.settings import OGC_SERVER
GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION']
url = "http://localhost:8080/geoserver/"
if GEOSERVER_BASE_URL != url:
print 'your GEOSERVER_BASE_URL does not match %s' % url
sys.exit(1)
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
data_dir = path('geoserver/data').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/jetty-runner.xml').abspath()
# @todo - we should not have set workdir to the datadir but a bug
# in geoserver
# prevents geonode security from initializing correctly otherwise
with pushd(data_dir):
javapath = "java"
loggernullpath = os.devnull
# checking if our loggernullpath exists and if not, reset it to
# something manageable
if loggernullpath == "nul":
try:
open("../../downloaded/null.txt", 'w+').close()
except IOError as e:
print "Chances are that you have Geoserver currently running.\
Youcan either stop all servers with paver stop or start only\
the django application with paver start_django."
sys.exit(1)
loggernullpath = "../../downloaded/null.txt"
try:
sh(('java -version'))
except BaseException:
print "Java was not found in your path. \
Trying some other options: "
javapath_opt = None
if os.environ.get('JAVA_HOME', None):
print "Using the JAVA_HOME environment variable"
javapath_opt = os.path.join(
os.path.abspath(os.environ['JAVA_HOME']), "bin",
"java.exe")
elif options.get('java_path'):
javapath_opt = options.get('java_path')
else:
print "Paver cannot find java in the Windows Environment. \
Please provide the --java_path flag with your full path to \
java.exe e.g. --java_path=C:/path/to/java/bin/java.exe"
sys.exit(1)
# if there are spaces
javapath = 'START /B "" "' + javapath_opt + '"'
sh((
'%(javapath)s -Xmx512m -XX:MaxPermSize=256m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
# workaround for JAI sealed jar issue and jetty classloader
' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()))
info('Starting GeoServer on %s' % url)
# wait for GeoServer to start
started = waitfor(url)
info('The logs are available at %s' % log_file)
if not started:
# If applications did not start in time we will give the user a chance
# to inspect them and stop them manually.
info(('GeoServer never started properly or timed out.'
'It may still be running in the background.'))
sys.exit(1)
def create_secret_key():
"""create secret key for SECRET_KEY in instance/config_site.py"""
info('Secret key: \'%s\'' % os.urandom(24).encode('hex'))
info('Copy/paste this key to set the SECRET_KEY')
info('value in instance/config_site.py')
def start():
"""
Start GeoNode (Django, GeoServer & Client)
"""
info("GeoNode is now available.")
def start(options):
"""
Start GeoNode (Django, GeoServer & Client)
"""
sh('sleep 30')
info("GeoNode is now available.")
def upgrade():
"""upgrade database if changed; be sure to backup first!"""
info('Upgrading database...')
with pushd(path('%s/GeoHealthCheck' % BASEDIR)):
sh('python manage.py db upgrade')
def install():
"""
Install G3W-SUITE
"""
info('G3W-SUITE installed with success')
def requirements():
info("Installing Python modules...")
sh('pip install -r requirements.txt')
info("Python modules installed.")
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
prefix = options.get('prefix')
local = str2bool(options.get('local', 'false'))
if local:
call_task('stop_geoserver')
_reset()
name = options.get('name', None)
settings = options.get('settings', '')
success = False
try:
call_task('setup', options={'settings': settings, 'force_exec': True})
if not settings:
settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=geonode.settings'
if name and name in ('geonode.tests.csw', 'geonode.tests.integration',
'geonode.geoserver.tests.integration'):
call_task('sync', options={'settings': settings})
if local:
call_task('start_geoserver',
options={
'settings': settings,
'force_exec': True
})
call_task('start', options={'settings': settings})
if integration_server_tests:
call_task('setup_data', options={'settings': settings})
elif 'geonode.geoserver' in INSTALLED_APPS:
if local:
sh("cp geonode/upload/tests/test_settings.py geonode/")
settings = 'geonode.test_settings'
sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py "
"makemigrations --noinput")
sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py "
"migrate --noinput")
sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py "
"loaddata sample_admin.json")
sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py "
"loaddata geonode/base/fixtures/default_oauth_apps.json")
sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py "
"loaddata geonode/base/fixtures/initial_data.json")
call_task('start_geoserver')
bind = options.get('bind', '0.0.0.0:8000')
foreground = '' if options.get('foreground', False) else '&'
sh(f'DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py runmessaging {foreground}'
)
sh(f'DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py runserver {bind} {foreground}'
)
sh('sleep 30')
settings = f'REUSE_DB=1 DJANGO_SETTINGS_MODULE={settings}'
else:
call_task('sync', options={'settings': settings})
live_server_option = ''
info("Running the tests now...")
sh(f'{settings} {prefix} manage.py test {name} -v 3 {_keepdb} --noinput {live_server_option}'
)
except BuildFailure as e:
info(f'Tests failed! {str(e)}')
else:
success = True
finally:
if local:
stop(options)
_reset()
if not success:
sys.exit(1)
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
prefix = options.get('prefix')
_backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND'])
if _backend == 'geonode.geoserver' or 'geonode.qgis_server' not in INSTALLED_APPS:
call_task('stop_geoserver')
_reset()
else:
call_task('stop_qgis_server')
_reset()
name = options.get('name', None)
settings = options.get('settings', '')
success = False
try:
call_task('setup', options={'settings': settings, 'force_exec': True})
if name and name in ('geonode.tests.csw', 'geonode.tests.integration',
'geonode.geoserver.tests.integration'):
if not settings:
settings = 'geonode.local_settings' if _backend == 'geonode.qgis_server' else 'geonode.settings'
settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=%s' % settings
call_task('sync', options={'settings': settings})
if _backend == 'geonode.geoserver':
call_task('start_geoserver',
options={
'settings': settings,
'force_exec': True
})
call_task('start', options={'settings': settings})
call_task('setup_data', options={'settings': settings})
elif not integration_csw_tests and _backend == 'geonode.geoserver' and 'geonode.geoserver' in INSTALLED_APPS:
sh("cp geonode/upload/tests/test_settings.py geonode/")
settings = 'geonode.test_settings'
sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py "
"makemigrations --noinput".format(settings))
sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py "
"migrate --noinput".format(settings))
sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py "
"loaddata sample_admin.json".format(settings))
sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py "
"loaddata geonode/base/fixtures/default_oauth_apps.json".format(
settings))
sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py "
"loaddata geonode/base/fixtures/initial_data.json".format(
settings))
call_task('start_geoserver')
bind = options.get('bind', '0.0.0.0:8000')
foreground = '' if options.get('foreground', False) else '&'
sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runmessaging %s'
% (settings, foreground))
sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runserver %s %s'
% (settings, bind, foreground))
sh('sleep 30')
settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=%s' % settings
live_server_option = ''
info("Running the tests now...")
sh(('%s %s manage.py test %s'
' %s --noinput %s' %
(settings, prefix, name, _keepdb, live_server_option)))
except BuildFailure as e:
info('Tests failed! %s' % str(e))
else:
success = True
finally:
stop(options)
_reset()
if not success:
sys.exit(1)
def package(options):
"""
Creates a tarball to use for building the system elsewhere
"""
import pkg_resources
import tarfile
import geonode
version = geonode.get_version()
# Use GeoNode's version for the package name.
pkgname = 'GeoNode-%s-all' % version
# Create the output directory.
out_pkg = path(pkgname)
out_pkg_tar = path("%s.tar.gz" % pkgname)
# Create a distribution in zip format for the geonode python package.
dist_dir = path('dist')
dist_dir.rmtree()
sh('python setup.py sdist --format=zip')
with pushd('package'):
#Delete old tar files in that directory
for f in glob.glob('GeoNode*.tar.gz'):
old_package = path(f)
if old_package != out_pkg_tar:
old_package.remove()
if out_pkg_tar.exists():
info('There is already a package for version %s' % version)
return
# Clean anything that is in the oupout package tree.
out_pkg.rmtree()
out_pkg.makedirs()
support_folder = path('support')
install_file = path('install.sh')
# And copy the default files from the package folder.
justcopy(support_folder, out_pkg / 'support')
justcopy(install_file, out_pkg)
geonode_dist = path('..') / 'dist' / 'GeoNode-%s.zip' % version
justcopy(geonode_dist, out_pkg)
# Create a tar file with all files in the output package folder.
tar = tarfile.open(out_pkg_tar, "w:gz")
for file in out_pkg.walkfiles():
tar.add(file)
# Add the README with the license and important links to documentation.
tar.add('README', arcname=('%s/README.rst' % out_pkg))
tar.close()
# Remove all the files in the temporary output package directory.
out_pkg.rmtree()
# Report the info about the new package.
info("%s created" % out_pkg_tar.abspath())
def create_jstools_userconfig():
current_dir = path.getcwd()
jst_conf = current_dir / DEFAULT_CFG
if not jst_conf.exists():
info("Creating %s" % jst_conf)
jst_conf.touch()
def deb(options):
"""
Creates debian packages.
Example uses:
paver deb
paver deb -k 12345
paver deb -k 12345 -p geonode/testing
"""
key = options.get('key', None)
ppa = options.get('ppa', None)
import geonode
from geonode.version import get_git_changeset
raw_version = geonode.__version__
version = geonode.get_version()
timestamp = get_git_changeset()
major, minor, revision, stage, edition = raw_version
branch = 'dev'
if stage == 'alpha' and edition == 0:
tail = '%s%s' % (branch, timestamp)
else:
tail = '%s%s' % (stage, edition)
simple_version = '%s.%s.%s+%s' % (major, minor, revision, tail)
info('Creating package for GeoNode version %s' % version)
with pushd('package'):
# Get rid of any uncommitted changes to debian/changelog
info('Getting rid of any uncommitted changes in debian/changelog')
sh('git checkout debian/changelog')
# Workaround for git-dch bug
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580
path('.git').makedirs()
# Install requirements
#sh('sudo apt-get -y install debhelper devscripts git-buildpackage')
sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s'
' --id-length=6 --ignore-branch --release' % (
simple_version)))
## Revert workaround for git-dhc bug
path('.git').rmtree()
if key is None and ppa is None:
# A local installable package
sh('debuild -uc -us -A')
elif key is None and ppa is not None:
# A sources package, signed by daemon
sh('debuild -S')
elif key is not None and ppa is None:
# A signed installable package
sh('debuild -k%s -A' % key)
elif key is not None and ppa is not None:
# A signed, source package
sh('debuild -k%s -S' % key)
if ppa is not None:
sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
_backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND'])
if _backend == 'geonode.geoserver' or 'geonode.qgis_server' not in INSTALLED_APPS:
call_task('stop_geoserver')
_reset()
# Start GeoServer
call_task('start_geoserver')
else:
call_task('stop_qgis_server')
_reset()
# Start QGis Server
call_task('start_qgis_server')
sh('sleep 30')
name = options.get('name', 'geonode.tests.integration')
settings = options.get('settings', '')
if not settings and name == 'geonode.upload.tests.integration':
if _django_11:
sh("cp geonode/upload/tests/test_settings.py geonode/")
settings = 'geonode.test_settings'
else:
settings = 'geonode.upload.tests.test_settings'
success = False
try:
if name == 'geonode.tests.csw':
call_task('sync', options={'settings': settings})
call_task('start', options={'settings': settings})
call_task('setup_data', options={'settings': settings})
settings = 'DJANGO_SETTINGS_MODULE=%s' % settings if settings else ''
if name == 'geonode.upload.tests.integration':
sh("%s python -W ignore manage.py makemigrations --noinput" % settings)
sh("%s python -W ignore manage.py migrate --noinput" % settings)
sh("%s python -W ignore manage.py loaddata sample_admin.json" % settings)
sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/default_oauth_apps.json" %
settings)
sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/initial_data.json" %
settings)
call_task('start_geoserver')
bind = options.get('bind', '0.0.0.0:8000')
foreground = '' if options.get('foreground', False) else '&'
sh('%s python -W ignore manage.py runmessaging %s' % (settings, foreground))
sh('%s python -W ignore manage.py runserver %s %s' %
(settings, bind, foreground))
sh('sleep 30')
settings = 'REUSE_DB=1 %s' % settings
live_server_option = '--liveserver=localhost:8000'
if _django_11:
live_server_option = ''
info("GeoNode is now available, running the tests now.")
sh(('%s python -W ignore manage.py test %s'
' %s --noinput %s' % (settings, name, _keepdb, live_server_option)))
except BuildFailure as e:
info('Tests failed! %s' % str(e))
else:
success = True
finally:
# don't use call task here - it won't run since it already has
stop()
call_task('stop_geoserver')
_reset()
if not success:
sys.exit(1)
def test():
info("Tests are done")
def install_jstools():
info("All Done")
# if there are spaces
javapath = 'START /B "" "' + javapath_opt + '"'
sh((
'%(javapath)s -Xmx512m -XX:MaxPermSize=256m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
# workaround for JAI sealed jar issue and jetty classloader
' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --port %(jetty_port)i'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()
))
info('Starting GeoServer on %s' % url)
# wait for GeoServer to start
started = waitfor(url)
info('The logs are available at %s' % log_file)
if not started:
# If applications did not start in time we will give the user a chance
# to inspect them and stop them manually.
info(('GeoServer never started properly or timed out.'
'It may still be running in the background.'))
sys.exit(1)
@task
def test(options):
def setup(options):
"""Get dependencies and prepare a GeoNode development environment."""
info(('GeoNode development environment successfully set up.'
'If you have not set up an administrative account,'
' please do so now. Use "paver start" to start up the server.'))
def setup():
"""setup plugin dependencies"""
config_file = options.base.home / 'GeoHealthCheck/config_main.py'
config_site = options.base.instance / 'config_site.py'
# setup dirs
if not os.path.exists(options.base.static_lib):
options.base.static_lib.mkdir()
if not os.path.exists(options.base.instance):
options.base.instance.mkdir()
data_dir = options.base.instance / 'data'
data_dir.mkdir()
data_dir.chmod(0777)
# setup config
config_file.copy(config_site)
# setup deps
sh('pip install -r requirements.txt')
# split URL to keep pep8 happy
skin = '/'.join([
'http://github.com', 'IronSummitMedia/startbootstrap-sb-admin-2',
'archive/v1.0.3.zip'
])
skin_dirs = ['dist', 'bower_components']
need_to_fetch = False
for skin_dir in skin_dirs:
skin_dir_path = os.sep.join(
['startbootstrap-sb-admin-2-1.0.3', skin_dir])
if not os.path.exists(skin_dir_path):
need_to_fetch = True
if need_to_fetch:
zipstr = StringIO(urlopen(skin).read())
zipfile_obj = zipfile.ZipFile(zipstr)
zipfile_obj.extractall(options.base.static_lib)
for zf_mem in skin_dirs:
src_loc = path(options.base.static_lib /
'startbootstrap-sb-admin-2-1.0.3' / zf_mem)
dest_loc = path(options.base.static_lib / zf_mem)
if not os.path.exists(dest_loc):
src_loc.move(dest_loc)
else:
info('directory already exists. Skipping')
shutil.rmtree(
path(options.base.static_lib / 'startbootstrap-sb-admin-2-1.0.3'))
# install sparklines to static/site/js
with open(path(options.base.static_lib / 'jspark.js'), 'w') as f:
content = urlopen('http://ejohn.org/files/jspark.js').read()
content.replace('red', 'green')
f.write(content)
# install leafletjs to static/lib
leafletjs = 'http://cdn.leafletjs.com/downloads/leaflet-0.7.5.zip'
zipstr = StringIO(urlopen(leafletjs).read())
zipfile_obj = zipfile.ZipFile(zipstr)
zipfile_obj.extractall(options.base.static_lib / 'leaflet')
# install html5shiv to static/lib
with open(path(options.base.static_lib / 'html5shiv.min.js'), 'w') as f:
url = 'http://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js'
content = urlopen(url).read()
f.write(content)
# install respond to static/lib
with open(path(options.base.static_lib / 'respond.min.js'), 'w') as f:
url = 'http://oss.maxcdn.com/respond/1.4.2/respond.min.js'
content = urlopen(url).read()
f.write(content)
# build i18n .mo files
call_task('compile_translations')
# build local docs
call_task('refresh_docs')
# message user
info('GeoHealthCheck is now built. Edit settings in %s' % config_site)
info('before deploying the application. Alternatively, you can start a')
info('development instance with "python GeoHealthCheck/app.py"')
def setup_qgis_server(options):
"""Prepare a testing instance of QGIS Server."""
# only start if using QGIS Server backend
if 'geonode.qgis_server' not in INSTALLED_APPS or OGC_SERVER['default'][
'BACKEND'] == 'geonode.geoserver':
return
# QGIS Server testing instance run on top of docker
try:
sh('scripts/misc/docker_check.sh')
except BaseException:
info("You need to have docker and docker-compose installed.")
return
info('Docker and docker-compose were installed.')
info('Proceeded to setup QGIS Server.')
info('Create QGIS Server related folder.')
try:
os.makedirs('geonode/qgis_layer')
except BaseException:
pass
try:
os.makedirs('geonode/qgis_tiles')
except BaseException:
pass
all_permission = 0o777
os.chmod('geonode/qgis_layer', all_permission)
stat = os.stat('geonode/qgis_layer')
info('Mode : %o' % stat.st_mode)
os.chmod('geonode/qgis_tiles', all_permission)
stat = os.stat('geonode/qgis_tiles')
info('Mode : %o' % stat.st_mode)
info('QGIS Server related folder successfully setup.')
# The gh_pages submodule need to checkout the
try:
gh_pages = repo.validate_gh_pages_submodule(gh_pages_root)
except ValueError, e:
sys.exit(
'You gh-pages is either not set or set on the wrong branch. \n'
'Update your submodule, checkout the gh-pages branch '
'(cd %s; git checkout -t origin/gh-pages) '
'and rebuild the documentation.' % gh_pages_root)
dry("Add modified and untracked content to git index", gh_pages.git.add,
'.')
if options.gh_pages_update.get('commit_message') is None:
info("No commit message set... "
"You will have to commit the last changes "
"and push them to GitHub")
return
msg = options.gh_pages_update.commit_message
dry('"Commit any changes with message "%s".' % msg, gh_pages.git.commit,
'-m', msg)
dry("Push any changes on the gh-pages branch.", gh_pages.git.push,
remote_name, 'gh-pages')
info('You might want to update your submodule reference:\n\t'
'git add %s\n\tgit commit -m "built html doc updated"' %
options.gh_pages.root)
@task
def gh_pages_clean():
"""Clean your documentation.
def start_geoserver(options):
"""
Start GeoServer with GeoNode extensions
"""
# only start if using Geoserver backend
if 'geonode.geoserver' not in INSTALLED_APPS or OGC_SERVER['default'][
'BACKEND'] == 'geonode.qgis_server':
return
GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION']
url = GEOSERVER_BASE_URL
if urlparse(GEOSERVER_BASE_URL).hostname != 'localhost':
print "Warning: OGC_SERVER['default']['LOCATION'] hostname is not equal to 'localhost'"
if not GEOSERVER_BASE_URL.endswith('/'):
print "Error: OGC_SERVER['default']['LOCATION'] does not end with a '/'"
sys.exit(1)
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / \
os.path.basename(dev_config['JETTY_RUNNER_URL'])
data_dir = path('geoserver/data').abspath()
geofence_dir = path('geoserver/data/geofence').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/misc/jetty-runner.xml').abspath()
jetty_port = urlparse(GEOSERVER_BASE_URL).port
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket_free = True
try:
s.bind(("127.0.0.1", jetty_port))
except socket.error as e:
socket_free = False
if e.errno == 98:
info('Port %s is already in use' % jetty_port)
else:
info(
'Something else raised the socket.error exception while checking port %s'
% jetty_port)
print(e)
finally:
s.close()
if socket_free:
# @todo - we should not have set workdir to the datadir but a bug in geoserver
# prevents geonode security from initializing correctly otherwise
with pushd(data_dir):
javapath = "java"
loggernullpath = os.devnull
# checking if our loggernullpath exists and if not, reset it to
# something manageable
if loggernullpath == "nul":
try:
open("../../downloaded/null.txt", 'w+').close()
except IOError as e:
print "Chances are that you have Geoserver currently running. You \
can either stop all servers with paver stop or start only \
the django application with paver start_django."
sys.exit(1)
loggernullpath = "../../downloaded/null.txt"
try:
sh(('java -version'))
except BaseException:
print "Java was not found in your path. Trying some other options: "
javapath_opt = None
if os.environ.get('JAVA_HOME', None):
print "Using the JAVA_HOME environment variable"
javapath_opt = os.path.join(
os.path.abspath(os.environ['JAVA_HOME']), "bin",
"java.exe")
elif options.get('java_path'):
javapath_opt = options.get('java_path')
else:
print "Paver cannot find java in the Windows Environment. \
Please provide the --java_path flag with your full path to \
java.exe e.g. --java_path=C:/path/to/java/bin/java.exe"
sys.exit(1)
# if there are spaces
javapath = 'START /B "" "' + javapath_opt + '"'
sh((
'%(javapath)s -Xms512m -Xmx2048m -server -XX:+UseConcMarkSweepGC -XX:MaxPermSize=512m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
' -Dgeofence.dir=%(geofence_dir)s'
# ' -Dgeofence-ovr=geofence-datasource-ovr.properties'
# workaround for JAI sealed jar issue and jetty classloader
# ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --port %(jetty_port)i'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()))
info('Starting GeoServer on %s' % url)
# wait for GeoServer to start
started = waitfor(url)
info('The logs are available at %s' % log_file)
if not started:
# If applications did not start in time we will give the user a chance
# to inspect them and stop them manually.
info(('GeoServer never started properly or timed out.'
'It may still be running in the background.'))
sys.exit(1)
def create_secret_key():
"""create secret key for SECRET_KEY in instance/config.py"""
info('Secret key: %s' % os.urandom(24))
info('Copy/paste this key to set the SECRET_KEY')
info('value in instance/config.py')
def deb(options):
"""
Creates debian packages.
Example uses:
paver deb
paver deb -k 12345
paver deb -k 12345 -p geonode/testing
"""
key = options.get('key', None)
ppa = options.get('ppa', None)
version, simple_version = versions()
info('Creating package for GeoNode version %s' % version)
# Get rid of any uncommitted changes to debian/changelog
info('Getting rid of any uncommitted changes in debian/changelog')
sh('git checkout package/debian/changelog')
# Workaround for git-dch bug
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580
sh('rm -rf %s/.git' % (os.path.realpath('package')))
sh('ln -s %s %s' % (os.path.realpath('.git'), os.path.realpath('package')))
with pushd('package'):
# Install requirements
# sh('sudo apt-get -y install debhelper devscripts git-buildpackage')
# sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s'
# ' --id-length=6 --ignore-branch --release' % (simple_version)))
# In case you publish from Ubuntu Xenial (git-dch is removed from upstream)
# use the following line instead:
# sh(('gbp dch --spawn-editor=snapshot --git-author --new-version=%s'
# ' --id-length=6 --ignore-branch --release' % (simple_version)))
distribution = "xenial"
sh(('gbp dch --distribution=%s --force-distribution --spawn-editor=snapshot --git-author --new-version=%s'
' --id-length=6 --ignore-branch --release' %
(distribution, simple_version)))
deb_changelog = path('debian') / 'changelog'
for idx, line in enumerate(
fileinput.input([deb_changelog], inplace=True)):
if idx == 0:
print "geonode (%s) %s; urgency=high" % (simple_version,
distribution),
else:
print line.replace("urgency=medium", "urgency=high"),
# Revert workaround for git-dhc bug
sh('rm -rf .git')
if key is None and ppa is None:
# A local installable package
sh('debuild -uc -us -A')
elif key is None and ppa is not None:
# A sources package, signed by daemon
sh('debuild -S')
elif key is not None and ppa is None:
# A signed installable package
sh('debuild -k%s -A' % key)
elif key is not None and ppa is not None:
# A signed, source package
sh('debuild -k%s -S' % key)
if ppa is not None:
sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))
def start_geoserver(options):
"""
Start GeoServer with GeoNode extensions
"""
# we use docker-compose for integration tests
if on_travis and not options.get('force_exec', False):
return
# only start if using Geoserver backend
if 'geonode.geoserver' not in INSTALLED_APPS:
return
GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION']
url = GEOSERVER_BASE_URL
if urlparse(GEOSERVER_BASE_URL).hostname != 'localhost':
logger.warning(
"Warning: OGC_SERVER['default']['LOCATION'] hostname is not equal to 'localhost'"
)
if not GEOSERVER_BASE_URL.endswith('/'):
logger.error(
"Error: OGC_SERVER['default']['LOCATION'] does not end with a '/'")
sys.exit(1)
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / \
os.path.basename(dev_config['JETTY_RUNNER_URL'])
data_dir = path('geoserver/data').abspath()
geofence_dir = path('geoserver/data/geofence').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/misc/jetty-runner.xml').abspath()
jetty_port = urlparse(GEOSERVER_BASE_URL).port
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket_free = True
try:
s.bind(("127.0.0.1", jetty_port))
except socket.error as e:
socket_free = False
if e.errno == 98:
info(f'Port {jetty_port} is already in use')
else:
info(
'Something else raised the socket.error exception while checking port %s'
% jetty_port)
print(e)
finally:
s.close()
if socket_free:
# @todo - we should not have set workdir to the datadir but a bug in geoserver
# prevents geonode security from initializing correctly otherwise
with pushd(data_dir):
javapath = "java"
if on_travis:
sh(('sudo apt install -y openjdk-8-jre openjdk-8-jdk;'
' sudo update-java-alternatives --set java-1.8.0-openjdk-amd64;'
' export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::");'
' export PATH=$JAVA_HOME\'bin/java\':$PATH;'))
# import subprocess
# result = subprocess.run(['update-alternatives', '--list', 'java'], stdout=subprocess.PIPE)
# javapath = result.stdout
javapath = "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
loggernullpath = os.devnull
# checking if our loggernullpath exists and if not, reset it to
# something manageable
if loggernullpath == "nul":
try:
open("../../downloaded/null.txt", 'w+').close()
except IOError:
print(
"Chances are that you have Geoserver currently running. You "
"can either stop all servers with paver stop or start only "
"the django application with paver start_django.")
sys.exit(1)
loggernullpath = "../../downloaded/null.txt"
try:
sh(('%(javapath)s -version') % locals())
except Exception:
logger.warning(
"Java was not found in your path. Trying some other options: "
)
javapath_opt = None
if os.environ.get('JAVA_HOME', None):
logger.info("Using the JAVA_HOME environment variable")
javapath_opt = os.path.join(
os.path.abspath(os.environ['JAVA_HOME']), "bin",
"java.exe")
elif options.get('java_path'):
javapath_opt = options.get('java_path')
else:
logger.critical(
"Paver cannot find java in the Windows Environment. "
"Please provide the --java_path flag with your full path to "
"java.exe e.g. --java_path=C:/path/to/java/bin/java.exe"
)
sys.exit(1)
# if there are spaces
javapath = 'START /B "" "' + javapath_opt + '"'
sh((
'%(javapath)s -Xms512m -Xmx2048m -server -XX:+UseConcMarkSweepGC -XX:MaxPermSize=512m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
' -DGEOSERVER_CSRF_DISABLED=true'
' -Dgeofence.dir=%(geofence_dir)s'
' -Djava.awt.headless=true'
# ' -Dgeofence-ovr=geofence-datasource-ovr.properties'
# workaround for JAI sealed jar issue and jetty classloader
# ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --port %(jetty_port)i'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()))
info(f'Starting GeoServer on {url}')
# wait for GeoServer to start
started = waitfor(url)
info(f'The logs are available at {log_file}')
if not started:
# If applications did not start in time we will give the user a chance
# to inspect them and stop them manually.
info(('GeoServer never started properly or timed out.'
'It may still be running in the background.'))
sys.exit(1)
def setup():
"""setup plugin dependencies"""
config_file = options.base.home / 'GeoHealthCheck/config_main.py'
config_site = options.base.instance / 'config_site.py'
# setup dirs
if not os.path.exists(options.base.static_lib):
options.base.static_lib.mkdir()
if not os.path.exists(options.base.instance):
options.base.instance.mkdir()
data_dir = options.base.instance / 'data'
data_dir.mkdir()
# data_dir.chmod(0777) gives failure on Python 2.7 Paver 1.2.1
os.chmod(path(data_dir), 0o777)
# setup config
config_file.copy(config_site)
# setup deps
sh('pip install -r requirements.txt')
skin = 'http://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v3.3.7+1.zip' # noqa
skin_dirs = ['dist', 'vendor']
need_to_fetch = False
for skin_dir in skin_dirs:
skin_dir_path = os.sep.join(
['startbootstrap-sb-admin-2-3.3.7-1', skin_dir])
if not os.path.exists(skin_dir_path):
need_to_fetch = True
if need_to_fetch:
zipstr = StringIO(urlopen(skin).read())
zipfile_obj = zipfile.ZipFile(zipstr)
zipfile_obj.extractall(options.base.static_lib)
for zf_mem in skin_dirs:
src_loc = path(options.base.static_lib /
'startbootstrap-sb-admin-2-3.3.7-1' / zf_mem)
dest_loc = path(options.base.static_lib / zf_mem)
if not os.path.exists(dest_loc):
src_loc.move(dest_loc)
else:
info('directory already exists. Skipping')
shutil.rmtree(
path(options.base.static_lib /
'startbootstrap-sb-admin-2-3.3.7-1'))
# install sparklines to static/site/js
with open(path(options.base.static_lib / 'jspark.js'), 'w') as f:
content = urlopen('http://ejohn.org/files/jspark.js').read()
content.replace('red', 'green')
f.write(content)
# install bootstrap-tagsinput to static/lib
info('Getting select2')
select2 = 'https://github.com/select2/select2/archive/4.0.3.zip'
zipstr = StringIO(urlopen(select2).read())
zipfile_obj = zipfile.ZipFile(zipstr)
zipfile_obj.extractall(options.base.static_lib)
dirname = glob.glob(options.base.static_lib / 'select2-*')[0]
dstdir = ''.join(dirname.rsplit('-', 1)[:-1])
try:
os.rename(dirname, dstdir)
except OSError:
shutil.rmtree(dstdir)
os.rename(dirname, dstdir)
# install leafletjs to static/lib
info('Getting leaflet')
leafletjs = 'http://cdn.leafletjs.com/downloads/leaflet-0.7.5.zip'
zipstr = StringIO(urlopen(leafletjs).read())
zipfile_obj = zipfile.ZipFile(zipstr)
zipfile_obj.extractall(options.base.static_lib / 'leaflet')
# install html5shiv to static/lib
with open(path(options.base.static_lib / 'html5shiv.min.js'), 'w') as f:
url = 'http://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js'
content = urlopen(url).read()
f.write(content)
# install respond to static/lib
with open(path(options.base.static_lib / 'respond.min.js'), 'w') as f:
url = 'http://oss.maxcdn.com/respond/1.4.2/respond.min.js'
content = urlopen(url).read()
f.write(content)
# build i18n .mo files
call_task('compile_translations')
# build local docs
call_task('refresh_docs')
# message user
info('GeoHealthCheck is now built. Edit settings in %s' % config_site)
info('before deploying the application. Alternatively, you can start a')
info('development instance with "python GeoHealthCheck/app.py"')
